Converted LOG(FATAL) into regular errors to prevent the process from crashing

on error.

PiperOrigin-RevId: 166257105

Author: benoitsteiner

tensorflow/core/common_runtime/gpu/gpu_device.cc

@@ -671,7 +671,10 @@ Status BaseGPUDeviceFactory::CreateGPUDevice(const SessionOptions& options,
   }
 
   int64 total_memory, available_memory;
-  CHECK(se->DeviceMemoryUsage(&available_memory, &total_memory));
+  if (!se->DeviceMemoryUsage(&available_memory, &total_memory)) {
+    return errors::Unknown(
+        strings::StrCat("Failed to query available memory for GPU ", gpu_id));
+  }
 
   int64 allocated_memory;
   double config_memory_fraction =

tensorflow/core/common_runtime/gpu/gpu_util.cc

@@ -354,7 +354,7 @@ Status GPUUtil::Sync(Device* gpu_device) {
   }
   dev_info->stream->BlockHostUntilDone();
   if (!dev_info->stream->ok()) {
-    LOG(FATAL) << "GPU sync failed";
+    return errors::Internal("GPU sync failed");
   }
   return Status::OK();
 }
@@ -367,7 +367,7 @@ Status GPUUtil::SyncAll(Device* gpu_device) {
   }
   if (!dev_info->stream->parent()->SynchronizeAllActivity() ||
       !dev_info->stream->ok()) {
-    LOG(FATAL) << "GPU sync failed";
+    return errors::Internal("GPU sync failed");
   }
   return Status::OK();
 }