[RPC] safe destructor when remote server get killed (#1086)

src/runtime/rpc/rpc_device_api.cc

@@ -34,8 +34,12 @@ class RPCDeviceAPI final : public DeviceAPI {
   }
   void FreeDataSpace(TVMContext ctx, void* ptr) final {
     RemoteSpace* space = static_cast<RemoteSpace*>(ptr);
+    try {
       GetSess(ctx)->CallRemote(
           RPCCode::kDevFreeData, ctx, space->data);
+    } catch (const dmlc::Error& e) {
+      // fault tolerance to remote close.
+    }
     delete space;
   }
   void CopyDataFromTo(const void* from,

src/runtime/rpc/rpc_module.cc

@@ -26,7 +26,11 @@ struct RPCWrappedFunc {
     sess_->CallFunc(handle_, args, rv, &fwrap_);
   }
   ~RPCWrappedFunc() {
+    try {
       sess_->CallRemote(RPCCode::kFreeFunc, handle_);
+    } catch (const dmlc::Error& e) {
+      // fault tolerance to remote close
+    }
   }
 
   static void WrapRemote(std::shared_ptr<RPCSession> sess,
@@ -48,7 +52,11 @@ class RPCModuleNode final : public ModuleNode {
   }
   ~RPCModuleNode() {
     if (module_handle_ != nullptr) {
+      try {
         sess_->CallRemote(RPCCode::kModuleFree, module_handle_);
+      } catch (const dmlc::Error& e) {
+        // fault tolerance to remote close
+      }
       module_handle_ = nullptr;
     }
   }