[Runtime][MISRA-C][Bundle] Bundle deployment with static linking (#5158)

* test file for static link added

* rename files

* Fixed static linking issue

* cleanup

* changed to dynamic and static demo

* MISRA-C static and dynamic test

* cleanup

* cleanup

* Update README.md

* cleanup headers

* update readme

apps/bundle_deploy/Makefile

@@ -33,22 +33,36 @@ PKG_LDFLAGS = -pthread
 
 build_dir := build
 
-demo: $(build_dir)/demo $(build_dir)/bundle.so $(build_dir)/bundle_c.so $(build_dir)/cat.bin
-	TVM_NUM_THREADS=1 $(build_dir)/demo $(build_dir)/bundle.so $(build_dir)/cat.bin
-	TVM_NUM_THREADS=1 $(build_dir)/demo $(build_dir)/bundle_c.so $(build_dir)/cat.bin
+demo_dynamic: $(build_dir)/demo_dynamic $(build_dir)/bundle.so $(build_dir)/bundle_c.so $(build_dir)/cat.bin
+	TVM_NUM_THREADS=1 $(build_dir)/demo_dynamic $(build_dir)/bundle.so $(build_dir)/cat.bin
+	TVM_NUM_THREADS=1 $(build_dir)/demo_dynamic $(build_dir)/bundle_c.so $(build_dir)/cat.bin
 
-test: $(build_dir)/test $(build_dir)/test_bundle.so $(build_dir)/test_bundle_c.so $(build_dir)/test_data.bin $(build_dir)/test_output.bin
-	TVM_NUM_THREADS=1 $(build_dir)/test $(build_dir)/test_bundle.so $(build_dir)/test_data.bin $(build_dir)/test_output.bin $(build_dir)/test_graph.json $(build_dir)/test_params.bin
-	TVM_NUM_THREADS=1 $(build_dir)/test $(build_dir)/test_bundle_c.so $(build_dir)/test_data.bin $(build_dir)/test_output.bin $(build_dir)/test_graph.json $(build_dir)/test_params.bin
+test_dynamic: $(build_dir)/test_dynamic $(build_dir)/test_bundle.so $(build_dir)/test_bundle_c.so $(build_dir)/test_data.bin $(build_dir)/test_output.bin
+	TVM_NUM_THREADS=1 $(build_dir)/test_dynamic $(build_dir)/test_bundle.so $(build_dir)/test_data.bin $(build_dir)/test_output.bin $(build_dir)/test_graph.json $(build_dir)/test_params.bin
+	TVM_NUM_THREADS=1 $(build_dir)/test_dynamic $(build_dir)/test_bundle_c.so $(build_dir)/test_data.bin $(build_dir)/test_output.bin $(build_dir)/test_graph.json $(build_dir)/test_params.bin
 
-$(build_dir)/demo: demo.cc ${build_dir}/graph.json.c ${build_dir}/params.bin.c
+demo_static: $(build_dir)/demo_static $(build_dir)/cat.bin
+	TVM_NUM_THREADS=1 $(build_dir)/demo_static $(build_dir)/cat.bin
+
+test_static: $(build_dir)/test_static $(build_dir)/test_data.bin $(build_dir)/test_output.bin
+	TVM_NUM_THREADS=1 $(build_dir)/test_static $(build_dir)/test_data.bin $(build_dir)/test_output.bin $(build_dir)/test_graph.json $(build_dir)/test_params.bin
+
+$(build_dir)/demo_dynamic: demo.cc ${build_dir}/graph.json.c ${build_dir}/params.bin.c
 	@mkdir -p $(@D)
 	g++ $(PKG_CXXFLAGS) -o $@  demo.cc -ldl
 
-$(build_dir)/test: test.cc ${build_dir}/test_graph.json ${build_dir}/test_params.bin
+$(build_dir)/test_dynamic: test.cc ${build_dir}/test_graph.json ${build_dir}/test_params.bin
 	@mkdir -p $(@D)
 	g++ $(PKG_CXXFLAGS) -o $@  test.cc -ldl
 
+$(build_dir)/demo_static: demo_static.c ${build_dir}/bundle_static.o ${build_dir}/model.o ${build_dir}/graph.json.c ${build_dir}/params.bin.c
+	@mkdir -p $(@D)
+	gcc $(PKG_CXXFLAGS) -o $@ demo_static.c ${build_dir}/bundle_static.o ${build_dir}/model.o -lm
+
+$(build_dir)/test_static: test_static.c ${build_dir}/bundle_static.o ${build_dir}/test_model.o
+	@mkdir -p $(@D)
+	gcc $(PKG_CXXFLAGS) -o $@ $^
+
 # Serialize our graph.json file.
 $(build_dir)/graph.json.c: $(build_dir)/graph.json
 	xxd -i $^  > $@
@@ -89,6 +103,10 @@ $(build_dir)/test_bundle_c.so: bundle.c runtime.c $(build_dir)/test_model.o
 	@mkdir -p $(@D)
 	gcc -shared $(PKG_CFLAGS) -fvisibility=hidden -o $@  $^ $(PKG_LDFLAGS)
 
+$(build_dir)/bundle_static.o: bundle_static.c
+	@mkdir -p $(@D)
+	gcc -c $(PKG_CFLAGS) -o $@  $^
+
 clean:
 	rm -rf $(build_dir)/bundle.so $(build_dir)/bundle_c.so $(build_dir)/test_bundle.so $(build_dir)/test_bundle_c.so
 

apps/bundle_deploy/README.md

@@ -39,7 +39,7 @@ Type the following command to run the sample code under the current folder,
 after building TVM first.
 
 ```bash
-make demo
+make demo_dynamic
 ```
 
 This will:
@@ -48,7 +48,20 @@ This will:
 - Compile the model with Relay
 - Build a `bundle.so` shared object containing the model specification and
   parameters
-- Build a `demo` executable that `dlopen`'s `bundle.so` (or `bundle_c.so` in 
+- Build a `demo_dynamic` executable that `dlopen`'s `bundle.so` (or `bundle_c.so` in 
   terms of the MISRA-C runtime), instantiates the contained graph runtime,
   and invokes the `GraphRuntime::Run` function on a cat image, then prints
   the output results.
+
+Type the following command to run the sample code with static linking.
+  
+```bash
+make demo_static
+```
+
+This will:
+- Download the mobilenet0.25 model from the MXNet Gluon Model Zoo
+- Compile the model with Relay and outputs `model.o`
+- Build a `bundle_static.o` object containing the runtime functions
+- Build a `demo_static` executable which has static link to `bundle_static.o` and 
+  `model.o`, functions on a cat image, then prints the output results.

apps/bundle_deploy/bundle.h

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef TVM_APPS_BUNDLE_DEPLOY_BUNDLE_H_
+#define TVM_APPS_BUNDLE_DEPLOY_BUNDLE_H_
+
+#include <tvm/runtime/c_runtime_api.h>
+
+TVM_DLL void * tvm_runtime_create(const char * json_data,
+                                  const char * params_data,
+                                  const uint64_t params_size);
+
+TVM_DLL void tvm_runtime_destroy(void * runtime);
+
+TVM_DLL void tvm_runtime_set_input(void * runtime,
+                                   const char * name, 
+                                   DLTensor * tensor);
+
+TVM_DLL void tvm_runtime_run(void * runtime);
+
+TVM_DLL void tvm_runtime_get_output(void * runtime,
+                                    int32_t index, 
+                                    DLTensor * tensor);
+
+#endif /* TVM_APPS_BUNDLE_DEPLOY_BUNDLE_H_ */

apps/bundle_deploy/bundle_static.c

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "bundle.h"
+#include "runtime.c"
+
+TVM_DLL void * tvm_runtime_create(const char * json_data,
+                                  const char * params_data,
+                                  const uint64_t params_size) {
+  int64_t device_type = kDLCPU;
+  int64_t device_id = 0;
+
+  TVMByteArray params;
+  params.data = params_data;
+  params.size = params_size;
+
+  TVMContext ctx;
+  ctx.device_type = (DLDeviceType)device_type;
+  ctx.device_id = device_id;
+
+  // declare pointers
+  void * (*SystemLibraryCreate)();
+  TVMGraphRuntime * (*TVMGraphRuntimeCreate)(const char *, const TVMModuleHandle, const TVMContext *);
+  int (*TVMGraphRuntime_LoadParams)(TVMModuleHandle, const char *, const uint32_t);
+
+  // get pointers
+  TVMFuncGetGlobal("runtime.SystemLib", (TVMFunctionHandle*)&SystemLibraryCreate);
+  TVMFuncGetGlobal("tvm.graph_runtime.create", (TVMFunctionHandle*)&TVMGraphRuntimeCreate);
+
+  // run modules
+  TVMModuleHandle mod_syslib = SystemLibraryCreate();
+  TVMModuleHandle mod = TVMGraphRuntimeCreate(json_data, mod_syslib, &ctx);
+  TVMModGetFunction(mod, "load_params", 0, (TVMFunctionHandle*)&TVMGraphRuntime_LoadParams);
+  TVMGraphRuntime_LoadParams(mod, params.data, params.size);
+  
+  return mod;
+}
+
+TVM_DLL void tvm_runtime_destroy(void * runtime) {
+  void (*TVMGraphRuntimeRelease)(TVMModuleHandle *);
+  TVMFuncGetGlobal("tvm.graph_runtime.release", (TVMFunctionHandle*)&TVMGraphRuntimeRelease);
+  TVMGraphRuntimeRelease(&runtime);
+}
+
+TVM_DLL void tvm_runtime_set_input(void * runtime, const char * name, DLTensor * tensor) {
+  void (*TVMGraphRuntime_SetInput)(TVMModuleHandle, const char *, DLTensor*);
+  TVMFuncGetGlobal("tvm.graph_runtime.set_input", (TVMFunctionHandle*)&TVMGraphRuntime_SetInput);
+  TVMGraphRuntime_SetInput(runtime, name, tensor);
+}
+
+TVM_DLL void tvm_runtime_run(void * runtime) {
+  void (*TVMGraphRuntime_Run)(TVMModuleHandle runtime);
+  TVMFuncGetGlobal("tvm.graph_runtime.run", (TVMFunctionHandle*)&TVMGraphRuntime_Run);
+  TVMGraphRuntime_Run(runtime);
+}
+
+TVM_DLL void tvm_runtime_get_output(void * runtime, int32_t index, DLTensor * tensor) {
+  int (*TVMGraphRuntime_GetOutput)(TVMModuleHandle, const int32_t, DLTensor *);
+  TVMFuncGetGlobal("tvm.graph_runtime.get_output", (TVMFunctionHandle*)&TVMGraphRuntime_GetOutput);
+  TVMGraphRuntime_GetOutput(runtime, index, tensor);
+}
\ No newline at end of file

apps/bundle_deploy/demo_static.c

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+
+#include <assert.h>
+#include <stdio.h>
+#include <sys/time.h>
+#include <float.h>
+
+#include "bundle.h"
+#include "build/graph.json.c"
+#include "build/params.bin.c"
+
+#define OUTPUT_LEN  1000
+
+int main(int argc, char **argv) {
+  assert(argc == 2 && "Usage: demo_static <cat.bin>");
+
+  char * json_data = (char *)(build_graph_json);
+  char * params_data = (char *)(build_params_bin);
+  uint64_t params_size = build_params_bin_len;
+
+  struct timeval t0, t1, t2, t3, t4, t5;
+  gettimeofday(&t0, 0);
+
+  auto *handle = tvm_runtime_create(json_data, params_data, params_size);
+  gettimeofday(&t1, 0);
+
+  float input_storage[1 * 3 * 224 * 224];
+  FILE * fp = fopen(argv[1], "rb");
+  fread(input_storage, 3 * 224 * 224, 4, fp);
+  fclose(fp);
+
+  DLTensor input;
+  input.data = input_storage;
+  DLContext ctx = {kDLCPU, 0};
+  input.ctx = ctx;
+  input.ndim = 4;
+  DLDataType dtype = {kDLFloat, 32, 1};
+  input.dtype = dtype;
+  int64_t shape [4] = {1, 3, 224, 224};
+  input.shape = &shape;
+  input.strides = NULL;
+  input.byte_offset = 0;
+
+  tvm_runtime_set_input(handle, "data", &input);
+  gettimeofday(&t2, 0);
+
+  tvm_runtime_run(handle);
+  gettimeofday(&t3, 0);
+
+  float output_storage[OUTPUT_LEN];
+  DLTensor output;
+  output.data = output_storage;
+  DLContext out_ctx = {kDLCPU, 0};
+  output.ctx = out_ctx;
+  output.ndim = 2;
+  DLDataType out_dtype = {kDLFloat, 32, 1};
+  output.dtype = out_dtype;
+  int64_t out_shape [2] = {1, OUTPUT_LEN};
+  output.shape = &out_shape;
+  output.strides = NULL;
+  output.byte_offset = 0;
+
+  tvm_runtime_get_output(handle, 0, &output);
+  gettimeofday(&t4, 0);
+
+  float max_iter = -FLT_MAX;
+  int32_t max_index = -1;
+  for (auto i = 0; i < OUTPUT_LEN; ++i) {
+    if (output_storage[i] > max_iter) {
+      max_iter = output_storage[i];
+      max_index = i;
+    }
+  }
+
+  tvm_runtime_destroy(handle);
+  gettimeofday(&t5, 0);
+
+  printf("The maximum position in output vector is: %d, with max-value %f.\n",
+         max_index, max_iter);
+  printf("timing: %.2f ms (create), %.2f ms (set_input), %.2f ms (run), "
+         "%.2f ms (get_output), %.2f ms (destroy)\n",
+         (t1.tv_sec-t0.tv_sec)*1000000 + (t1.tv_usec-t0.tv_usec)/1000.f,
+         (t2.tv_sec-t1.tv_sec)*1000000 + (t2.tv_usec-t1.tv_usec)/1000.f,
+         (t3.tv_sec-t2.tv_sec)*1000000 + (t3.tv_usec-t2.tv_usec)/1000.f,
+         (t4.tv_sec-t3.tv_sec)*1000000 + (t4.tv_usec-t3.tv_usec)/1000.f,
+         (t5.tv_sec-t4.tv_sec)*1000000 + (t5.tv_usec-t4.tv_usec)/1000.f);
+  
+  return 0;
+}

apps/bundle_deploy/test_static.c

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <tvm/runtime/c_runtime_api.h>
+
+#include <assert.h>
+#include <stdio.h>
+#include <sys/time.h>
+#include <sys/stat.h>
+
+#include "bundle.h"
+
+
+int main(int argc, char **argv) {
+  assert(argc == 5 && "Usage: test_static <data.bin> <output.bin> <graph.json> <params.bin>");
+
+  struct stat st;
+  char * json_data;
+  char * params_data;
+  uint64_t params_size;
+
+  FILE * fp = fopen(argv[3], "rb");
+  stat(argv[3], &st);
+  json_data = (char*)malloc(st.st_size);
+  fread(json_data, st.st_size, 1, fp);
+  fclose(fp);
+
+  fp = fopen(argv[4], "rb");
+  stat(argv[4], &st);
+  params_data = (char*)malloc(st.st_size);
+  fread(params_data, st.st_size, 1, fp);
+  params_size = st.st_size;
+  fclose(fp);
+
+  struct timeval t0, t1, t2, t3, t4, t5;
+  gettimeofday(&t0, 0);
+
+  auto *handle = tvm_runtime_create(json_data, params_data, params_size);
+  gettimeofday(&t1, 0);
+
+  float input_storage[10 * 5];
+  fp = fopen(argv[1], "rb");
+  fread(input_storage, 10 * 5, 4, fp);
+  fclose(fp);
+
+  float result_storage[10 * 5];
+  fp = fopen(argv[2], "rb");
+  fread(result_storage, 10 * 5, 4, fp);
+  fclose(fp);
+
+  DLTensor input;
+  input.data = input_storage;
+  DLContext ctx = {kDLCPU, 0};
+  input.ctx = ctx;
+  input.ndim = 2;
+  DLDataType dtype = {kDLFloat, 32, 1};
+  input.dtype = dtype;
+  int64_t shape [2] = {10, 5};
+  input.shape = &shape;
+  input.strides = NULL;
+  input.byte_offset = 0;
+
+  tvm_runtime_set_input(handle, "x", &input);
+  gettimeofday(&t2, 0);
+
+  tvm_runtime_run(handle);
+  gettimeofday(&t3, 0);
+
+  float output_storage[10 * 5];
+  DLTensor output;
+  output.data = output_storage;
+  DLContext out_ctx = {kDLCPU, 0};
+  output.ctx = out_ctx;
+  output.ndim = 2;
+  DLDataType out_dtype = {kDLFloat, 32, 1};
+  output.dtype = out_dtype;
+  int64_t out_shape [2] = {10, 5};
+  output.shape = &out_shape;
+  output.strides = NULL;
+  output.byte_offset = 0;
+  
+  tvm_runtime_get_output(handle, 0, &output);
+  gettimeofday(&t4, 0);
+
+  for (auto i = 0; i < 10 * 5; ++i) {
+    assert(fabs(output_storage[i] - result_storage[i]) < 1e-5f);
+    if (fabs(output_storage[i] - result_storage[i]) >= 1e-5f) {
+      printf("got %f, expected %f\n", output_storage[i], result_storage[i]);
+    }
+  }
+
+  tvm_runtime_destroy(handle);
+  gettimeofday(&t5, 0);
+
+  printf("timing: %.2f ms (create), %.2f ms (set_input), %.2f ms (run), "
+         "%.2f ms (get_output), %.2f ms (destroy)\n",
+         (t1.tv_sec-t0.tv_sec)*1000000 + (t1.tv_usec-t0.tv_usec)/1000.f,
+         (t2.tv_sec-t1.tv_sec)*1000000 + (t2.tv_usec-t1.tv_usec)/1000.f,
+         (t3.tv_sec-t2.tv_sec)*1000000 + (t3.tv_usec-t2.tv_usec)/1000.f,
+         (t4.tv_sec-t3.tv_sec)*1000000 + (t4.tv_usec-t3.tv_usec)/1000.f,
+         (t5.tv_sec-t4.tv_sec)*1000000 + (t5.tv_usec-t4.tv_usec)/1000.f);
+
+  free(json_data);
+  free(params_data);
+  
+  return 0;
+}

tests/scripts/task_python_integration.sh

@@ -33,7 +33,7 @@ make cython3
 # Test MISRA-C runtime
 cd apps/bundle_deploy
 rm -rf build
-make test
+make test_dynamic test_static
 cd ../..
 
 # Test extern package