[VTA][TSIM][Build] Towards TSIM CI testing (#3704)

* building TSIM specific library along with fast simulator to quickly switch between dlls

* cmake controlled TSIM libraries

* always build tsim driver in either simulation modes

* build DLLs based on CMAKE flags

* updating the jenkinsfile

* small restructuring

* reducing the cmake flags

* update instructions

* reverting to 3 flags

* update Jenkinsfile

* adding new line

* enabling TSIM unit and integration tests

* fix description

* temporarily disabling task_python_vta tests in CPU Build stage

* move CPU tests in unit test stage

* stage  reorg

* better make

* disabling TSIM tests for now

* reverting some restructuring

* fix

Jenkinsfile

@@ -48,7 +48,10 @@ tvm_runtime = "build/libtvm_runtime.so, build/config.cmake"
 tvm_lib = "build/libtvm.so, " + tvm_runtime
 // LLVM upstream lib
 tvm_multilib = "build/libtvm.so, " +
-             "build/libvta.so, build/libtvm_topi.so, build/libnnvm_compiler.so, " + tvm_runtime
+               "build/libvta_tsim.so, " +
+               "build/libvta_fsim.so, " +
+               "build/libtvm_topi.so, " +
+               "build/libnnvm_compiler.so, " + tvm_runtime
 
 // command to start a docker container
 docker_run = 'docker/bash.sh'
@@ -190,11 +193,11 @@ stage('Build') {
         make(ci_cpu, 'build', '-j2')
         pack_lib('cpu', tvm_lib)
         timeout(time: max_time, unit: 'MINUTES') {
-          sh "${docker_run} ${ci_cpu} ./tests/scripts/task_python_vta.sh"
           sh "${docker_run} ${ci_cpu} ./tests/scripts/task_rust.sh"
           sh "${docker_run} ${ci_cpu} ./tests/scripts/task_golang.sh"
           sh "${docker_run} ${ci_cpu} ./tests/scripts/task_python_unittest.sh"
           sh "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh"
+          sh "${docker_run} ${ci_cpu} ./tests/scripts/task_python_vta.sh"
         }
       }
     }

cmake/config.cmake

@@ -137,3 +137,12 @@ set(USE_ANTLR OFF)
 
 # Whether use Relay debug mode
 set(USE_RELAY_DEBUG OFF)
+
+# Whether to build fast VTA simulator driver
+set(USE_VTA_FSIM ON)
+
+# Whether to build cycle-accurate VTA simulator driver
+set(USE_VTA_TSIM ON)
+
+# Whether to build VTA FPGA driver (device side only)
+set(USE_VTA_FPGA OFF)

cmake/modules/VTA.cmake

@@ -37,44 +37,61 @@ elseif(PYTHON)
 
   string(REGEX MATCHALL "(^| )-D[A-Za-z0-9_=.]*" VTA_DEFINITIONS "${__vta_defs}")
 
-  file(GLOB VTA_RUNTIME_SRCS vta/src/*.cc)
-  # Add sim driver sources
-  if(${VTA_TARGET} STREQUAL "sim")
-    file(GLOB __vta_target_srcs vta/src/sim/*.cc)
-  endif()
-  # Add tsim driver sources
-  if(${VTA_TARGET} STREQUAL "tsim")
-    file(GLOB __vta_target_srcs vta/src/tsim/*.cc)
-    file(GLOB RUNTIME_DPI_SRCS vta/src/dpi/module.cc)
-    list(APPEND RUNTIME_SRCS ${RUNTIME_DPI_SRCS})
-  endif()
-  # Add pynq driver sources
-  if(${VTA_TARGET} STREQUAL "pynq" OR ${VTA_TARGET} STREQUAL "ultra96")
-    file(GLOB __vta_target_srcs vta/src/pynq/*.cc)
+  # Fast simulator driver build
+  if(USE_VTA_FSIM)
+    # Add fsim driver sources
+    file(GLOB FSIM_RUNTIME_SRCS vta/src/*.cc)
+    list(APPEND FSIM_RUNTIME_SRCS vta/src/sim/sim_driver.cc)
+    # Target lib: vta_fsim
+    add_library(vta_fsim SHARED ${FSIM_RUNTIME_SRCS})
+    target_include_directories(vta_fsim PUBLIC vta/include)
+    foreach(__def ${VTA_DEFINITIONS})
+      string(SUBSTRING ${__def} 3 -1 __strip_def)
+      target_compile_definitions(vta_fsim PUBLIC ${__strip_def})
+    endforeach()
+    include_directories("vta/include")
+    if(APPLE)
+      set_target_properties(vta_fsim PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+    endif(APPLE)
   endif()
-  list(APPEND VTA_RUNTIME_SRCS ${__vta_target_srcs})
-
-  add_library(vta SHARED ${VTA_RUNTIME_SRCS})
 
-  target_include_directories(vta PUBLIC vta/include)
-
-  foreach(__def ${VTA_DEFINITIONS})
-    string(SUBSTRING ${__def} 3 -1 __strip_def)
-    target_compile_definitions(vta PUBLIC ${__strip_def})
-  endforeach()
-
-  # Enable tsim macro
-  if(${VTA_TARGET} STREQUAL "tsim")
+  # Cycle accurate simulator driver build
+  if(USE_VTA_TSIM)
+    # Add tsim driver sources
+    file(GLOB TSIM_RUNTIME_SRCS vta/src/*.cc)
+    list(APPEND TSIM_RUNTIME_SRCS vta/src/tsim/tsim_driver.cc)
+    list(APPEND TSIM_RUNTIME_SRCS vta/src/dpi/module.cc)
+    # Target lib: vta_tsim
+    add_library(vta_tsim SHARED ${TSIM_RUNTIME_SRCS})
+    target_include_directories(vta_tsim PUBLIC vta/include)
+    foreach(__def ${VTA_DEFINITIONS})
+      string(SUBSTRING ${__def} 3 -1 __strip_def)
+      target_compile_definitions(vta_tsim PUBLIC ${__strip_def})
+    endforeach()
     include_directories("vta/include")
-    target_compile_definitions(vta PUBLIC USE_TSIM)
+    # Set USE_TSIM macro
+    target_compile_definitions(vta_tsim PUBLIC USE_TSIM)
+    if(APPLE)
+      set_target_properties(vta_tsim PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+    endif(APPLE)
   endif()
 
-  if(APPLE)
-    set_target_properties(vta PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
-  endif(APPLE)
-
-  # PYNQ rules for Pynq v2.4
-  if(${VTA_TARGET} STREQUAL "pynq" OR ${VTA_TARGET} STREQUAL "ultra96")
+  # VTA FPGA driver sources
+  if(USE_VTA_FPGA)
+    file(GLOB FPGA_RUNTIME_SRCS vta/src/*.cc)
+    # Rules for Zynq-class FPGAs with pynq OS support (see pynq.io)
+    if(${VTA_TARGET} STREQUAL "pynq" OR
+       ${VTA_TARGET} STREQUAL "ultra96")
+      file(GLOB FPGA_RUNTIME_SRCS vta/src/pynq/pynq_driver.cc)
+    endif()
+    # Target lib: vta
+    add_library(vta SHARED ${FPGA_RUNTIME_SRCS})
+    target_include_directories(vta PUBLIC vta/include)
+    foreach(__def ${VTA_DEFINITIONS})
+      string(SUBSTRING ${__def} 3 -1 __strip_def)
+      target_compile_definitions(vta PUBLIC ${__strip_def})
+    endforeach()
+    # Rules for Pynq v2.4
     find_library(__cma_lib NAMES cma PATH /usr/lib)
     target_link_libraries(vta ${__cma_lib})
   endif()

docs/vta/install.md

@@ -117,6 +117,9 @@ ssh xilinx@192.168.2.99
 cd /home/xilinx/tvm
 mkdir build
 cp cmake/config.cmake build/.
+echo 'set(USE_VTA_FSIM OFF)' >> build/config.cmake
+echo 'set(USE_VTA_TSIM OFF)' >> build/config.cmake
+echo 'set(USE_VTA_FPGA ON)' >> build/config.cmake
 # Copy pynq specific configuration
 cp vta/config/pynq_sample.json vta/config/vta_config.json
 cd build

tests/scripts/task_python_vta.sh

@@ -36,16 +36,16 @@ echo "Running integration test in fsim..."
 python3 -m nose -v vta/tests/python/integration
 
 # # Build VTA chisel design and verilator simulator
-# cd vta/hardware/chisel/ && make && cd -
+# (make -C vta/hardware/chisel/)
 
 # # Set default VTA config to use TSIM cycle accurate sim
 # cp vta/config/tsim_sample.json vta/config/vta_config.json
 
-# # Run unit tests in functional/fast simulator
+# # Run unit tests in cycle accurate simulator
 # echo "Running unittest in tsim..."
 # python3 -m nose -v vta/tests/python/unittest
 
-# # Run unit tests in functional/fast simulator
+# # Run unit tests in cycle accurate simulator
 # echo "Running integration test in tsim..."
 # python3 -m nose -v vta/tests/python/integration
 

vta/python/vta/testing/simulator.py

@@ -21,28 +21,32 @@ import tvm
 from ..environment import get_env
 from ..libinfo import find_libvta
 
-
 def _load_sw():
-    """Load software library, assuming they are simulator."""
-    lib_sw = find_libvta("libvta", optional=True)
-    if not lib_sw:
-        return []
+    """Load hardware library for simulator."""
+
+    env = get_env()
+    lib_driver_name = "libvta_tsim" if env.TARGET == "tsim" else "libvta_fsim"
+
+    # Load driver library
+    lib_driver = find_libvta(lib_driver_name, optional=True)
+    assert lib_driver
     try:
-        return [ctypes.CDLL(lib_sw[0], ctypes.RTLD_GLOBAL)]
+        libs = [ctypes.CDLL(lib_driver[0], ctypes.RTLD_GLOBAL)]
     except OSError:
         return []
 
-
-def _load_all():
-    """Load hardware library for tsim."""
-    lib = _load_sw()
-    env = get_env()
     if env.TARGET == "tsim":
-        lib = find_libvta("libvta_hw", optional=True)
-        f = tvm.get_global_func("vta.tsim.init")
-        m = tvm.module.load(lib[0], "vta-tsim")
-        f(m)
-    return lib
+        lib_hw = find_libvta("libvta_hw", optional=True)
+        assert lib_hw # make sure to build vta/hardware/chisel
+        try:
+            f = tvm.get_global_func("vta.tsim.init")
+            m = tvm.module.load(lib_hw[0], "vta-tsim")
+            f(m)
+            return lib_hw
+        except OSError:
+            return []
+
+    return libs
 
 
 def enabled():
@@ -91,4 +95,4 @@ def debug_mode(flag):
     tvm.get_global_func("vta.simulator.profiler_debug_mode")(flag)
 
 
-LIBS = _load_all()
+LIBS = _load_sw()