[RFC] Pytest environment improvements (#5421)

* [RFC] Pass pytest options globally.

In many places having a global pytest flag is useful . For me with the
build and test of tvm , I would like to be able to globally pass in
pytest options as part of development flow or CI flows where one would
like to measure other things regularly that need measurements including
pytest coverage data that I would like to experiment with across the stack.

This has been achieved with an additional setup-pytest-env.sh file in
tests/scripts rather than putting in something in every single task test
script and something I would like to avoid.

This now means the -v option to pytest is superfluous. I did consider
having a pytest.ini file but that doesn't allow me to pass any old
environment variable in and this seems to be the compromise.

* Improve other use case documentation

* Rationalize pytest environment.

* Remove the setting from docker/with_same_user.
* Take the opportunity to migrate common PYTHONPATH and
TVM_PATH into the common environment setting.

* Fixup vta fsim

* Be more explicit with common PYTHONPATH

* Fix python path for task_python_vta_fsim.sh properly

* Fix nit in documentation.

docker/bash.sh

@@ -89,6 +89,7 @@ ${DOCKER_BINARY} run --rm --pid=host\
     -e "CI_BUILD_GROUP=$(id -g -n)" \
     -e "CI_BUILD_GID=$(id -g)" \
     -e "PYTHONPATH=python:topi/python"\
+    -e "CI_PYTEST_ADD_OPTIONS=$CI_PYTEST_ADD_OPTIONS" \
     ${CUDA_ENV}\
     ${CI_DOCKER_EXTRA_PARAMS[@]} \
     ${DOCKER_IMAGE_NAME}\

docker/build.sh

@@ -162,6 +162,7 @@ ${DOCKER_BINARY} run --rm --pid=host \
     -e "CI_BUILD_UID=$(id -u)" \
     -e "CI_BUILD_GROUP=$(id -g -n)" \
     -e "CI_BUILD_GID=$(id -g)" \
+    -e "CI_PYTEST_ADD_OPTIONS=$CI_PYTEST_ADD_OPTIONS" \
     ${CUDA_ENV}\
     ${CI_DOCKER_EXTRA_PARAMS[@]} \
     ${DOCKER_IMG_NAME} \

docker/with_the_same_user

@@ -41,6 +41,7 @@ getent passwd "${CI_BUILD_UID}" || adduser --gid "${CI_BUILD_GID}" --uid "${CI_B
     --gecos "${CI_BUILD_USER} (generated by with_the_same_user script)" \
     --disabled-password --home "${CI_BUILD_HOME}" --quiet "${CI_BUILD_USER}"
 usermod -a -G sudo "${CI_BUILD_USER}"
+# This is a grotesque hack to get PYTEST_ADD_OPTS available to all task scripts.
 echo "${CI_BUILD_USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-nopasswd-sudo
 
 if [[ ! -z $CUDA_VISIBLE_DEVICES ]]; then

docs/contribute/pull_request.rst

@@ -118,3 +118,6 @@ If you want to run a single test:
   rm -rf python/tvm/*.pyc python/tvm/*/*.pyc python/tvm/*/*/*.pyc
 
   TVM_FFI=ctypes python -m pytest -v tests/python/unittest/test_pass_storage_rewrite.py
+
+  # Additionally if you want to run a single test, for example test_all_elemwise inside a file.
+  TVM_FFI=ctypes python -m pytest -v -k "test_all_elemwise" tests/python/frontend/tflite/test_forward.py

tests/scripts/setup-pytest-env.sh

+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -u
+set -e
+export TVM_PATH=`pwd`
+export PYTHONPATH=${TVM_PATH}/python:${TVM_PATH}/topi/python
+export PYTEST_ADDOPTS="-v $CI_PYTEST_ADD_OPTIONS"

tests/scripts/task_python_docs.sh

@@ -19,6 +19,7 @@
 set -e
 set -u
 
+source tests/scripts/setup-pytest-env.sh
 # cleanup old states
 rm -rf docs/_build
 mkdir -p docs/_build/html

tests/scripts/task_python_frontend.sh

@@ -19,7 +19,7 @@
 set -e
 set -u
 
-export PYTHONPATH=python:topi/python
+source tests/scripts/setup-pytest-env.sh
 # to avoid openblas threading error
 export TVM_BIND_THREADS=0
 export OMP_NUM_THREADS=1
@@ -30,28 +30,28 @@ find . -type f -path "*.pyc" | xargs rm -f
 make cython3
 
 echo "Running relay TFLite frontend test..."
-python3 -m pytest -v tests/python/frontend/tflite
+python3 -m pytest tests/python/frontend/tflite
 
 echo "Running relay MXNet frontend test..."
-python3 -m pytest -v tests/python/frontend/mxnet
+python3 -m pytest tests/python/frontend/mxnet
 
 echo "Running relay Keras frontend test..."
-python3 -m pytest -v tests/python/frontend/keras
+python3 -m pytest tests/python/frontend/keras
 
 echo "Running relay ONNX frontend test..."
-python3 -m pytest -v tests/python/frontend/onnx
+python3 -m pytest tests/python/frontend/onnx
 
 echo "Running relay CoreML frontend test..."
-python3 -m pytest -v tests/python/frontend/coreml
+python3 -m pytest tests/python/frontend/coreml
 
 echo "Running relay Tensorflow frontend test..."
-python3 -m pytest -v tests/python/frontend/tensorflow
+python3 -m pytest tests/python/frontend/tensorflow
 
 echo "Running relay caffe2 frontend test..."
-python3 -m pytest -v tests/python/frontend/caffe2
+python3 -m pytest tests/python/frontend/caffe2
 
 echo "Running relay DarkNet frontend test..."
-python3 -m pytest -v tests/python/frontend/darknet
+python3 -m pytest tests/python/frontend/darknet
 
 echo "Running relay PyTorch frontend test..."
-python3 -m pytest -v tests/python/frontend/pytorch
+python3 -m pytest tests/python/frontend/pytorch

tests/scripts/task_python_integration.sh

@@ -19,7 +19,8 @@
 set -e
 set -u
 
-export PYTHONPATH=`pwd`/python:`pwd`/topi/python:`pwd`/apps/extension/python
+source tests/scripts/setup-pytest-env.sh
+export PYTHONPATH=${PYTHONPATH}:${TVM_PATH}/apps/extension/python
 export LD_LIBRARY_PATH="build:${LD_LIBRARY_PATH:-}"
 export TVM_BIND_THREADS=0
 export TVM_NUM_THREADS=2
@@ -42,26 +43,26 @@ rm -rf lib
 make
 cd ../..
 
-TVM_FFI=cython python3 -m pytest -v apps/extension/tests
-TVM_FFI=ctypes python3 -m pytest -v apps/extension/tests
+TVM_FFI=cython python3 -m pytest apps/extension/tests
+TVM_FFI=ctypes python3 -m pytest apps/extension/tests
 
 # Test dso plugin
 cd apps/dso_plugin_module
 rm -rf lib
 make
 cd ../..
-TVM_FFI=cython python3 -m pytest -v apps/dso_plugin_module
-TVM_FFI=ctypes python3 -m pytest -v apps/dso_plugin_module
+TVM_FFI=cython python3 -m pytest apps/dso_plugin_module
+TVM_FFI=ctypes python3 -m pytest apps/dso_plugin_module
 
 # Do not enable TensorFlow op
 # TVM_FFI=cython sh prepare_and_test_tfop_module.sh
 # TVM_FFI=ctypes sh prepare_and_test_tfop_module.sh
 
-TVM_FFI=ctypes python3 -m pytest -v tests/python/integration
-TVM_FFI=ctypes python3 -m pytest -v tests/python/contrib
+TVM_FFI=ctypes python3 -m pytest tests/python/integration
+TVM_FFI=ctypes python3 -m pytest tests/python/contrib
 
-TVM_FFI=ctypes python3 -m pytest -v tests/python/relay
+TVM_FFI=ctypes python3 -m pytest tests/python/relay
 
 # Do not enable OpenGL
-# TVM_FFI=cython python -m pytest -v tests/webgl
-# TVM_FFI=ctypes python3 -m pytest -v tests/webgl
+# TVM_FFI=cython python -m pytest tests/webgl
+# TVM_FFI=ctypes python3 -m pytest tests/webgl

tests/scripts/task_python_nightly.sh

@@ -19,7 +19,7 @@
 set -e
 set -u
 
-export PYTHONPATH=python:topi/python
+source tests/scripts/setup-pytest-env.sh
 
 # Rebuild cython
 make cython3
@@ -27,4 +27,4 @@ make cython3
 # cleanup pycache
 find . -type f -path "*.pyc" | xargs rm -f
 
-python3 -m pytest -v topi/tests/python/nightly
+python3 -m pytest topi/tests/python/nightly

tests/scripts/task_python_topi.sh

@@ -19,12 +19,11 @@
 set -e
 set -u
 
-export PYTHONPATH=python:topi/python
-
+source tests/scripts/setup-pytest-env.sh
 # Rebuild cython
 make cython3
 
 # cleanup pycache
 find . -type f -path "*.pyc" | xargs rm -f
 
-python3 -m pytest -v topi/tests/python
+python3 -m pytest topi/tests/python

tests/scripts/task_python_unittest.sh

@@ -19,11 +19,11 @@
 set -e
 set -u
 
-export PYTHONPATH=python:topi/python
+source tests/scripts/setup-pytest-env.sh
 
 # cleanup pycache
 find . -type f -path "*.pyc" | xargs rm -f
 
-TVM_FFI=ctypes python3 -m pytest -v tests/python/unittest
+TVM_FFI=ctypes python3 -m pytest tests/python/unittest
 make cython3
-TVM_FFI=cython python3 -m pytest -v tests/python/unittest
+TVM_FFI=cython python3 -m pytest tests/python/unittest

tests/scripts/task_python_vta_fsim.sh

@@ -19,8 +19,8 @@
 set -e
 set -u
 
-export TVM_PATH=`pwd`
-export PYTHONPATH=${TVM_PATH}/python:${TVM_PATH}/vta/python:${TVM_PATH}/topi/python
+source tests/scripts/setup-pytest-env.sh
+export PYTHONPATH=${PYTHONPATH}:${TVM_PATH}/vta/python
 export VTA_HW_PATH=`pwd`/3rdparty/vta-hw
 
 # cleanup pycache
@@ -36,8 +36,8 @@ cp ${VTA_HW_PATH}/config/fsim_sample.json ${VTA_HW_PATH}/config/vta_config.json
 
 # Run unit tests in functional/fast simulator
 echo "Running unittest in fsim..."
-python3 -m pytest -v ${TVM_PATH}/vta/tests/python/unittest
+python3 -m pytest ${TVM_PATH}/vta/tests/python/unittest
 
 # Run unit tests in functional/fast simulator
 echo "Running integration test in fsim..."
-python3 -m pytest -v ${TVM_PATH}/vta/tests/python/integration
+python3 -m pytest ${TVM_PATH}/vta/tests/python/integration

tests/scripts/task_python_vta_tsim.sh

@@ -19,8 +19,8 @@
 set -e
 set -u
 
-export TVM_PATH=`pwd`
-export PYTHONPATH=${TVM_PATH}/python:${TVM_PATH}/vta/python:${TVM_PATH}/topi/python
+source tests/scripts/setup-pytest-env.sh
+export PYTHONPATH=${PYTHONPATH}:${TVM_PATH}/vta/python
 export VTA_HW_PATH=`pwd`/3rdparty/vta-hw
 
 # cleanup pycache
@@ -51,11 +51,11 @@ make -C ${VTA_HW_PATH}/hardware/chisel USE_THREADS=0 lib
 
 # Run unit tests in cycle accurate simulator
 echo "Running unittest in tsim..."
-python3 -m pytest -v ${TVM_PATH}/vta/tests/python/unittest
+python3 -m pytest ${TVM_PATH}/vta/tests/python/unittest
 
 # Run unit tests in cycle accurate simulator
 echo "Running integration test in tsim..."
-python3 -m pytest -v ${TVM_PATH}/vta/tests/python/integration
+python3 -m pytest ${TVM_PATH}/vta/tests/python/integration
 
 # Reset default fsim simulation
 cp ${VTA_HW_PATH}/config/fsim_sample.json ${VTA_HW_PATH}/config/vta_config.json