Skip to content

T
tic
Commit 552fa71b by Tianqi Chen Committed by GitHub
Options

[BUILD] Upgrade build system to default python3 (#1260)

parent 2d3031ee
Showing with 51 additions and 44 deletions
Jenkinsfile
...@@ -106,7 +106,7 @@ stage('Build') { ...@@ -106,7 +106,7 @@ stage('Build') {
cp ../cmake/config.cmake . cp ../cmake/config.cmake .
echo set\\(USE_OPENCL ON\\) >> config.cmake echo set\\(USE_OPENCL ON\\) >> config.cmake
echo set\\(USE_ROCM ON\\) >> config.cmake echo set\\(USE_ROCM ON\\) >> config.cmake
echo set\\(USE_VULKAN ON\\) >> config.cmake echo set\\(USE_VULKAN OFF\\) >> config.cmake
""" """
make('gpu', 'build2', '-j2') make('gpu', 'build2', '-j2')
} }
......
Makefile
...@@ -48,18 +48,18 @@ build/libtvm_web_runtime.js: build/libtvm_web_runtime.bc ...@@ -48,18 +48,18 @@ build/libtvm_web_runtime.js: build/libtvm_web_runtime.bc
# Lint scripts # Lint scripts
cpplint: cpplint:
python dmlc-core/scripts/lint.py topi cpp topi/include; python3 dmlc-core/scripts/lint.py topi cpp topi/include;
python dmlc-core/scripts/lint.py nnvm cpp nnvm/include nnvm/src; python3 dmlc-core/scripts/lint.py nnvm cpp nnvm/include nnvm/src;
python dmlc-core/scripts/lint.py tvm cpp include src verilog\ python3 dmlc-core/scripts/lint.py tvm cpp include src verilog\
examples/extension/src examples/graph_executor/src examples/extension/src examples/graph_executor/src
pylint: pylint:
pylint python/tvm --rcfile=$(ROOTDIR)/tests/lint/pylintrc python3 -m pylint python/tvm --rcfile=$(ROOTDIR)/tests/lint/pylintrc
pylint topi/python/topi --rcfile=$(ROOTDIR)/tests/lint/pylintrc python3 -m pylint topi/python/topi --rcfile=$(ROOTDIR)/tests/lint/pylintrc
pylint nnvm/python/nnvm --rcfile=$(ROOTDIR)/tests/lint/pylintrc python3 -m pylint nnvm/python/nnvm --rcfile=$(ROOTDIR)/tests/lint/pylintrc
jnilint: jnilint:
python dmlc-core/scripts/lint.py tvm4j-jni cpp jvm/native/src python3 dmlc-core/scripts/lint.py tvm4j-jni cpp jvm/native/src
lint: cpplint pylint jnilint lint: cpplint pylint jnilint
......
docs/Makefile
...@@ -3,15 +3,10 @@ ...@@ -3,15 +3,10 @@
# You can set these variables from the command line. # You can set these variables from the command line.
SPHINXOPTS = SPHINXOPTS =
SPHINXBUILD = sphinx-build SPHINXBUILD = python3 -m sphinx
PAPER = PAPER =
BUILDDIR = _build BUILDDIR = _build
# User-friendly check for sphinx-build
ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
endif
# Internal variables. # Internal variables.
PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_a4 = -D latex_paper_size=a4
PAPEROPT_letter = -D latex_paper_size=letter PAPEROPT_letter = -D latex_paper_size=letter
......
docs/install/index.rst
...@@ -38,7 +38,7 @@ Our goal is to build the shared libraries: ...@@ -38,7 +38,7 @@ Our goal is to build the shared libraries:
The minimal building requirements are The minimal building requirements are
- A recent c++ compiler supporting C++ 11 (g++-4.8 or higher) - A recent c++ compiler supporting C++ 11 (g++-4.8 or higher)
- CMake 3.7 or higher - CMake 3.5 or higher
- We highly recommend to build with LLVM to enable all the features. - We highly recommend to build with LLVM to enable all the features.
- It is possible to build without llvm dependency if we only want to use CUDA/OpenCL - It is possible to build without llvm dependency if we only want to use CUDA/OpenCL
...@@ -49,6 +49,7 @@ The configuration of tvm can be modified by `config.cmake`. ...@@ -49,6 +49,7 @@ The configuration of tvm can be modified by `config.cmake`.
- First, check the cmake in your system, you do not have cmake - First, check the cmake in your system, you do not have cmake
you can obtain the latest version from `official website <https://cmake.org/download/>`_ you can obtain the latest version from `official website <https://cmake.org/download/>`_
- First create a build directory, copy the ``cmake/config.cmake`` to the directory. - First create a build directory, copy the ``cmake/config.cmake`` to the directory.
.. code:: bash .. code:: bash
mkdir build mkdir build
...@@ -76,6 +77,7 @@ The configuration of tvm can be modified by `config.cmake`. ...@@ -76,6 +77,7 @@ The configuration of tvm can be modified by `config.cmake`.
- We can then build tvm and related libraries. - We can then build tvm and related libraries.
.. code:: bash .. code:: bash
cd build cd build
cmake .. cmake ..
make -j4 make -j4
......
nnvm/python/nnvm/frontend/coreml.py
...@@ -5,6 +5,7 @@ import numpy as np ...@@ -5,6 +5,7 @@ import numpy as np
import tvm import tvm
from .. import symbol as _sym from .. import symbol as _sym
from .._base import string_types
from .common import SymbolTable from .common import SymbolTable
__all__ = ['from_coreml'] __all__ = ['from_coreml']
...@@ -281,7 +282,7 @@ def coreml_op_to_nnvm(op, inname, outname, symtab): ...@@ -281,7 +282,7 @@ def coreml_op_to_nnvm(op, inname, outname, symtab):
classname = type(op).__name__ classname = type(op).__name__
if classname not in _convert_map: if classname not in _convert_map:
raise NotImplementedError("%s is not supported" % (classname)) raise NotImplementedError("%s is not supported" % (classname))
if isinstance(inname, (str, unicode)): if isinstance(inname, string_types):
insym = symtab.get_var(inname) insym = symtab.get_var(inname)
else: else:
insym = [symtab.get_var(i) for i in inname] insym = [symtab.get_var(i) for i in inname]
......
nnvm/python/nnvm/frontend/keras.py
...@@ -487,6 +487,7 @@ def from_keras(model): ...@@ -487,6 +487,7 @@ def from_keras(model):
import keras import keras
except ImportError: except ImportError:
raise ImportError('Keras must be installed') raise ImportError('Keras must be installed')
assert isinstance(model, keras.engine.training.Model) assert isinstance(model, keras.engine.training.Model)
if keras.backend.image_data_format() != 'channels_last': if keras.backend.image_data_format() != 'channels_last':
raise ValueError("Keras frontend currently supports data_format = channels_last only.") raise ValueError("Keras frontend currently supports data_format = channels_last only.")
...@@ -494,7 +495,7 @@ def from_keras(model): ...@@ -494,7 +495,7 @@ def from_keras(model):
symtab = SymbolTable() symtab = SymbolTable()
for keras_layer in model.layers: for keras_layer in model.layers:
if isinstance(keras_layer, keras.engine.topology.InputLayer): if isinstance(keras_layer, keras.engine.InputLayer):
symtab.get_var(keras_layer.name, must_contain=False) symtab.get_var(keras_layer.name, must_contain=False)
else: else:
inbound_nodes = keras_layer.inbound_nodes if hasattr(keras_layer, 'inbound_nodes') \ inbound_nodes = keras_layer.inbound_nodes if hasattr(keras_layer, 'inbound_nodes') \
...@@ -512,7 +513,7 @@ def from_keras(model): ...@@ -512,7 +513,7 @@ def from_keras(model):
# would confuse users, so we should keep them as far as possible. Fortunately, # would confuse users, so we should keep them as far as possible. Fortunately,
# they are named uniquely to input_1, input_2, input_3 ... by default. # they are named uniquely to input_1, input_2, input_3 ... by default.
for pred_idx, pred in zip(node.node_indices, node.inbound_layers): for pred_idx, pred in zip(node.node_indices, node.inbound_layers):
if isinstance(pred, keras.engine.topology.InputLayer): if isinstance(pred, keras.engine.InputLayer):
_sym = symtab.get_var(pred.name, must_contain=True) _sym = symtab.get_var(pred.name, must_contain=True)
else: else:
_sym = symtab.get_var(pred.name + ':' + str(pred_idx), must_contain=True) _sym = symtab.get_var(pred.name + ':' + str(pred_idx), must_contain=True)
...@@ -522,6 +523,6 @@ def from_keras(model): ...@@ -522,6 +523,6 @@ def from_keras(model):
insym = insym[0] insym = insym[0]
keras_op_to_nnvm(insym, keras_layer, keras_layer.name + ':' + str(my_idx), symtab) keras_op_to_nnvm(insym, keras_layer, keras_layer.name + ':' + str(my_idx), symtab)
outsym = symtab.get_var(model.output_layers[0].name + ':0') outsym = symtab.get_var(model._output_layers[0].name + ':0')
tvmparams = {k:tvm.nd.array(np.array(v, dtype=np.float32)) for k, v in symtab.params.items()} tvmparams = {k:tvm.nd.array(np.array(v, dtype=np.float32)) for k, v in symtab.params.items()}
return outsym, tvmparams return outsym, tvmparams
nnvm/tests/python/frontend/keras/test_forward.py
...@@ -15,9 +15,9 @@ set_session(tf.Session(config=config)) ...@@ -15,9 +15,9 @@ set_session(tf.Session(config=config))
def verify_keras_frontend(keras_model): def verify_keras_frontend(keras_model):
in_shapes = [] in_shapes = []
for layer in keras_model.input_layers: for layer in keras_model._input_layers:
in_shapes.append(tuple(dim.value if dim.value is not None else 1 for dim in layer.input.shape)) in_shapes.append(tuple(dim.value if dim.value is not None else 1 for dim in layer.input.shape))
out_shape = [dim.value if dim.value is not None else 1 for dim in keras_model.output_layers[0].output.shape] out_shape = [dim.value if dim.value is not None else 1 for dim in keras_model._output_layers[0].output.shape]
def get_keras_output(xs, dtype='float32'): def get_keras_output(xs, dtype='float32'):
return keras_model.predict(xs) return keras_model.predict(xs)
...@@ -41,7 +41,7 @@ def verify_keras_frontend(keras_model): ...@@ -41,7 +41,7 @@ def verify_keras_frontend(keras_model):
tvm_out = get_tvm_output([x.transpose([0,3,1,2]) for x in xs], target, ctx) tvm_out = get_tvm_output([x.transpose([0,3,1,2]) for x in xs], target, ctx)
np.testing.assert_allclose(keras_out, tvm_out, rtol=1e-5, atol=1e-5) np.testing.assert_allclose(keras_out, tvm_out, rtol=1e-5, atol=1e-5)
def test_forward_elemwise_add(): def test_forward_elemwise_add():
r = [] r = []
data = keras.layers.Input(shape=(32,32,3)) data = keras.layers.Input(shape=(32,32,3))
...@@ -74,7 +74,7 @@ def test_forward_dense(): ...@@ -74,7 +74,7 @@ def test_forward_dense():
def test_forward_transpose_conv(): def test_forward_transpose_conv():
data = keras.layers.Input(shape=(32,32,3)) data = keras.layers.Input(shape=(32,32,3))
x = keras.layers.Conv2D(filters=10, kernel_size=(3,3), strides=(2,2), padding='same')(data) x = keras.layers.Conv2D(filters=10, kernel_size=(3,3), strides=(2,2), padding='same')(data)
x = keras.applications.mobilenet.DepthwiseConv2D(kernel_size=(3,3), padding='same')(x) x = keras.layers.DepthwiseConv2D(kernel_size=(3,3), padding='same')(x)
x = keras.layers.Conv2DTranspose(filters=64, kernel_size=(3,3), padding='valid')(x) x = keras.layers.Conv2DTranspose(filters=64, kernel_size=(3,3), padding='valid')(x)
x = keras.layers.GlobalMaxPooling2D()(x) x = keras.layers.GlobalMaxPooling2D()(x)
keras_model = keras.models.Model(data, x) keras_model = keras.models.Model(data, x)
...@@ -136,7 +136,6 @@ def test_forward_activations(): ...@@ -136,7 +136,6 @@ def test_forward_activations():
act_funcs = [keras.layers.Activation('softmax'), act_funcs = [keras.layers.Activation('softmax'),
keras.layers.Activation('softplus'), keras.layers.Activation('softplus'),
keras.layers.LeakyReLU(alpha=0.3), keras.layers.LeakyReLU(alpha=0.3),
keras.layers.Activation(keras.applications.mobilenet.relu6),
keras.layers.PReLU(weights=weights, alpha_initializer="zero"), keras.layers.PReLU(weights=weights, alpha_initializer="zero"),
keras.layers.ELU(alpha=0.5), keras.layers.ELU(alpha=0.5),
keras.layers.Activation('selu'), keras.layers.Activation('selu'),
......
nnvm/tests/python/frontend/mxnet/test_forward.py
...@@ -100,8 +100,7 @@ def test_forward_rrelu(): ...@@ -100,8 +100,7 @@ def test_forward_rrelu():
def test_forward_prelu(): def test_forward_prelu():
data = mx.sym.var('data') data = mx.sym.var('data')
data = mx.sym.concat(data, -data, dim=1) # negative part explicitly data = mx.sym.concat(data, -data, dim=1) # negative part explicitly
gamma = mx.sym.zeros(shape=(6,)) mx_sym = mx.sym.LeakyReLU(data, act_type='prelu')
mx_sym = mx.sym.LeakyReLU(data, gamma=gamma, act_type='prelu')
verify_mxnet_frontend_impl(mx_sym, (1, 3, 100, 100), (1, 6, 100, 100)) verify_mxnet_frontend_impl(mx_sym, (1, 3, 100, 100), (1, 6, 100, 100))
def test_forward_softrelu(): def test_forward_softrelu():
......
tests/ci_build/Dockerfile.gpu
...@@ -23,7 +23,7 @@ RUN bash /install/ubuntu_install_sphinx.sh ...@@ -23,7 +23,7 @@ RUN bash /install/ubuntu_install_sphinx.sh
# Fix recommonmark to latest version # Fix recommonmark to latest version
RUN git clone https://github.com/rtfd/recommonmark RUN git clone https://github.com/rtfd/recommonmark
RUN cd recommonmark; python setup.py install RUN cd recommonmark; python3 setup.py install
# Enable doxygen for c++ doc build # Enable doxygen for c++ doc build
RUN apt-get update && apt-get install -y doxygen graphviz libprotobuf-dev protobuf-compiler RUN apt-get update && apt-get install -y doxygen graphviz libprotobuf-dev protobuf-compiler
...@@ -40,9 +40,6 @@ RUN bash /install/ubuntu_install_rocm.sh ...@@ -40,9 +40,6 @@ RUN bash /install/ubuntu_install_rocm.sh
COPY install/ubuntu_install_opengl.sh /install/ubuntu_install_opengl.sh COPY install/ubuntu_install_opengl.sh /install/ubuntu_install_opengl.sh
RUN bash /install/ubuntu_install_opengl.sh RUN bash /install/ubuntu_install_opengl.sh
COPY install/ubuntu_install_vulkan.sh /install/ubuntu_install_vulkan.sh
RUN bash /install/ubuntu_install_vulkan.sh
# DL Frameworks # DL Frameworks
COPY install/ubuntu_install_mxnet.sh /install/ubuntu_install_mxnet.sh COPY install/ubuntu_install_mxnet.sh /install/ubuntu_install_mxnet.sh
...@@ -60,7 +57,11 @@ RUN bash /install/ubuntu_install_darknet.sh ...@@ -60,7 +57,11 @@ RUN bash /install/ubuntu_install_darknet.sh
COPY install/ubuntu_install_onnx.sh /install/ubuntu_install_onnx.sh COPY install/ubuntu_install_onnx.sh /install/ubuntu_install_onnx.sh
RUN bash /install/ubuntu_install_onnx.sh RUN bash /install/ubuntu_install_onnx.sh
RUN pip install Pillow RUN pip3 install Pillow
# disable vulkan for now
# COPY install/ubuntu_install_vulkan.sh /install/ubuntu_install_vulkan.sh
# RUN bash /install/ubuntu_install_vulkan.sh
# Environment variables # Environment variables
ENV PATH=/usr/local/nvidia/bin:${PATH} ENV PATH=/usr/local/nvidia/bin:${PATH}
......
tests/ci_build/Dockerfile.lint
# For lint test # For lint test
FROM ubuntu:16.04 FROM ubuntu:16.04
RUN apt-get update && apt-get install -y python-pip sudo RUN apt-get update && apt-get install -y sudo wget
COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
RUN bash /install/ubuntu_install_python.sh
RUN apt-get install -y doxygen graphviz RUN apt-get install -y doxygen graphviz
RUN pip install cpplint pylint RUN pip3 install cpplint pylint
tests/ci_build/install/ubuntu_install_coreml.sh
pip2 install coremltools pip3 install coremltools
tests/ci_build/install/ubuntu_install_keras.sh
pip2 install keras tensorflow h5py pip2 install keras tensorflow h5py
pip3 install keras tensorflow h5py
tests/ci_build/install/ubuntu_install_mxnet.sh
pip2 install mxnet
pip3 install mxnet pip3 install mxnet
tests/ci_build/install/ubuntu_install_python.sh
# install python and pip, don't modify this, modify install_python_package.sh # install python and pip, don't modify this, modify install_python_package.sh
apt-get update && apt-get install -y python-pip python-dev python3-dev apt-get update && apt-get install -y python-dev
# the version of the pip shipped with ubuntu may be too lower, install a recent version here # python 3.6
cd /tmp && wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && python2 get-pip.py apt-get update && yes | apt-get install software-properties-common
add-apt-repository ppa:jonathonf/python-3.6 &&\
apt-get update && apt-get install -y python-pip python-dev python3.6 python3.6-dev
rm -f /usr/bin/python3 && ln -s /usr/bin/python3.6 /usr/bin/python3
# Install pip
cd /tmp && wget https://bootstrap.pypa.io/get-pip.py && python2 get-pip.py && python3.6 get-pip.py
tests/ci_build/install/ubuntu_install_python_package.sh
# install libraries for python package on ubuntu # install libraries for python package on ubuntu
pip2 install nose pylint numpy nose-timer cython decorator scipy tornado pip2 install nose pylint numpy nose-timer cython decorator scipy tornado
pip3 install nose pylint numpy nose-timer cython decorator scipy tornado pip3 install nose pylint numpy nose-timer cython decorator scipy tornado typed_ast
tests/ci_build/install/ubuntu_install_sphinx.sh
pip install sphinx sphinx-gallery sphinx_rtd_theme matplotlib Image commonmark>=0.7.3 docutils>=0.11 pip3 install sphinx sphinx-gallery sphinx_rtd_theme matplotlib Image commonmark>=0.7.3 docutils>=0.11
tests/scripts/task_clean.sh
#!/bin/bash #!/bin/bash
echo "Cleanup data..." echo "Cleanup data..."
rm -rf $1 cd $1 && rm -rf Cmake* && cd ..
tests/scripts/task_python_nnvm.sh
...@@ -11,10 +11,10 @@ python -m nose -v nnvm/tests/python/compiler || exit -1 ...@@ -11,10 +11,10 @@ python -m nose -v nnvm/tests/python/compiler || exit -1
python3 -m nose -v nnvm/tests/python/compiler || exit -1 python3 -m nose -v nnvm/tests/python/compiler || exit -1
echo "Running ONNX frontend test..." echo "Running ONNX frontend test..."
python -m nose -v nnvm/tests/python/frontend/onnx || exit -1 python3 -m nose -v nnvm/tests/python/frontend/onnx || exit -1
echo "Running MXNet frontend test..." echo "Running MXNet frontend test..."
python -m nose -v nnvm/tests/python/frontend/mxnet || exit -1 python3 -m nose -v nnvm/tests/python/frontend/mxnet || exit -1
echo "Running Keras frontend test..." echo "Running Keras frontend test..."
python -m nose -v nnvm/tests/python/frontend/keras || exit -1 python3 -m nose -v nnvm/tests/python/frontend/keras || exit -1
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment