Skip to content

T
tic
Commit 40901446 by Tianqi Chen
Options

[RUNTIME] Remove runtime, rely on tvm only (#38)

parent 3a5b9fcb
Showing with 66 additions and 1249 deletions
nnvm/Makefile
......@@ -53,10 +53,10 @@ else
NO_WHOLE_ARCH= --no-whole-archive
endif
all: lib/libnnvm.a lib/libnnvm_top.$(SHARED_LIBRARY_SUFFIX) lib/libnnvm_top_runtime.$(SHARED_LIBRARY_SUFFIX)
all: lib/libnnvm.a lib/libnnvm_top.$(SHARED_LIBRARY_SUFFIX)
SRC = $(wildcard src/*.cc src/c_api/*.cc src/core/*.cc src/pass/*.cc)
SRC_TOP = $(wildcard src/top/*/*.cc src/runtime/*.cc src/compiler/*.cc src/compiler/*/*.cc)
SRC_TOP = $(wildcard src/top/*/*.cc src/compiler/*.cc src/compiler/*/*.cc)
ALL_OBJ = $(patsubst %.cc, build/%.o, $(SRC))
TOP_OBJ = $(patsubst %.cc, build/%.o, $(SRC_TOP))
ALL_DEP = $(ALL_OBJ)
......@@ -78,10 +78,6 @@ lib/libnnvm_top.$(SHARED_LIBRARY_SUFFIX): lib/libnnvm.a ${TOP_OBJ}
@mkdir -p $(@D)
$(CXX) $(CFLAGS) -shared -o $@ $(filter %.o, $^) $(LDFLAGS) -Wl,${WHOLE_ARCH} lib/libnnvm.a -Wl,${NO_WHOLE_ARCH}
lib/libnnvm_top_runtime.$(SHARED_LIBRARY_SUFFIX): deploy/nnvm_runtime.cc
@mkdir -p $(@D)
$(CXX) $(CFLAGS) -shared -o $@ $(filter %.cc, $^) $(LDFLAGS)
cython:
cd python; python setup.py build_ext --inplace
......
nnvm/deploy/REAMD.md deleted 100644 → 0
All in One Deployment File
==========================
This folder contains an all in one deployment file that contains minimum dependencies
needed to run nnvm top runtime.
\ No newline at end of file
nnvm/deploy/nnvm_runtime.cc deleted 100644 → 0
/*!
* Copyright (c) 2017 by Contributors
* All in one runtime
* \file nnvm_runtime.cc
*/
/*
#include "../src/core/graph.cc"
#include "../src/core/node.cc"
#include "../src/core/pass.cc"
#include "../src/core/op.cc"
#include "../src/pass/saveload_json.cc"
#include "../src/runtime/graph_executor.cc"*/
#include "../src/runtime/graph_runtime.cc"
nnvm/python/nnvm/graph.py
......@@ -174,6 +174,10 @@ class Graph(object):
check_call(_LIB.NNGraphGetSymbol(self.handle, ctypes.byref(shandle)))
return Symbol(shandle)
def _tvm_graph_json(self):
"""Get TVM graph json"""
return self.apply("SaveJSON").json_attr("json")
@property
def index(self):
if not self._index:
......
nnvm/python/nnvm/runtime.py deleted 100644 → 0
"""Runtime environment for nnvm relies on TVM."""
import tvm
from tvm.contrib import rpc
class Module(object):
"""Wrapper runtime module.
This is a thin wrapper of the underlying TVM module.
you can also directly call set_input, run, and get_output
of underlying module functions
Parameters
----------
tvm_module : tvm.Module
The interal tvm module
"""
def __init__(self, tvm_module):
self.tvm_module = tvm_module
self._set_input = tvm_module["set_input"]
self._run = tvm_module["run"]
self._get_output = tvm_module["get_output"]
def set_input(self, key=None, value=None, **params):
"""Set inputs to the module via kwargs
Parameters
----------
key : int or str
The input key
value : the input value.
The input key
params : dict of str to NDArray
Additonal arguments
"""
if key:
self._set_input(key, tvm.nd.array(value))
for k, v in params.items():
self._set_input(k, tvm.nd.array(v))
return self
def run(self, **input_dict):
"""Run forward execution of the graph
Parameters
----------
input_dict: dict of str to NDArray
List of input values to be feed to
"""
if input_dict:
self.set_input(**input_dict)
self._run()
def get_output(self, index, out):
"""Get index-th output to out
Parameters
----------
index : int
The input index
out : tvm.NDArray
The output array container
"""
self._get_output(index, out)
return out
def __getitem__(self, key):
"""Get internal module function
Parameters
----------
key : str
The key to the module.
"""
return self.tvm_module[key]
def create(graph, libmod, ctx):
"""Create a runtime executor module given the graph and module.
Parameters
----------
graph : The graph to be deployed
The graph to be loaded.
libmod : tvm.Module
The module of the corresponding function
ctx : TVMContext
The context to deploy the module, can be local or remote.
Returns
-------
graph_module : tvm.Module
Runtime graph module to execute the graph.
"""
json_str = graph if isinstance(graph, str) else graph.apply("SaveJSON").json_attr("json")
device_type = ctx.device_type
device_id = ctx.device_id
if device_type >= rpc.RPC_SESS_MASK:
assert libmod.type_key == "rpc"
assert rpc._SessTableIndex(libmod) == ctx._rpc_sess._tbl_index
hmod = rpc._ModuleHandle(libmod)
fcreate = ctx._rpc_sess.get_function("nnvm.runtime.remote_create")
device_type = device_type % rpc.RPC_SESS_MASK
return Module(fcreate(json_str, hmod, device_type, device_id))
fcreate = tvm.get_global_func("nnvm.runtime.create")
return Module(fcreate(json_str, libmod, device_type, device_id))
nnvm/src/compiler/graph_fuse.cc
......@@ -14,11 +14,31 @@
#include <tvm/runtime/packed_func.h>
#include <tvm/lowered_func.h>
#include "./compile_engine.h"
#include "../runtime/graph_executor.h"
#include "../../tvm/src/runtime/graph/graph_runtime.h"
namespace nnvm {
namespace compiler {
using tvm::runtime::TVMOpParam;
// parser
inline void TVMOpParamParser(nnvm::NodeAttrs* attrs) {
TVMOpParam param;
param.Init(attrs->dict);
attrs->parsed = std::move(param);
}
NNVM_REGISTER_OP(tvm_op)
.set_attr_parser(TVMOpParamParser)
.set_num_inputs([](const NodeAttrs& attrs) {
const TVMOpParam& param = nnvm::get<TVMOpParam>(attrs.parsed);
return param.num_inputs;
})
.set_num_outputs([](const NodeAttrs& attrs) {
const TVMOpParam& param = nnvm::get<TVMOpParam>(attrs.parsed);
return param.num_outputs;
});
using namespace tvm;
// The single fuse rule.
......
nnvm/src/runtime/graph_executor.cc deleted 100644 → 0
/*!
* Copyright (c) 2017 by Contributors
* \file graph_executor.cc
*/
#include <dmlc/memory_io.h>
#include <tvm/runtime/registry.h>
#include <numeric>
#include "./graph_executor.h"
namespace nnvm {
namespace runtime {
/*! \brief macro to do C API call */
#define TVM_CCALL(func) \
{ \
int ret = (func); \
CHECK_EQ(ret, 0) \
<< TVMGetLastError(); \
}
using ::tvm::runtime::PackedFunc;
using ::tvm::runtime::TVMArgs;
using ::tvm::runtime::TVMRetValue;
PackedFunc GraphExecutor::GetFunction(
const std::string& name,
const std::shared_ptr<ModuleNode>& sptr_to_self) {
// return member functions during query.
if (name == "set_input") {
return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) {
if (args[0].type_code() == kStr) {
this->SetInput(this->GetInputIndex(args[0]), args[1]);
} else {
this->SetInput(args[0], args[1]);
}
});
} else if (name == "get_output") {
return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) {
this->GetOutput(args[0], args[1]);
});
} else if (name == "run") {
return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) {
this->Run();
});
} else if (name == "load_params") {
return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) {
this->LoadParams(args[0].operator std::string());
});
} else {
return PackedFunc();
}
}
GraphExecutor::~GraphExecutor() {
for (DLTensor* t : storage_pool_) {
TVM_CCALL(TVMArrayFree(t));
}
}
void GraphExecutor::Run() {
// setup the array and requirements.
for (size_t i = 0; i < op_execs_.size(); ++i) {
if (op_execs_[i]) op_execs_[i]();
}
}
void GraphExecutor::Init(Graph graph,
tvm::runtime::Module module,
TVMContext ctx) {
graph_ = std::move(graph);
module_ = std::move(module);
ctx_ = ctx;
this->SetupStorage();
this->SetupOpExecs();
}
int GraphExecutor::GetInputIndex(const std::string& name) {
const auto& idx = graph_.indexed_graph();
for (size_t i = 0; i< idx.input_nodes().size(); ++i) {
if (idx[idx.input_nodes()[i]].source->attrs.name == name) {
return static_cast<int>(i);
}
}
LOG(FATAL) << "cannot find " << name << " among input";
return -1;
}
void GraphExecutor::SetInput(int index, DLTensor* data_in) {
const auto& idx = graph_.indexed_graph();
CHECK_LT(static_cast<size_t>(index), idx.input_nodes().size());
uint32_t eid = idx.entry_id(idx.input_nodes()[index], 0);
TVM_CCALL(TVMArrayCopyFromTo(data_in, &data_entry_[eid], nullptr));
}
void GraphExecutor::GetOutput(int index, DLTensor* data_out) {
const auto& idx = graph_.indexed_graph();
CHECK_LT(static_cast<size_t>(index), idx.outputs().size());
uint32_t eid = idx.entry_id(idx.outputs()[index]);
TVM_CCALL(TVMArrayCopyFromTo(&data_entry_[eid], data_out, nullptr));
}
bool LoadDLTensor(dmlc::Stream* strm, DLTensor* tensor) {
uint64_t header, reserved;
CHECK(strm->Read(&header, sizeof(header)))
<< "Invalid DLTensor file format";
CHECK(strm->Read(&reserved, sizeof(reserved)))
<< "Invalid DLTensor file format";
CHECK(header == kTVMNDArrayMagic)
<< "Invalid DLTensor file format";
CHECK(strm->Read(&tensor->ctx, sizeof(tensor->ctx)))
<< "Invalid DLTensor file format";
CHECK(strm->Read(&tensor->ndim, sizeof(tensor->ndim)))
<< "Invalid DLTensor file format";
CHECK(strm->Read(&tensor->dtype, sizeof(tensor->dtype)))
<< "Invalid DLTensor file format";
int ndim = tensor->ndim;
CHECK(strm->Read(tensor->shape, sizeof(int64_t) * ndim))
<< "Invalid DLTensor file format";
int64_t size = 1;
int type_size = tensor->dtype.bits / 8;
for (int i = 0; i < ndim; ++i) {
size *= tensor->shape[i];
}
int64_t data_byte_size;
CHECK(strm->Read(&data_byte_size, sizeof(data_byte_size)))
<< "Invalid DLTensor file format";
CHECK(data_byte_size == type_size * size)
<< "Invalid DLTensor file format";
CHECK(strm->Read(tensor->data, type_size * size))
<< "Invalid DLTensor file format";
return true;
}
void GraphExecutor::LoadParams(dmlc::Stream* strm) {
uint64_t header, reserved;
CHECK(strm->Read(&header))
<< "Invalid parameters file format";
CHECK(header == kTVMNDArrayListMagic)
<< "Invalid parameters file format";
CHECK(strm->Read(&reserved))
<< "Invalid parameters file format";
std::vector<std::string> names;
CHECK(strm->Read(&names))
<< "Invalid parameters file format";
std::unordered_map<std::string, size_t> name_eid;
const auto& idx = graph_.indexed_graph();
for (int nid : idx.input_nodes()) {
name_eid.emplace(idx[nid].source->attrs.name, idx.entry_id(nid, 0));
}
uint64_t sz;
strm->Read(&sz, sizeof(sz));
size_t size = static_cast<size_t>(sz);
CHECK(size == names.size())
<< "Invalid parameters file format";
for (size_t i = 0; i < size; ++i) {
auto iter = name_eid.find(names[i]);
CHECK(iter != name_eid.end());
CHECK(LoadDLTensor(strm, &data_entry_[iter->second]))
<< "Invalid parameters file format";
}
}
void GraphExecutor::LoadParams(const std::string& param_blob) {
dmlc::MemoryStringStream strm(const_cast<std::string*>(&param_blob));
this->LoadParams(&strm);
}
void GraphExecutor::SetupStorage() {
const auto& idx = graph_.indexed_graph();
// Grab saved optimization plan from graph.
auto vstorage = graph_.MoveCopyAttr<StorageVector>("storage_id");
std::vector<TVMType> vtype;
for (const std::string& s_type :
graph_.GetAttr<std::vector<std::string> >("dltype")) {
vtype.push_back(tvm::runtime::String2TVMType(s_type));
}
data_shape_ = graph_.GetAttr<ShapeVector>("shape");
data_entry_.resize(idx.num_node_entries());
// Find the maximum space size.
int max_id = 0;
for (size_t i = 0; i < data_shape_.size(); ++i) {
max_id = std::max(vstorage[i] + 1, max_id);
}
for (const auto& e : idx.input_nodes()) {
vstorage[idx.entry_id(e, 0)] = max_id++;
}
// size of each storage pool entry
std::vector<size_t> pool_entry_bytes;
// Find the maximum space size.
for (size_t i = 0; i < data_shape_.size(); ++i) {
int storage_id = vstorage[i];
size_t size = data_shape_[i].Size();
CHECK_GE(storage_id, 0) << "Do not support runtime shape op";
DLDataType t = vtype[i];
size_t bits = t.bits * t.lanes;
CHECK_EQ(bits % 8U, 0U);
size_t bytes = (bits / 8U) * size;
size_t sid = static_cast<size_t>(storage_id);
if (sid >= pool_entry_bytes.size()) {
pool_entry_bytes.resize(sid + 1, 0);
}
pool_entry_bytes[sid] = std::max(pool_entry_bytes[sid], bytes);
}
// Allocate the space.
for (size_t i = 0; i < pool_entry_bytes.size(); ++i) {
TShape shape{static_cast<int64_t>(pool_entry_bytes[i] + 3) / 4};
DLTensor* tensor;
TVM_CCALL(TVMArrayAlloc(
shape.data(), 1, kFloat, 32, 1, ctx_.device_type, ctx_.device_id, &tensor));
storage_pool_.push_back(tensor);
}
// Assign the pooled entries.
for (size_t i = 0; i < data_entry_.size(); ++i) {
int storage_id = vstorage[i];
data_entry_[i] = *storage_pool_[storage_id];
data_entry_[i].shape = const_cast<int64_t*>(data_shape_[i].data());
data_entry_[i].ndim = data_shape_[i].ndim();
data_entry_[i].dtype = vtype[i];
}
}
void GraphExecutor::SetupOpExecs() {
static const nnvm::Op* tvm_op = nnvm::Op::Get("tvm_op");
const auto& idx = graph_.indexed_graph();
op_execs_.resize(idx.num_nodes());
// setup the array and requirements.
for (uint32_t nid = 0; nid < idx.num_nodes(); ++nid) {
const auto& inode = idx[nid];
if (inode.source->is_variable()) continue;
std::vector<DLTensor> args;
for (const auto& e : inode.inputs) {
args.push_back(data_entry_[idx.entry_id(e)]);
}
for (uint32_t index = 0; index < inode.source->num_outputs(); ++index) {
uint32_t eid = idx.entry_id(nid, index);
args.push_back(data_entry_[eid]);
}
CHECK_EQ(inode.source->op(), tvm_op)
<< "transform the graph to tvm op";
op_execs_[nid] = CreateTVMOp(
inode.source->attrs, args, inode.inputs.size());
}
}
std::function<void()> GraphExecutor::CreateTVMOp(
const nnvm::NodeAttrs& attrs,
const std::vector<DLTensor>& args,
size_t num_inputs) {
struct OpArgs {
std::vector<DLTensor> args;
std::vector<TVMValue> arg_values;
std::vector<int> arg_tcodes;
std::vector<int64_t> shape_data;
};
const TVMOpParam& param = nnvm::get<TVMOpParam>(attrs.parsed);
std::shared_ptr<OpArgs> arg_ptr = std::make_shared<OpArgs>();
// setup address.
arg_ptr->args = std::move(args);
if (param.flatten_data) {
arg_ptr->shape_data.resize(arg_ptr->args.size());
}
for (size_t i = 0; i < arg_ptr->args.size(); ++i) {
TVMValue v;
DLTensor* t = &(arg_ptr->args[i]);
v.v_handle = t;
arg_ptr->arg_values.push_back(v);
arg_ptr->arg_tcodes.push_back(kArrayHandle);
if (param.flatten_data) {
arg_ptr->shape_data[i] = std::accumulate(
t->shape, t->shape + t->ndim, 1, std::multiplies<int64_t>());
t->ndim = 1;
t->shape = &(arg_ptr->shape_data[i]);
}
}
// get compiled function from module.
tvm::runtime::PackedFunc pf = module_.GetFunction(param.func_name, false);
CHECK(pf != nullptr) << "no such function in module: " << param.func_name;
auto fexec = [arg_ptr, pf] () {
TVMRetValue rv;
TVMArgs targs(arg_ptr->arg_values.data(),
arg_ptr->arg_tcodes.data(),
static_cast<int>(arg_ptr->arg_values.size()));
pf.CallPacked(targs, &rv);
};
return fexec;
}
// parser
inline void TVMOpParamParser(nnvm::NodeAttrs* attrs) {
TVMOpParam param;
param.Init(attrs->dict);
attrs->parsed = std::move(param);
}
NNVM_REGISTER_OP(tvm_op)
.set_attr_parser(TVMOpParamParser)
.set_num_inputs([](const NodeAttrs& attrs) {
const TVMOpParam& param = nnvm::get<TVMOpParam>(attrs.parsed);
return param.num_inputs;
})
.set_num_outputs([](const NodeAttrs& attrs) {
const TVMOpParam& param = nnvm::get<TVMOpParam>(attrs.parsed);
return param.num_outputs;
});
tvm::runtime::Module RuntimeCreate(std::string sym_json,
tvm::runtime::Module m,
int device_type,
int device_id) {
TVMContext ctx;
ctx.device_type = static_cast<DLDeviceType>(device_type);
ctx.device_id = device_id;
// load graph from json string
nnvm::Graph g;
g.attrs["json"] = std::make_shared<nnvm::any>(sym_json);
g = nnvm::ApplyPass(std::move(g), "LoadJSON");
std::shared_ptr<GraphExecutor> exec = std::make_shared<GraphExecutor>();
exec->Init(g, m, ctx);
return tvm::runtime::Module(exec);
}
TVM_REGISTER_GLOBAL("nnvm.runtime.createx")
.set_body([](TVMArgs args, TVMRetValue *rv) {
*rv = RuntimeCreate(args[0], args[1], args[2], args[3]);
});
TVM_REGISTER_GLOBAL("nnvm.runtime.remote_createx")
.set_body([](TVMArgs args, TVMRetValue *rv) {
void* mhandle = args[1];
*rv = RuntimeCreate(args[0],
*static_cast<tvm::runtime::Module*>(mhandle),
args[2], args[3]);
});
} // namespace runtime
} // namespace nnvm
nnvm/src/runtime/graph_executor.h deleted 100644 → 0
/*!
* Copyright (c) 2017 by Contributors
*
* Runtime module for graph deployment.
*
* \file graph_executor.h
*/
#ifndef NNVM_RUNTIME_GRAPH_EXECUTOR_H_
#define NNVM_RUNTIME_GRAPH_EXECUTOR_H_
#include <dmlc/io.h>
#include <tvm/runtime/packed_func.h>
#include <tvm/runtime/module.h>
#include <nnvm/graph.h>
#include <nnvm/graph_attr_types.h>
#include <nnvm/tuple.h>
#include <nnvm/pass.h>
#include <vector>
#include <string>
#include "./graph_runtime.h"
namespace nnvm {
namespace runtime {
/*!
* \brief TVM Graph Executor.
* This is a minimum graph executor, embedded in TVM runtime
* without any framework dependency.
*
* This runtime can be acccesibly in various language via
* TVM runtime PackedFunc API.
*/
class GraphExecutor : public ::tvm::runtime::ModuleNode {
public:
/*!
* \return The type key of the executor.
*/
const char* type_key() const final {
return "GraphExecutor";
}
/*!
* \brief Get member function to front-end
* \param name The name of the function.
* \param sptr_to_self The pointer to the module node.
* \return The corresponding member function.
*/
tvm::runtime::PackedFunc GetFunction(
const std::string& name,
const std::shared_ptr<ModuleNode>& sptr_to_self) final;
/*! \brief destructor */
~GraphExecutor();
/*!
* \brief Initialize the graph executor with graph and context.
* \param graph The execution graph.
* \param module The module containing the compiled functions.
* \param ctx The context where the graph should sit on
*/
void Init(Graph graph,
tvm::runtime::Module module,
TVMContext ctx);
/*!
* \brief Get the input index given the name of input.
* \param name The name of the input.
* \return The index of input.
*/
int GetInputIndex(const std::string& name);
/*!
* \brief set index-th input to the graph.
* \param index The input index.
* \param data The input data.
*/
void SetInput(int index, DLTensor* data);
/*!
* \brief Copy index-th output to data_out.
* \param index The output index.
* \param data_out the output data.
*/
void GetOutput(int index, DLTensor* data_out);
/*!
* \brief Load parameters from binary stream
* \param strm The input stream.
*/
void LoadParams(dmlc::Stream* strm);
/*!
* \brief Load parameters from parameter blob.
* \param param_blob A binary blob of parameter.
*/
void LoadParams(const std::string& param_blob);
/*!
* \brief Execute the graph, update output.
*/
void Run();
private:
/*! \brief Setup the temporal storage */
void SetupStorage();
/*! \brief Setup the executors */
void SetupOpExecs();
/*!
* \brief Create a executtion function given input.
* \param attrs The node attributes
* \param args The arguments to the functor, including inputs and outputs.
* \param num_inputs Number of inputs
* \return The created executor.
*/
std::function<void()> CreateTVMOp(const NodeAttrs& attrs,
const std::vector<DLTensor>& args,
size_t num_inputs);
/*! \brief The graph */
Graph graph_;
/*! \brief The code module */
tvm::runtime::Module module_;
/*! \brief execution context */
TVMContext ctx_;
/*! \brief common storage pool */
std::vector<DLTensor*> storage_pool_;
/*! \brief data shape of each node entry */
std::vector<TShape> data_shape_;
/*! \brief data entry of each node */
std::vector<DLTensor> data_entry_;
/*! \brief operator on each node */
std::vector<std::function<void()> > op_execs_;
};
} // namespace runtime
} // namespace nnvm
#endif // NNVM_RUNTIME_GRAPH_EXECUTOR_H_
nnvm/src/runtime/graph_runtime.cc deleted 100644 → 0
/*!
* Copyright (c) 2017 by Contributors
* \file graph_executor.cc
*/
#include <dmlc/memory_io.h>
#include <dmlc/json.h>
#include <tvm/runtime/packed_func.h>
#include <tvm/runtime/registry.h>
#include <numeric>
#include "./graph_runtime.h"
namespace nnvm {
namespace runtime {
/*! \brief macro to do C API call */
#define TVM_CCALL(func) \
{ \
int ret = (func); \
CHECK_EQ(ret, 0) \
<< TVMGetLastError(); \
}
using ::tvm::runtime::PackedFunc;
using ::tvm::runtime::TVMArgs;
using ::tvm::runtime::TVMRetValue;
/*!
* \brief Minimum graph structure for deployment
* This is a minimum graph executor, embedded in TVM runtime
* without any framework dependency.
*
* This runtime can be acccesibly in various language via
* TVM runtime PackedFunc API.
*/
class GraphRuntime : public ::tvm::runtime::ModuleNode {
public:
~GraphRuntime() {
for (DLTensor* t : storage_pool_) {
TVM_CCALL(TVMArrayFree(t));
}
}
/*!
* \brief Get member function to front-end
* \param name The name of the function.
* \param sptr_to_self The pointer to the module node.
* \return The corresponding member function.
*/
tvm::runtime::PackedFunc GetFunction(
const std::string& name,
const std::shared_ptr<ModuleNode>& sptr_to_self) final;
/*!
* \return The type key of the executor.
*/
const char* type_key() const final {
return "GraphRuntime";
}
void Run() {
// setup the array and requirements.
for (size_t i = 0; i < op_execs_.size(); ++i) {
if (op_execs_[i]) op_execs_[i]();
}
}
/*!
* \brief Initialize the graph executor with graph and context.
* \param graph The execution graph.
* \param module The module containing the compiled functions.
* \param ctx The context where the graph should sit on
*/
void Init(const std::string& graph_json,
tvm::runtime::Module module,
TVMContext ctx) {
std::istringstream is(graph_json);
dmlc::JSONReader reader(&is);
this->Load(&reader);
module_ = module;
ctx_ = ctx;
this->SetupStorage();
this->SetupOpExecs();
}
/*!
* \brief Get the input index given the name of input.
* \param name The name of the input.
* \return The index of input.
*/
int GetInputIndex(const std::string& name) {
for (size_t i = 0; i< input_nodes_.size(); ++i) {
uint32_t nid = input_nodes_[i];
if (nodes_[nid].name == name) {
return static_cast<int>(i);
}
}
LOG(FATAL) << "cannot find " << name << " among input";
return -1;
}
/*!
* \brief set index-th input to the graph.
* \param index The input index.
* \param data The input data.
*/
void SetInput(int index, DLTensor* data_in) {
CHECK_LT(static_cast<size_t>(index), input_nodes_.size());
uint32_t eid = this->entry_id(input_nodes_[index], 0);
TVM_CCALL(TVMArrayCopyFromTo(data_in, &data_entry_[eid], nullptr));
}
/*!
* \brief Copy index-th output to data_out.
* \param index The output index.
* \param data_out the output data.
*/
void GetOutput(int index, DLTensor* data_out) {
CHECK_LT(static_cast<size_t>(index), outputs_.size());
uint32_t eid = this->entry_id(outputs_[index]);
TVM_CCALL(TVMArrayCopyFromTo(&data_entry_[eid], data_out, nullptr));
}
/*!
* \brief Load parameters from binary stream
* \param strm The input stream.
*/
void LoadParams(dmlc::Stream* strm);
/*!
* \brief Load parameters from parameter blob.
* \param param_blob A binary blob of parameter.
*/
void LoadParams(const std::string& param_blob) {
dmlc::MemoryStringStream strm(const_cast<std::string*>(&param_blob));
this->LoadParams(&strm);
}
private:
// Node entry
struct NodeEntry {
uint32_t node_id;
uint32_t index;
uint32_t version;
// JSON Loader
void Load(dmlc::JSONReader *reader) {
reader->BeginArray();
CHECK(reader->NextArrayItem()) << "invalid json format";
reader->Read(&node_id);
CHECK(reader->NextArrayItem()) << "invalid json format";
reader->Read(&index);
if (reader->NextArrayItem()) {
reader->Read(&version);
CHECK(!reader->NextArrayItem()) << "invalid json format";
} else {
version = 0;
}
}
};
// Node
struct Node {
// operator type in string
std::string op_type;
// name of the op
std::string name;
// parameters
TVMOpParam param;
// inputs
std::vector<NodeEntry> inputs;
// control deps
std::vector<uint32_t> control_deps;
// JSON Loader
void Load(dmlc::JSONReader *reader) {
reader->BeginObject();
std::unordered_map<std::string, std::string> dict;
int bitmask = 0;
std::string key;
while (reader->NextObjectItem(&key)) {
if (key == "op") {
reader->Read(&op_type);
bitmask |= 1;
} else if (key == "name") {
reader->Read(&name);
bitmask |= 2;
} else if (key == "inputs") {
reader->Read(&inputs);
bitmask |= 4;
} else if (key == "attr") {
reader->Read(&dict);
param.Init(dict);
} else if (key == "control_deps") {
reader->Read(&control_deps);
} else {
LOG(FATAL) << "do not support key" << key;
}
}
CHECK_EQ(bitmask, 1|2|4) << "invalid format";
}
};
struct GraphAttr {
size_t storage_num_not_alloctaed{0};
std::vector<int> storage_id;
std::vector<std::string> dltype;
std::vector<std::vector<int64_t> > shape;
// The graph attribute fields.
void Load(dmlc::JSONReader *reader) {
reader->BeginObject();
int bitmask = 0;
std::string key, type;
while (reader->NextObjectItem(&key)) {
if (key == "dltype") {
reader->BeginArray();
CHECK(reader->NextArrayItem());
reader->Read(&type);
CHECK_EQ(type, "list_str");
CHECK(reader->NextArrayItem());
reader->Read(&dltype);
CHECK(!reader->NextArrayItem());
bitmask |= 1;
} else if (key == "storage_id") {
reader->BeginArray();
CHECK(reader->NextArrayItem());
reader->Read(&type);
CHECK_EQ(type, "list_int");
CHECK(reader->NextArrayItem());
reader->Read(&storage_id);
CHECK(!reader->NextArrayItem());
bitmask |= 2;
} else if (key == "shape") {
reader->BeginArray();
CHECK(reader->NextArrayItem());
reader->Read(&type);
CHECK_EQ(type, "list_shape");
CHECK(reader->NextArrayItem());
reader->Read(&shape);
CHECK(!reader->NextArrayItem());
bitmask |= 4;
} else {
reader->BeginArray();
CHECK(reader->NextArrayItem());
reader->Read(&type);
if (type == "list_int") {
CHECK(reader->NextArrayItem());
std::vector<int> temp;
reader->Read(&temp);
} else if (type == "size_t") {
CHECK(reader->NextArrayItem());
size_t temp;
reader->Read(&temp);
} else {
LOG(FATAL) << "cannot skip graph attr " << key;
}
CHECK(!reader->NextArrayItem());
}
}
CHECK_EQ(bitmask, 1|2|4) << "invalid format";
}
};
// The graph attribute fields.
void Load(dmlc::JSONReader *reader) {
reader->BeginObject();
int bitmask = 0;
std::string key;
while (reader->NextObjectItem(&key)) {
if (key == "nodes") {
reader->Read(&nodes_);
bitmask |= 1;
} else if (key == "arg_nodes") {
reader->Read(&input_nodes_);
bitmask |= 2;
} else if (key == "node_row_ptr") {
reader->Read(&node_row_ptr_);
bitmask |= 4;
} else if (key == "heads") {
reader->Read(&outputs_);
bitmask |= 8;
} else if (key == "attrs") {
reader->Read(&attrs_);
bitmask |= 16;
}
}
CHECK_EQ(bitmask, 1|2|4|8|16) << "invalid format";
}
bool LoadDLTensor(dmlc::Stream* strm, DLTensor* tensor);
/*! \brief Setup the temporal storage */
void SetupStorage();
/*! \brief Setup the executors */
void SetupOpExecs();
/*!
* \brief Create a executtion function given input.
* \param attrs The node attributes
* \param args The arguments to the functor, including inputs and outputs.
* \param num_inputs Number of inputs
* \return The created executor.
*/
std::function<void()> CreateTVMOp(const TVMOpParam& attrs,
const std::vector<DLTensor>& args,
size_t num_inputs);
// Get node entry index.
uint32_t entry_id(uint32_t nid, uint32_t index) const {
return node_row_ptr_[nid] + index;
}
// Get node entry index.
uint32_t entry_id(const NodeEntry& e) const {
return entry_id(e.node_id, e.index);
}
// Number of node entries
uint32_t num_node_entries() const {
return node_row_ptr_.back();
}
// Number of nodes.
uint32_t num_nodes() const {
return static_cast<uint32_t>(nodes_.size());
}
// The graph nodes.
std::vector<Node> nodes_;
// The argument nodes.
std::vector<uint32_t> input_nodes_;
// used or quick entry indexing
std::vector<uint32_t> node_row_ptr_;
// output entries
std::vector<NodeEntry> outputs_;
// Additional graph attributes
GraphAttr attrs_;
/*! \brief The code module */
tvm::runtime::Module module_;
/*! \brief execution context */
TVMContext ctx_;
/*! \brief common storage pool */
std::vector<DLTensor*> storage_pool_;
/*! \brief data entry of each node */
std::vector<DLTensor> data_entry_;
/*! \brief operator on each node */
std::vector<std::function<void()> > op_execs_;
};
DMLC_REGISTER_PARAMETER(TVMOpParam);
bool GraphRuntime::LoadDLTensor(dmlc::Stream* strm, DLTensor* tensor) {
uint64_t header, reserved;
CHECK(strm->Read(&header, sizeof(header)))
<< "Invalid DLTensor file format";
CHECK(strm->Read(&reserved, sizeof(reserved)))
<< "Invalid DLTensor file format";
CHECK(header == kTVMNDArrayMagic)
<< "Invalid DLTensor file format";
CHECK(strm->Read(&tensor->ctx, sizeof(tensor->ctx)))
<< "Invalid DLTensor file format";
CHECK(strm->Read(&tensor->ndim, sizeof(tensor->ndim)))
<< "Invalid DLTensor file format";
CHECK(strm->Read(&tensor->dtype, sizeof(tensor->dtype)))
<< "Invalid DLTensor file format";
int ndim = tensor->ndim;
CHECK(strm->Read(tensor->shape, sizeof(int64_t) * ndim))
<< "Invalid DLTensor file format";
int64_t size = 1;
int type_size = tensor->dtype.bits / 8;
for (int i = 0; i < ndim; ++i) {
size *= tensor->shape[i];
}
int64_t data_byte_size;
CHECK(strm->Read(&data_byte_size, sizeof(data_byte_size)))
<< "Invalid DLTensor file format";
CHECK(data_byte_size == type_size * size)
<< "Invalid DLTensor file format";
CHECK(strm->Read(tensor->data, type_size * size))
<< "Invalid DLTensor file format";
return true;
}
void GraphRuntime::LoadParams(dmlc::Stream* strm) {
uint64_t header, reserved;
CHECK(strm->Read(&header))
<< "Invalid parameters file format";
CHECK(header == kTVMNDArrayListMagic)
<< "Invalid parameters file format";
CHECK(strm->Read(&reserved))
<< "Invalid parameters file format";
std::vector<std::string> names;
CHECK(strm->Read(&names))
<< "Invalid parameters file format";
uint64_t sz;
strm->Read(&sz, sizeof(sz));
size_t size = static_cast<size_t>(sz);
CHECK(size == names.size())
<< "Invalid parameters file format";
for (size_t i = 0; i < size; ++i) {
uint32_t in_idx = GetInputIndex(names[i]);
CHECK(LoadDLTensor(strm, &data_entry_[this->entry_id(input_nodes_[in_idx], 0)]))
<< "Invalid parameters file format";
}
}
void GraphRuntime::SetupStorage() {
// Grab saved optimization plan from graph.
std::vector<TVMType> vtype;
for (const std::string& s_type : attrs_.dltype) {
vtype.push_back(tvm::runtime::String2TVMType(s_type));
}
data_entry_.resize(num_node_entries());
// Find the maximum space size.
int max_id = 0;
for (size_t i = 0; i < attrs_.shape.size(); ++i) {
max_id = std::max(attrs_.storage_id[i] + 1, max_id);
}
for (uint32_t nid : input_nodes_) {
attrs_.storage_id[this->entry_id(nid, 0)] = max_id++;
}
// size of each storage pool entry
std::vector<size_t> pool_entry_bytes;
// Find the maximum space size.
for (size_t i = 0; i < attrs_.shape.size(); ++i) {
int storage_id = attrs_.storage_id[i];
size_t size = 1;
for (int64_t sz : attrs_.shape[i]) {
size *= static_cast<size_t>(sz);
}
CHECK_GE(storage_id, 0) << "Do not support runtime shape op";
DLDataType t = vtype[i];
size_t bits = t.bits * t.lanes;
CHECK_EQ(bits % 8U, 0U);
size_t bytes = (bits / 8U) * size;
size_t sid = static_cast<size_t>(storage_id);
if (sid >= pool_entry_bytes.size()) {
pool_entry_bytes.resize(sid + 1, 0);
}
pool_entry_bytes[sid] = std::max(pool_entry_bytes[sid], bytes);
}
// Allocate the space.
for (size_t i = 0; i < pool_entry_bytes.size(); ++i) {
int64_t shape[] = {static_cast<int64_t>(pool_entry_bytes[i] + 3) / 4};
DLTensor* tensor;
TVM_CCALL(TVMArrayAlloc(
shape, 1, kFloat, 32, 1, ctx_.device_type, ctx_.device_id, &tensor));
storage_pool_.push_back(tensor);
}
// Assign the pooled entries.
for (size_t i = 0; i < data_entry_.size(); ++i) {
int storage_id = attrs_.storage_id[i];
data_entry_[i] = *storage_pool_[storage_id];
data_entry_[i].shape = const_cast<int64_t*>(attrs_.shape[i].data());
data_entry_[i].ndim = static_cast<int>(attrs_.shape[i].size());
data_entry_[i].dtype = vtype[i];
}
}
/*! \brief Setup the executors */
void GraphRuntime::SetupOpExecs() {
op_execs_.resize(this->num_nodes());
// setup the array and requirements.
for (uint32_t nid = 0; nid < this->num_nodes(); ++nid) {
const auto& inode = nodes_[nid];
if (inode.op_type == "null") continue;
std::vector<DLTensor> args;
for (const auto& e : inode.inputs) {
args.push_back(data_entry_[this->entry_id(e)]);
}
for (uint32_t index = 0; index < inode.param.num_outputs; ++index) {
uint32_t eid = this->entry_id(nid, index);
args.push_back(data_entry_[eid]);
}
CHECK_EQ(inode.op_type, "tvm_op")
<< "transform the graph to tvm op5A";
op_execs_[nid] = CreateTVMOp(inode.param, args, inode.inputs.size());
}
}
std::function<void()> GraphRuntime::CreateTVMOp(
const TVMOpParam& param,
const std::vector<DLTensor>& args,
size_t num_inputs) {
struct OpArgs {
std::vector<DLTensor> args;
std::vector<TVMValue> arg_values;
std::vector<int> arg_tcodes;
std::vector<int64_t> shape_data;
};
std::shared_ptr<OpArgs> arg_ptr = std::make_shared<OpArgs>();
// setup address.
arg_ptr->args = std::move(args);
if (param.flatten_data) {
arg_ptr->shape_data.resize(arg_ptr->args.size());
}
for (size_t i = 0; i < arg_ptr->args.size(); ++i) {
TVMValue v;
DLTensor* t = &(arg_ptr->args[i]);
v.v_handle = t;
arg_ptr->arg_values.push_back(v);
arg_ptr->arg_tcodes.push_back(kArrayHandle);
if (param.flatten_data) {
arg_ptr->shape_data[i] = std::accumulate(
t->shape, t->shape + t->ndim, 1, std::multiplies<int64_t>());
t->ndim = 1;
t->shape = &(arg_ptr->shape_data[i]);
}
}
// get compiled function from module.
tvm::runtime::PackedFunc pf = module_.GetFunction(param.func_name, false);
CHECK(pf != nullptr) << "no such function in module: " << param.func_name;
auto fexec = [arg_ptr, pf] () {
TVMRetValue rv;
TVMArgs targs(arg_ptr->arg_values.data(),
arg_ptr->arg_tcodes.data(),
static_cast<int>(arg_ptr->arg_values.size()));
pf.CallPacked(targs, &rv);
};
return fexec;
}
PackedFunc GraphRuntime::GetFunction(
const std::string& name,
const std::shared_ptr<ModuleNode>& sptr_to_self) {
// return member functions during query.
if (name == "set_input") {
return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) {
if (args[0].type_code() == kStr) {
this->SetInput(this->GetInputIndex(args[0]), args[1]);
} else {
this->SetInput(args[0], args[1]);
}
});
} else if (name == "get_output") {
return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) {
this->GetOutput(args[0], args[1]);
});
} else if (name == "run") {
return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) {
this->Run();
});
} else if (name == "load_params") {
return PackedFunc([sptr_to_self, this](TVMArgs args, TVMRetValue* rv) {
this->LoadParams(args[0].operator std::string());
});
} else {
return PackedFunc();
}
}
tvm::runtime::Module GraphRuntimeCreate(std::string sym_json,
tvm::runtime::Module m,
int device_type,
int device_id) {
TVMContext ctx;
ctx.device_type = static_cast<DLDeviceType>(device_type);
ctx.device_id = device_id;
std::shared_ptr<GraphRuntime> exec = std::make_shared<GraphRuntime>();
exec->Init(sym_json, m, ctx);
return tvm::runtime::Module(exec);
}
TVM_REGISTER_GLOBAL("nnvm.runtime.create")
.set_body([](TVMArgs args, TVMRetValue *rv) {
*rv = GraphRuntimeCreate(args[0], args[1], args[2], args[3]);
});
TVM_REGISTER_GLOBAL("nnvm.runtime.remote_create")
.set_body([](TVMArgs args, TVMRetValue *rv) {
void* mhandle = args[1];
*rv = GraphRuntimeCreate(args[0],
*static_cast<tvm::runtime::Module*>(mhandle),
args[2], args[3]);
});
} // namespace runtime
} // namespace nnvm
nnvm/src/runtime/graph_runtime.h deleted 100644 → 0
/*!
* Copyright (c) 2017 by Contributors
*
* Runtime module for graph deployment.
*
* \file graph_executor.h
*/
#ifndef NNVM_RUNTIME_GRAPH_RUNTIME_H_
#define NNVM_RUNTIME_GRAPH_RUNTIME_H_
#include <dmlc/parameter.h>
#include <string>
namespace nnvm {
namespace runtime {
/*! \brief Magic number for NDArray file */
constexpr uint64_t kTVMNDArrayMagic = 0xDD5E40F096B4A13F;
/*! \brief Magic number for NDArray list file */
constexpr uint64_t kTVMNDArrayListMagic = 0xF7E58D4F05049CB7;
/*! \brief operator attributes about tvm op */
struct TVMOpParam : public dmlc::Parameter<TVMOpParam> {
std::string func_name;
uint32_t num_inputs;
uint32_t num_outputs;
uint32_t flatten_data;
DMLC_DECLARE_PARAMETER(TVMOpParam) {
DMLC_DECLARE_FIELD(func_name);
DMLC_DECLARE_FIELD(num_inputs).set_default(1);
DMLC_DECLARE_FIELD(num_outputs).set_default(1);
DMLC_DECLARE_FIELD(flatten_data).set_default(0);
}
};
} // namespace runtime
} // namespace nnvm
#endif // NNVM_RUNTIME_GRAPH_RUNTIME_H_
nnvm/tests/python/compiler/test_build.py
import numpy as np
import tvm
from tvm.contrib import graph_runtime
import nnvm.symbol as sym
import nnvm.compiler
import nnvm.runtime
from nnvm.compiler.build_module import _run_graph, precompute_prune
def test_compile():
......@@ -14,7 +14,7 @@ def test_compile():
dtype = tvm.float32
shape_dict = {"x": shape, "y": shape}
def verify(graph, lib):
m = nnvm.runtime.create(graph, lib, tvm.cpu(0))
m = graph_runtime.create(graph, lib, tvm.cpu(0))
# get member functions
set_input, run, get_output = m["set_input"], m["run"], m["get_output"]
na = tvm.nd.array(np.random.uniform(size=shape).astype(dtype))
......@@ -67,7 +67,7 @@ def test_precompute_prune():
graph, lib, params = nnvm.compiler.build(
z, "llvm", shape={"y": ny.shape}, params=params)
assert graph.index.num_nodes == 4
m = nnvm.runtime.create(graph, lib, tvm.cpu(0))
m = graph_runtime.create(graph, lib, tvm.cpu(0))
params["y"] = ny
res = tvm.nd.empty(shape)
m.run(**params)
......
nnvm/tests/python/compiler/test_compiler_cache.py
import numpy as np
import tvm
from tvm.contrib import graph_runtime
import nnvm.symbol as sym
import nnvm.compiler
import nnvm.runtime
def test_compile_cache():
x = sym.Variable("x")
......@@ -12,7 +12,7 @@ def test_compile_cache():
dtype = tvm.float32
shape_dict = {"x": shape, "y": shape}
def verify(graph, lib):
m = nnvm.runtime.create(graph, lib, tvm.cpu(0))
m = graph_runtime.create(graph, lib, tvm.cpu(0))
# get member functions
na = tvm.nd.array(np.random.uniform(size=shape).astype(dtype))
nb = tvm.nd.array(np.random.uniform(size=shape).astype(dtype))
......
nnvm/tests/python/compiler/test_op_fusion.py
......@@ -2,6 +2,7 @@ import nnvm
import numpy as np
import tvm
import topi
from tvm.contrib import graph_runtime
from nnvm import symbol as sym
from nnvm.compiler import graph_util, graph_attr
from nnvm.testing import ctx_list
......@@ -17,7 +18,7 @@ def test_ewise_injective():
for target, ctx in ctx_list():
graph, lib, _ = nnvm.compiler.build(y, target, shape_dict)
assert graph.index.num_nodes == 2
m = nnvm.runtime.create(graph, lib, ctx)
m = graph_runtime.create(graph, lib, ctx)
x_np = np.random.uniform(size=dshape).astype(dtype)
m.run(x=x_np)
out = m.get_output(0, tvm.nd.empty((10, 6)))
......@@ -39,7 +40,7 @@ def test_conv_ewise_injective():
for target, ctx in ctx_list():
graph, lib, _ = nnvm.compiler.build(y, target, shape_dict)
m = nnvm.runtime.create(graph, lib, ctx)
m = graph_runtime.create(graph, lib, ctx)
# print(graph.ir(join_entry_attrs=["shape"]))
assert graph.index.num_nodes == 5
# set input
......@@ -66,7 +67,7 @@ def test_injective_reduce_injective():
for target, ctx in ctx_list():
graph, lib, _ = nnvm.compiler.build(y, target, shape_dict)
m = nnvm.runtime.create(graph, lib, ctx)
m = graph_runtime.create(graph, lib, ctx)
assert graph.index.num_nodes == 2
data = np.random.uniform(size=dshape).astype(dtype)
m.run(x=data)
......
nnvm/tests/python/compiler/test_rpc_exec.py
from tvm.contrib import util, rpc
import tvm
from tvm.contrib import util, rpc, graph_runtime
import nnvm.symbol as sym
import nnvm.compiler
import nnvm.runtime
import numpy as np
def test_rpc_executor():
......@@ -29,7 +28,7 @@ def test_rpc_executor():
rlib = remote.load_module("net.o")
# Create remotemodule
m = nnvm.runtime.create(graph, rlib, remote.cpu(0))
m = graph_runtime.create(graph, rlib, remote.cpu(0))
# get member functions
set_input, run, get_output = m["set_input"], m["run"], m["get_output"]
na = tvm.nd.array(np.ones(shape).astype(dtype), ctx)
......
nnvm/tests/python/compiler/test_top_level1.py
import numpy as np
import tvm
from tvm.contrib import graph_runtime
import topi
import nnvm.symbol as sym
import nnvm.compiler
import nnvm.runtime
from nnvm.testing.config import ctx_list
def test_relu():
......@@ -15,7 +15,7 @@ def test_relu():
oshape = dshape
for target, ctx in ctx_list():
graph, lib, _ = nnvm.compiler.build(y, target, {"x": dshape})
m = nnvm.runtime.create(graph, lib, ctx)
m = graph_runtime.create(graph, lib, ctx)
data = np.random.uniform(size=dshape).astype(dtype)
m.run(x=data)
data = (data < 0) * data * 0.3 + (data>0) * data - 0.2
......@@ -32,7 +32,7 @@ def test_exp():
oshape = dshape
for target, ctx in ctx_list():
graph, lib, _ = nnvm.compiler.build(y, target, {"x": dshape})
m = nnvm.runtime.create(graph, lib, ctx)
m = graph_runtime.create(graph, lib, ctx)
data = np.random.uniform(size=dshape).astype(dtype)
m.run(x=data)
out = m.get_output(0, tvm.nd.empty(oshape, dtype))
......@@ -49,7 +49,7 @@ def test_log():
for target, ctx in ctx_list():
with nnvm.compiler.build_config(opt_level=1):
graph, lib, _ = nnvm.compiler.build(y, target, {"x": dshape})
m = nnvm.runtime.create(graph, lib, ctx)
m = graph_runtime.create(graph, lib, ctx)
data = np.random.uniform(size=dshape).astype(dtype)
m.run(x=data)
out = m.get_output(0, tvm.nd.empty(oshape, dtype))
......@@ -66,7 +66,7 @@ def test_tanh():
for target, ctx in ctx_list():
with nnvm.compiler.build_config(opt_level=1):
graph, lib, _ = nnvm.compiler.build(y, target, {"x": dshape})
m = nnvm.runtime.create(graph, lib, ctx)
m = graph_runtime.create(graph, lib, ctx)
data = np.random.uniform(size=dshape).astype(dtype)
m.run(x=data)
out = m.get_output(0, tvm.nd.empty(oshape, dtype))
......@@ -82,7 +82,7 @@ def test_sigmoid():
oshape = dshape
for target, ctx in ctx_list():
graph, lib, _ = nnvm.compiler.build(y, target, {"x": dshape})
m = nnvm.runtime.create(graph, lib, ctx)
m = graph_runtime.create(graph, lib, ctx)
data = np.random.uniform(size=dshape).astype(dtype)
m.run(x=data)
out = m.get_output(0, tvm.nd.empty(oshape, dtype))
......@@ -99,7 +99,7 @@ def test_softmax():
for target, ctx in ctx_list():
with nnvm.compiler.build_config(opt_level=1):
graph, lib, _ = nnvm.compiler.build(y, target, {"x": dshape})
m = nnvm.runtime.create(graph, lib, ctx)
m = graph_runtime.create(graph, lib, ctx)
data = np.random.uniform(size=dshape).astype(dtype)
m.run(x=data)
out = m.get_output(0, tvm.nd.empty(oshape, dtype))
......@@ -116,7 +116,7 @@ def test_log_softmax():
for target, ctx in ctx_list():
with nnvm.compiler.build_config(opt_level=1):
graph, lib, _ = nnvm.compiler.build(y, target, {"x": dshape})
m = nnvm.runtime.create(graph, lib, ctx)
m = graph_runtime.create(graph, lib, ctx)
data = np.random.uniform(size=dshape).astype(dtype)
m.run(x=data)
out = m.get_output(0, tvm.nd.empty(oshape, dtype))
......@@ -136,7 +136,7 @@ def test_dense():
}
for target, ctx in ctx_list():
graph, lib, _ = nnvm.compiler.build(y, target, shape)
m = nnvm.runtime.create(graph, lib, ctx)
m = graph_runtime.create(graph, lib, ctx)
x_np = np.random.uniform(size=shape["x"]).astype(dtype)
w_np = np.random.uniform(size=shape["dense_weight"]).astype(dtype)
b_np = np.random.uniform(size=shape["dense_bias"]).astype(dtype)
......@@ -162,7 +162,7 @@ def test_batchnorm():
for target, ctx in ctx_list():
graph, lib, _ = nnvm.compiler.build(y, "llvm", {"x": shape})
m = nnvm.runtime.create(graph, lib, tvm.cpu(0))
m = graph_runtime.create(graph, lib, tvm.cpu(0))
x_np = np.random.uniform(size=shape).astype(dtype)
mean_np = np.random.uniform(size=shape[1]).astype(dtype)
var_np = np.random.uniform(size=shape[1]).astype(dtype)
......
nnvm/tests/python/compiler/test_top_level2.py
import numpy as np
import tvm
from tvm.contrib import graph_runtime
import topi
import nnvm.symbol as sym
import nnvm.compiler
import nnvm.runtime
from nnvm.testing.config import ctx_list
......@@ -19,7 +19,7 @@ def test_conv2d():
shape_dict = {"x": dshape}
for target, ctx in ctx_list():
graph, lib, _ = nnvm.compiler.build(y, target, shape_dict)
m = nnvm.runtime.create(graph, lib, ctx)
m = graph_runtime.create(graph, lib, ctx)
data = tvm.nd.array(np.random.uniform(size=dshape).astype(dtype))
kernel = tvm.nd.array(np.random.uniform(size=kshape).astype(dtype))
bias = tvm.nd.array(np.random.uniform(size=kshape[0]).astype(dtype))
......@@ -42,7 +42,7 @@ def test_grouped_conv2d():
shape_dict = {"x": dshape}
for target, ctx in ctx_list():
graph, lib, _ = nnvm.compiler.build(y, target, shape_dict)
m = nnvm.runtime.create(graph, lib, ctx)
m = graph_runtime.create(graph, lib, ctx)
data = tvm.nd.array(np.random.uniform(size=dshape).astype(dtype))
kernel = tvm.nd.array(np.random.uniform(size=kshape).astype(dtype))
bias = tvm.nd.array(np.random.uniform(size=kshape[0]).astype(dtype))
......@@ -63,7 +63,7 @@ def test_max_pool2d():
shape_dict = {"x": dshape}
for target, ctx in ctx_list():
graph, lib, _ = nnvm.compiler.build(y, target, shape_dict)
m = nnvm.runtime.create(graph, lib, ctx)
m = graph_runtime.create(graph, lib, ctx)
data = tvm.nd.array(np.random.uniform(size=dshape).astype(dtype))
m.run(x=data)
out = m.get_output(0, tvm.nd.empty(oshape, dtype))
......@@ -80,7 +80,7 @@ def test_avg_pool2d():
shape_dict = {"x": dshape}
for target, ctx in ctx_list():
graph, lib, _ = nnvm.compiler.build(y, target, shape_dict)
m = nnvm.runtime.create(graph, lib, ctx)
m = graph_runtime.create(graph, lib, ctx)
data = tvm.nd.array(np.random.uniform(size=dshape).astype(dtype))
m.run(x=data)
out = m.get_output(0, tvm.nd.empty(oshape, dtype))
......@@ -97,7 +97,7 @@ def test_global_max_pool2d():
shape_dict = {"x": dshape}
for target, ctx in ctx_list():
graph, lib, _ = nnvm.compiler.build(y, target, shape_dict)
m = nnvm.runtime.create(graph, lib, ctx)
m = graph_runtime.create(graph, lib, ctx)
data = tvm.nd.array(np.random.uniform(size=dshape).astype(dtype))
m.run(x=data)
out = m.get_output(0, tvm.nd.empty(oshape, dtype))
......@@ -114,7 +114,7 @@ def test_global_avg_pool2d():
shape_dict = {"x": dshape}
for target, ctx in ctx_list():
graph, lib, _ = nnvm.compiler.build(y, target, shape_dict)
m = nnvm.runtime.create(graph, lib, ctx)
m = graph_runtime.create(graph, lib, ctx)
data = tvm.nd.array(np.random.uniform(size=dshape).astype(dtype))
m.run(x=data)
out = m.get_output(0, tvm.nd.empty(oshape, dtype))
......
nnvm/tests/python/compiler/test_top_level4.py
import numpy as np
import tvm
from tvm.contrib import graph_runtime
import topi
import nnvm.symbol as sym
import nnvm.compiler
import nnvm.runtime
from nnvm.testing.config import ctx_list
def verify_transpose(dshape, axes):
......@@ -16,7 +16,7 @@ def verify_transpose(dshape, axes):
dtype = "float32"
for target, ctx in ctx_list():
graph, lib, _ = nnvm.compiler.build(y, target, {"x": dshape})
m = nnvm.runtime.create(graph, lib, ctx)
m = graph_runtime.create(graph, lib, ctx)
# set input
data = tvm.nd.array(np.random.uniform(size=dshape).astype(dtype))
m.run(x=data)
......@@ -31,7 +31,7 @@ def verify_reduce(dshape, fnp, fsym, **kwargs):
dtype = "float32"
for target, ctx in ctx_list():
graph, lib, _ = nnvm.compiler.build(y, target, {"x": dshape})
m = nnvm.runtime.create(graph, lib, ctx)
m = graph_runtime.create(graph, lib, ctx)
# set input
data = np.random.uniform(size=dshape).astype(dtype)
out_np = fnp(data + 1, **kwargs)
......
nnvm/tests/python/frontend/mxnet/test_forward.py
......@@ -2,9 +2,9 @@ import numpy as np
import topi
import tvm
from tvm.contrib import graph_runtime
import nnvm.symbol as sym
import nnvm.compiler
import nnvm.runtime
from nnvm.testing.config import ctx_list
from nnvm import frontend
import mxnet as mx
......@@ -28,7 +28,7 @@ def test_mxnet_frontend_impl(mx_symbol, data_shape=(1, 3, 224, 224), out_shape=(
dshape = x.shape
shape_dict = {'data': dshape}
graph, lib, params = nnvm.compiler.build(new_sym, target, shape_dict, params=params)
m = nnvm.runtime.create(graph, lib, ctx)
m = graph_runtime.create(graph, lib, ctx)
# set inputs
m.set_input("data", tvm.nd.array(x.astype(dtype)))
m.set_input(**params)
......
nnvm/tutorials/mobilenet_inference_gpu.py
......@@ -8,10 +8,9 @@ This is an example of using NNVM to compile MobileNet model and deploy its infer
To begin with, we import nnvm(for compilation) and TVM(for deployment).
"""
import tvm
from tvm.contrib import nvcc, graph_runtime
import nnvm.compiler
import nnvm.runtime
import nnvm.testing
from tvm.contrib import nvcc
######################################################################
# Register the NVCC Compiler Option
......@@ -56,7 +55,7 @@ net, params = nnvm.testing.mobilenet.get_workload(
#
# To compile the graph, we call the build function with the graph
# configuration and parameters.
# When parameters are provided, NNVM will pre-compute certain part of the graph if possible,
# When parameters are provided, NNVM will pre-compute certain part of the graph if possible,
# the new parameter set returned as the third return value.
graph, lib, params = nnvm.compiler.build(
......@@ -66,12 +65,14 @@ graph, lib, params = nnvm.compiler.build(
# Run the Compiled Module
# -----------------------
#
# To deploy the module, we call :any:`nnvm.runtime.create` passing in the graph the lib and context.
# To deploy the module, we call :any:`tvm.contrib.graph_runtime.create` passing in the graph the lib and context.
# Thanks to TVM, we can deploy the compiled module to many platforms and languages.
# The deployment module is designed to contain minimum dependencies.
# This example runs on the same machine.
#
# Note that the code below no longer depends on NNVM, and only relies TVM's runtime to run(deploy).
module = nnvm.runtime.create(graph, lib, ctx)
module = graph_runtime.create(graph, lib, ctx)
# set input
module.set_input(**params)
# run
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment