Commit ee64bda4 by guoyuxuan

init commit

parents

Too many changes to show.

To preserve performance only 1000 of 1000+ files are displayed.

source /tools/module_env.sh
module load caffe-dep/cpu
module load cuda-cudnn/10.2-7.6.5
module load python3/3.6.8
module load git/2.17.1
module load gcc/5.4.0
module load llvm/7.0.0
module load cmake/3.15.7
export TVM_HOME=/home/S/guoyuxuan/code/tvm08dev/tvm
export PYTHONPATH=$TVM_HOME/python:$TVM_HOME/topi/python:$TVM_HOME/vta/python:${PYTHONPATH}
export VTA_HW_PATH=$TVM_HOME/3rdparty/vta-hw
export PATH=$PATH:/home/S/guoyuxuan/sbt/bin/
source ../tvm-exp/venv/bin/activate
//
// Created by bb on 2020/10/15.
//
#include <tvm/tir/var.h>
// #include <tvm/te/operation.h>
// #include <tvm/te/tensor.h>
#include <tvm/te/schedule.h>
#include <tvm/te/schedule_pass.h>
// #include <tvm/tir/buffer.h>
#include <tvm/tir/stmt_functor.h>
#include <tvm/tsl/te/operation.h>
#include <tvm/tsl/te/tensor.h>
#include <tvm/tsl/tir/buffer.h>
#include <tvm/tsl/tir/expr.h>
// #include <tvm/tir/expr.h>
#include <tvm/node/repr_printer.h>
#include <tvm/runtime/c_runtime_api.h>
#include <tvm/tsl/tsl/ast_dumper.h>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
using namespace tvm;
int main() {
int M = 256;
int N = 128;
int K = 512;
te::TensorUnion A = te::tsl_placeholder({1, 1}, {M, K}, DataType::Float(32), "A");
te::TensorUnion B = te::tsl_placeholder({1, 1}, {K, N}, DataType::Float(32), "B");
// te::Tensor A = te::placeholder({M, K}, DataType::Float(32), "A");
// te::Tensor B = te::placeholder({K, N}, DataType::Float(32), "B");
tir::IterVar k(Range(0, K), tir::Var(), tir::IterVarType::kCommReduce); // reduce axis
te::Tensor C =
te::compute({M, N}, std::function<PrimExpr(tir::Var, tir::Var)>([=](tir::Var i, tir::Var j) {
return sum(A[i][k] * B[k][j], {k});
}),
"C");
te::Schedule s = te::create_schedule({C->op});
// lower
s = s.normalize();
auto bound = te::InferBound(s);
auto stmt = te::ScheduleOps(s, bound, true);
tsl::dump_ast(stmt, "./schOps.dot");
return 0;
// tir::IterVar inner(Range(),tir::Var(),tir::IterVarType::kDataPar);
// //sch[C].split(C->op->root_iter_vars()[0],4,&outer,&inner);
// tir::IterVar threadidxx(Range(),tir::Var(),tir::IterVarType::kThreadIndex,"pipeline");
// //sch[C].bind(C->op->root_iter_vars()[0],threadidxx);
// sch[C].pragma(C->op->root_iter_vars()[0],"PIPELINE",0);
// sch=sch.normalize();
// auto boundmap=te::InferBound(sch);
// auto result=te::ScheduleOps(sch,boundmap,true);
// //std::cout<<((tir::ProducerRealizeNode*)result.get())->bounds<<std::endl;
// dump_ast(result,"./schOps.dot");
// auto pf=te::SchedulePostProcToPrimFunc({A,B},result,Optional<Map<te::Tensor,te::Buffer>>());
// dump_ast(pf.get()->body,"./schPP.dot");
// int x=1;
}
//
// Created by bb on 2020/10/15.
//
#include <tvm/tir/var.h>
// #include <tvm/te/operation.h>
// #include <tvm/te/tensor.h>
#include <tvm/te/schedule.h>
#include <tvm/te/schedule_pass.h>
// #include <tvm/tir/buffer.h>
#include <tvm/tir/stmt_functor.h>
#include <tvm/tsl/te/operation.h>
#include <tvm/tsl/te/tensor.h>
#include <tvm/tsl/tir/buffer.h>
#include <tvm/tsl/tir/expr.h>
// #include <tvm/tir/expr.h>
#include <tvm/node/repr_printer.h>
#include <tvm/runtime/c_runtime_api.h>
#include <tvm/tsl/tsl/ast_dumper.h>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
using namespace tvm;
int main() {
int M = 256;
int N = 128;
int K = 512;
te::Tensor A = te::placeholder({M, K}, DataType::Float(32), "A");
te::Tensor B = te::placeholder({K, N}, DataType::Float(32), "B");
tir::IterVar k(Range(0, K), tir::Var(), tir::IterVarType::kCommReduce); // reduce axis
te::Tensor C =
te::compute({M, N}, std::function<PrimExpr(tir::Var, tir::Var)>([=](tir::Var i, tir::Var j) {
return sum(A[i][k] * B[k][j], {k});
}),
"C");
te::Schedule s = te::create_schedule({C->op});
// lower
s = s.normalize();
auto bound = te::InferBound(s);
auto stmt = te::ScheduleOps(s, bound, true);
tsl::dump_ast(stmt, "./schOps.dot");
return 0;
// tir::IterVar inner(Range(),tir::Var(),tir::IterVarType::kDataPar);
// //sch[C].split(C->op->root_iter_vars()[0],4,&outer,&inner);
// tir::IterVar threadidxx(Range(),tir::Var(),tir::IterVarType::kThreadIndex,"pipeline");
// //sch[C].bind(C->op->root_iter_vars()[0],threadidxx);
// sch[C].pragma(C->op->root_iter_vars()[0],"PIPELINE",0);
// sch=sch.normalize();
// auto boundmap=te::InferBound(sch);
// auto result=te::ScheduleOps(sch,boundmap,true);
// //std::cout<<((tir::ProducerRealizeNode*)result.get())->bounds<<std::endl;
// dump_ast(result,"./schOps.dot");
// auto pf=te::SchedulePostProcToPrimFunc({A,B},result,Optional<Map<te::Tensor,te::Buffer>>());
// dump_ast(pf.get()->body,"./schPP.dot");
// int x=1;
}
//
// Created by bb on 2020/10/15.
//
#include <tvm/tir/var.h>
// #include <tvm/te/operation.h>
// #include <tvm/te/tensor.h>
#include <tvm/te/schedule.h>
#include <tvm/te/schedule_pass.h>
// #include <tvm/tir/buffer.h>
#include <tvm/tir/stmt_functor.h>
#include <tvm/tsl/te/operation.h>
#include <tvm/tsl/te/tensor.h>
#include <tvm/tsl/tir/buffer.h>
#include <tvm/tsl/tir/expr.h>
// #include <tvm/tir/expr.h>
#include <tvm/node/repr_printer.h>
#include <tvm/runtime/c_runtime_api.h>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
int main() {
tvm::te::TensorUnion U =
tvm::te::tsl_placeholder({1, 1}, {64, 64}, tvm::DataType::Float(32), "U");
tvm::Array<tvm::PrimExpr> indices = {0, 0};
auto load = U.operator()(indices).as<tvm::tir::TULoadNode>();
}
import tvm
from tvm import te
import sys
# compute
k = te.reduce_axis((0, 512), "k")
A = te.placeholder((256, 512), name="A")
B = te.placeholder((512, 128), name="B")
C = te.compute((256, 128), lambda x, y: te.sum(A[x, k] * B[k, y], axis=k), name="C")
# schedule
s = te.create_schedule(C.op)
# lower and build
# target = "cuda"
stmt = tvm.lower(s,[A, B, C], simple_mode=True)
# func = tvm.build(s, [A, B, C])
# print(func)
\ No newline at end of file
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
github:
description: "Open deep learning compiler stack for cpu, gpu and specialized accelerators"
homepage: https://tvm.apache.org/
labels:
- tvm
- compiler
- tensor
- deep-learning
- gpu
- opencl
- metal
- performance
- javascript
- rocm
- vulkan
- spirv
- machine-learning
# Run the following command to reformat a file:
# clang-format -i -style=Google <file>
# Or use clang-format-diff to only reformat the changed lines:
# https://clang.llvm.org/docs/ClangFormat.html
BasedOnStyle: Google
DerivePointerAlignment: false
ColumnLimit: 100
PointerAlignment: Left
# Github owner file
# List of code reviewers for TVM modules
# Global reviewers
* @dmlc/tvm-committers
# LLVM backends
src/codegen/llvm/* @aatluri
# ROCM runtime
src/runtime/rocm/* @aatluri
# SGX support
src/runtime/sgx/* @nhynes
apps/sgx/* @nhynes
# JVM language
jvm/* @yzhliu
# WebGL backends
src/runtime/opengl/* @phisiart
src/codegen/*opengl* @phisiart
# TOPI
topi/python/topi/* @Laurawly @Huyuwei
Thanks for participating in the TVM community! We use https://discuss.tvm.ai for any general usage questions and discussions. The issue tracker is used for actionable items such as feature proposals discussion, roadmaps, and bug tracking. You are always welcomed to post on the forum first :)
Issues that are inactive for a period of time may get closed. We adopt this policy so that we won't lose track of actionable issues that may fall at the bottom of the pile. Feel free to reopen a new one if you feel there is an additional problem that needs attention when an old one gets closed.
For bug reports, to help the developer act on the issues, please include a description of your environment, preferably a minimum script to reproduce the problem.
For feature proposals, list clear, small actionable items so we can track the progress of the change.
Thanks for contributing to TVM! Please refer to guideline https://tvm.apache.org/docs/contribute/ for useful information and tips. After the pull request is submitted, please request code reviews from [Reviewers](https://github.com/apache/incubator-tvm/blob/master/CONTRIBUTORS.md#reviewers) by @ them in the pull request thread.
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# GH actions.
# We use it to cover windows and mac builds
# Jenkins is still the primary CI
name: WinMacBuild
on:
push:
branches:
- main
pull_request:
branches:
- main
jobs:
Build:
strategy:
matrix:
os: [windows-2016, macOS-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v2
- name: Initialize submodules
run: git submodule update --recursive --init
- name: Lint Python
if: startsWith(matrix.os, 'macOS')
run: |
python3 -m pip install flake8
python3 -m flake8 . --count --select=E9,F63,F7 --show-source --statistics
- uses: actions/cache@v1
env:
CACHE_NUMBER: 0
with:
path: ~/conda_pkgs_dir
key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('conda/build-environment.yaml') }}
- uses: conda-incubator/setup-miniconda@v1
with:
activate-environment: tvm-build
channel-priority: strict
environment-file: conda/build-environment.yaml
auto-activate-base: false
use-only-tar-bz2: true
- name: Conda info
run: |
conda info
conda list
- name: Conda-Build@Win
if: startsWith(matrix.os, 'windows')
shell: cmd /C call {0}
run: >-
conda build --output-folder=conda/pkg conda/recipe &&
conda install tvm -c ./conda/pkg
- name: Conda-Build@MacOS
if: startsWith(matrix.os, 'macOS')
shell: bash -l {0}
run: >-
conda build --output-folder=conda/pkg conda/recipe &&
conda install tvm -c ./conda/pkg
- name: Test@Win
if: startsWith(matrix.os, 'windows')
shell: cmd /C call {0}
run: >-
python -m pytest -v tests/python/all-platform-minimal-test
- name: Test@MacOS
if: startsWith(matrix.os, 'macOS')
shell: bash -l {0}
run: >-
python -m pytest -v tests/python/all-platform-minimal-test
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
*.S
# C extensions
*.so
*.ll
.npm
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
.conda/
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
.hypothesis/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
docs/gen_modules
# PyBuilder
/target/
# IPython Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# dotenv
.env
# virtualenv
venv/
ENV/
# Spyder project settings
.spyderproject
# Rope project settings
.ropeproject
*~
*.pyc
*~
config.mk
config.cmake
Win32
*.dir
perf
*.wasm
.emscripten
## IOS
DerivedData/
## Java
*.class
jvm/*/target/
jvm/*/*/target/
*.worksheet
*.idea
*.iml
*.classpath
*.project
*.settings
*/node_modules/
## Various settings
*.pbxuser
!default.pbxuser
*.mode1v3
!default.mode1v3
*.mode2v3
!default.mode2v3
*.perspectivev3
!default.perspectivev3
xcuserdata/
.pkl_memoize_*
.emscripten*
.m2
# Compiled Dynamic libraries
*.so
*.dylib
*.dll
# Compiled Object files
*.slo
*.lo
*.o
*.obj
# Precompiled Headers
*.gch
*.pch
# Compiled Static libraries
*.lai
*.la
*.a
*.lib
# Executables
*.exe
*.out
*.app
## Other
*.moved-aside
*.xccheckout
*.xcscmblueprint
.DS_Store
tags
cscope*
*.lock
# vim temporary files
*.swp
*.swo
# TVM generated code
perf
.bash_history
*.json
*.params
*.onnx
*.h5
synset.txt
cat.jpg
cat.png
docs.tgz
cat.png
*.mlmodel
tvm_u.*
tvm_t.*
# Mac OS X
.DS_Store
# Jetbrain
.idea
.ipython
.jupyter
.nv
.pylint.d
.python_history
.pytest_cache
.local
cmake-build-debug
# Visual Studio
.vs
# Visual Studio Code
.vscode
# tmp file
.nfs*
# keys
*.pem
*.p12
*.pfx
*.cer
*.crt
*.der
# patch sentinel
patched.txt
# Python type checking
.mypy_cache/
.pyre/
# pipenv files
Pipfile
Pipfile.lock
# conda package artifacts
conda/Dockerfile.cuda*
conda/pkg
.node_repl_history
# nix files
.envrc
*.nix
[submodule "dmlc-core"]
path = 3rdparty/dmlc-core
url = https://github.com/dmlc/dmlc-core
[submodule "dlpack"]
path = 3rdparty/dlpack
url = https://github.com/dmlc/dlpack
[submodule "3rdparty/rang"]
path = 3rdparty/rang
url = https://github.com/agauniyal/rang
[submodule "3rdparty/vta-hw"]
path = 3rdparty/vta-hw
url = https://github.com/apache/incubator-tvm-vta
/* cma.h
*
* The MIT License (MIT)
*
* COPYRIGHT (C) 2017 Institute of Electronics and Computer Science (EDI), Latvia.
* AUTHOR: Rihards Novickis (rihards.novickis@edi.lv)
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#ifndef VTA_DE10_NANO_KERNEL_MODULE_CMA_H_
#define VTA_DE10_NANO_KERNEL_MODULE_CMA_H_
/* Should be defined in settings.mk file */
#ifndef CMA_IOCTL_MAGIC
#define CMA_IOCTL_MAGIC 0xf2
#endif
#define CMA_ALLOC_CACHED _IOC(_IOC_WRITE | _IOC_READ, CMA_IOCTL_MAGIC, 1, 4)
#define CMA_ALLOC_NONCACHED _IOC(_IOC_WRITE | _IOC_READ, CMA_IOCTL_MAGIC, 2, 4)
#define CMA_FREE _IOC(_IOC_WRITE, CMA_IOCTL_MAGIC, 3, 4)
#define CMA_GET_PHY_ADDR _IOC(_IOC_WRITE | _IOC_READ, CMA_IOCTL_MAGIC, 4, 4)
#define CMA_GET_SIZE _IOC(_IOC_WRITE | _IOC_READ, CMA_IOCTL_MAGIC, 5, 4)
#define CMA_IOCTL_MAXNR 5
#endif // VTA_DE10_NANO_KERNEL_MODULE_CMA_H_
/*
* The MIT License (MIT)
*
* COPYRIGHT (C) 2017 Institute of Electronics and Computer Science (EDI), Latvia.
* AUTHOR: Rihards Novickis (rihards.novickis@edi.lv)
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*
*/
/*!
* Copyright (c) 2018 by Contributors
* \file cma_api.cc
* \brief Application layer implementation for contigous memory allocation.
*/
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <unistd.h>
#include "cma_api.h"
#ifndef CMA_IOCTL_MAGIC
#define CMA_IOCTL_MAGIC 0xf2
#endif
#define CMA_ALLOC_CACHED _IOC(_IOC_WRITE | _IOC_READ, CMA_IOCTL_MAGIC, 1, 4)
#define CMA_ALLOC_NONCACHED _IOC(_IOC_WRITE | _IOC_READ, CMA_IOCTL_MAGIC, 2, 4)
#define CMA_FREE _IOC(_IOC_WRITE, CMA_IOCTL_MAGIC, 3, 4)
#define CMA_GET_PHY_ADDR _IOC(_IOC_WRITE | _IOC_READ, CMA_IOCTL_MAGIC, 4, 4)
#define CMA_GET_SIZE _IOC(_IOC_WRITE | _IOC_READ, CMA_IOCTL_MAGIC, 5, 4)
#define CMA_IOCTL_MAXNR 5
#ifndef CMA_DEBUG
#define CMA_DEBUG 0
#endif
#ifndef DRIVER_NODE_NAME
#define DRIVER_NODE_NAME "cma"
#endif
#if CMA_DEBUG == 1
#define __DEBUG(fmt, args...) printf("CMA_API_DEBUG: " fmt, ##args)
#else
#define __DEBUG(fmt, args...)
#endif
#define ROUND_UP(N, S) ((((N) + (S)-1) / (S)) * (S))
/* Private functions */
void* cma_alloc(size_t size, unsigned ioctl_cmd);
/* Global file descriptor */
int cma_fd = 0;
int cma_init(void) {
__DEBUG("Opening \"/dev/" DRIVER_NODE_NAME "\" file\n");
cma_fd = open("/dev/" DRIVER_NODE_NAME, O_RDWR);
if (cma_fd == -1) {
__DEBUG("Failed to initialize api - \"%s\"\n", strerror(errno));
return -1;
}
return 0;
}
int cma_release(void) {
__DEBUG("Closing \"/dev/" DRIVER_NODE_NAME "\" file\n");
if (close(cma_fd) == -1) {
__DEBUG("Failed to finilize api - \"%s\"\n", strerror(errno));
return -1;
}
return 0;
}
void* cma_alloc_cached(size_t size) { return cma_alloc(size, CMA_ALLOC_CACHED); }
void* cma_alloc_noncached(size_t size) { return cma_alloc(size, CMA_ALLOC_NONCACHED); }
int cma_free(void* mem) {
__DEBUG("Releasing contigous memory from 0x%x\n", (unsigned)mem);
unsigned data, v_addr;
/* save user space pointer value */
data = (unsigned)mem;
v_addr = (unsigned)mem;
if (ioctl(cma_fd, CMA_GET_SIZE, &data) == -1) {
__DEBUG("cma_free - ioctl command unsuccsessful - 0\n");
return -1;
}
/* data now contains size */
/* unmap memory */
munmap(mem, data);
/* free cma entry */
if (ioctl(cma_fd, CMA_FREE, &v_addr) == -1) {
__DEBUG("cma_free - ioctl command unsuccsessful - 1\n");
return -1;
}
return 0;
}
unsigned cma_get_phy_addr(void* mem) {
unsigned data;
__DEBUG("Getting physical address from 0x%x\n", (unsigned)mem);
/* save user space pointer value */
data = (unsigned)mem;
/* get physical address */
if (ioctl(cma_fd, CMA_GET_PHY_ADDR, &data) == -1) {
__DEBUG("cma_free - ioctl command unsuccsessful\n");
return 0;
}
/* data now contains physical address */
return data;
}
void* cma_alloc(size_t size, unsigned ioctl_cmd) {
unsigned data;
void* mem;
__DEBUG("Allocating 0x%x bytes of contigous memory\n", size);
/* Page align size */
size = ROUND_UP(size, getpagesize());
/* ioctl cmd to allocate contigous memory */
data = (unsigned)size;
if (ioctl(cma_fd, ioctl_cmd, &data) == -1) {
__DEBUG("cma_alloc - ioctl command unsuccsessful\n");
return NULL;
}
/* at this point phy_addr is written to data */
/* mmap memory */
mem = mmap(NULL, size, PROT_WRITE | PROT_READ, MAP_SHARED, cma_fd, data);
if (mem == MAP_FAILED) {
__DEBUG("cma_alloc - mmap unsuccsessful\n");
return NULL;
}
return mem;
}
# ==================== COMPILATION RELATED SETTINGS ====================
# Path to the kernel sources (from "./driver", if relative path is used)
KSOURCE_DIR=/opt/intel/linux-socfpga-rel_socfpga-4.9.78-ltsi_18.08.02_pr
# Cross compiler "prepend" string
CROSS_COMPILE=arm-linux-gnueabihf-
# Architecture
ARCH=arm
# Compile with debug information
CMA_DEBUG?=0
# ==================== DRIVER RELATED SETTINGS ====================
# Node name used in "/dev" folder
DRIVER_NODE_NAME="cma"
# Unique (across system) ioctl magic number. Every ioctl interface should have one.
CMA_IOC_MAGIC=0xf2
/*
* Copyright (c) 2009-2015 by llvm/compiler-rt contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
* \file builtin_fp16.h
* \brief Functions for conversion between fp32 and fp16, adopted from compiler-rt.
*/
#ifndef COMPILER_RT_BUILTIN_FP16_H_
#define COMPILER_RT_BUILTIN_FP16_H_
#ifdef _MSC_VER
#pragma warning(disable : 4305 4805)
#endif
#include <cstdint>
static inline uint32_t __clz(uint32_t x) {
// count leading zeros
int n = 32;
uint32_t y;
y = x >> 16;
if (y) {
n = n - 16;
x = y;
}
y = x >> 8;
if (y) {
n = n - 8;
x = y;
}
y = x >> 4;
if (y) {
n = n - 4;
x = y;
}
y = x >> 2;
if (y) {
n = n - 2;
x = y;
}
y = x >> 1;
if (y) return n - 2;
return n - x;
}
template <typename SRC_T, typename SRC_REP_T, int SRC_SIG_BITS, typename DST_T, typename DST_REP_T,
int DST_SIG_BITS>
static inline DST_T __truncXfYf2__(SRC_T a) {
// Various constants whose values follow from the type parameters.
// Any reasonable optimizer will fold and propagate all of these.
const int srcBits = sizeof(SRC_T) * 8;
const int srcExpBits = srcBits - SRC_SIG_BITS - 1;
const int srcInfExp = (1 << srcExpBits) - 1;
const int srcExpBias = srcInfExp >> 1;
const SRC_REP_T srcMinNormal = SRC_REP_T(1) << SRC_SIG_BITS;
const SRC_REP_T srcSignificandMask = srcMinNormal - 1;
const SRC_REP_T srcInfinity = (SRC_REP_T)srcInfExp << SRC_SIG_BITS;
const SRC_REP_T srcSignMask = SRC_REP_T(1) << (SRC_SIG_BITS + srcExpBits);
const SRC_REP_T srcAbsMask = srcSignMask - 1;
const SRC_REP_T roundMask = (SRC_REP_T(1) << (SRC_SIG_BITS - DST_SIG_BITS)) - 1;
const SRC_REP_T halfway = SRC_REP_T(1) << (SRC_SIG_BITS - DST_SIG_BITS - 1);
const SRC_REP_T srcQNaN = SRC_REP_T(1) << (SRC_SIG_BITS - 1);
const SRC_REP_T srcNaNCode = srcQNaN - 1;
const int dstBits = sizeof(DST_T) * 8;
const int dstExpBits = dstBits - DST_SIG_BITS - 1;
const int dstInfExp = (1 << dstExpBits) - 1;
const int dstExpBias = dstInfExp >> 1;
const int underflowExponent = srcExpBias + 1 - dstExpBias;
const int overflowExponent = srcExpBias + dstInfExp - dstExpBias;
const SRC_REP_T underflow = (SRC_REP_T)underflowExponent << SRC_SIG_BITS;
const SRC_REP_T overflow = (SRC_REP_T)overflowExponent << SRC_SIG_BITS;
const DST_REP_T dstQNaN = DST_REP_T(1) << (DST_SIG_BITS - 1);
const DST_REP_T dstNaNCode = dstQNaN - 1;
// Break a into a sign and representation of the absolute value
union SrcExchangeType {
SRC_T f;
SRC_REP_T i;
};
SrcExchangeType src_rep;
src_rep.f = a;
const SRC_REP_T aRep = src_rep.i;
const SRC_REP_T aAbs = aRep & srcAbsMask;
const SRC_REP_T sign = aRep & srcSignMask;
DST_REP_T absResult;
if (aAbs - underflow < aAbs - overflow) {
// The exponent of a is within the range of normal numbers in the
// destination format. We can convert by simply right-shifting with
// rounding and adjusting the exponent.
absResult = aAbs >> (SRC_SIG_BITS - DST_SIG_BITS);
absResult -= (DST_REP_T)(srcExpBias - dstExpBias) << DST_SIG_BITS;
const SRC_REP_T roundBits = aAbs & roundMask;
// Round to nearest
if (roundBits > halfway) absResult++;
// Ties to even
else if (roundBits == halfway)
absResult += absResult & 1;
} else if (aAbs > srcInfinity) {
// a is NaN.
// Conjure the result by beginning with infinity, setting the qNaN
// bit and inserting the (truncated) trailing NaN field.
absResult = (DST_REP_T)dstInfExp << DST_SIG_BITS;
absResult |= dstQNaN;
absResult |= ((aAbs & srcNaNCode) >> (SRC_SIG_BITS - DST_SIG_BITS)) & dstNaNCode;
} else if (aAbs >= overflow) {
// a overflows to infinity.
absResult = (DST_REP_T)dstInfExp << DST_SIG_BITS;
} else {
// a underflows on conversion to the destination type or is an exact
// zero. The result may be a denormal or zero. Extract the exponent
// to get the shift amount for the denormalization.
const int aExp = aAbs >> SRC_SIG_BITS;
const int shift = srcExpBias - dstExpBias - aExp + 1;
const SRC_REP_T significand = (aRep & srcSignificandMask) | srcMinNormal;
// Right shift by the denormalization amount with sticky.
if (shift > SRC_SIG_BITS) {
absResult = 0;
} else {
const bool sticky = significand << (srcBits - shift);
SRC_REP_T denormalizedSignificand = significand >> shift | sticky;
absResult = denormalizedSignificand >> (SRC_SIG_BITS - DST_SIG_BITS);
const SRC_REP_T roundBits = denormalizedSignificand & roundMask;
// Round to nearest
if (roundBits > halfway) absResult++;
// Ties to even
else if (roundBits == halfway)
absResult += absResult & 1;
}
}
// Apply the signbit to (DST_T)abs(a).
const DST_REP_T result = absResult | sign >> (srcBits - dstBits);
union DstExchangeType {
DST_T f;
DST_REP_T i;
};
DstExchangeType dst_rep;
dst_rep.i = result;
return dst_rep.f;
}
template <typename SRC_T, typename SRC_REP_T, int SRC_SIG_BITS, typename DST_T, typename DST_REP_T,
int DST_SIG_BITS>
static inline DST_T __extendXfYf2__(SRC_T a) {
// Various constants whose values follow from the type parameters.
// Any reasonable optimizer will fold and propagate all of these.
const int srcBits = sizeof(SRC_T) * 8;
const int srcExpBits = srcBits - SRC_SIG_BITS - 1;
const int srcInfExp = (1 << srcExpBits) - 1;
const int srcExpBias = srcInfExp >> 1;
const SRC_REP_T srcMinNormal = SRC_REP_T(1) << SRC_SIG_BITS;
const SRC_REP_T srcInfinity = (SRC_REP_T)srcInfExp << SRC_SIG_BITS;
const SRC_REP_T srcSignMask = SRC_REP_T(1) << (SRC_SIG_BITS + srcExpBits);
const SRC_REP_T srcAbsMask = srcSignMask - 1;
const SRC_REP_T srcQNaN = SRC_REP_T(1) << (SRC_SIG_BITS - 1);
const SRC_REP_T srcNaNCode = srcQNaN - 1;
const int dstBits = sizeof(DST_T) * 8;
const int dstExpBits = dstBits - DST_SIG_BITS - 1;
const int dstInfExp = (1 << dstExpBits) - 1;
const int dstExpBias = dstInfExp >> 1;
const DST_REP_T dstMinNormal = DST_REP_T(1) << DST_SIG_BITS;
// Break a into a sign and representation of the absolute value
union SrcExchangeType {
SRC_T f;
SRC_REP_T i;
};
SrcExchangeType src_rep;
src_rep.f = a;
const SRC_REP_T aRep = src_rep.i;
const SRC_REP_T aAbs = aRep & srcAbsMask;
const SRC_REP_T sign = aRep & srcSignMask;
DST_REP_T absResult;
// If sizeof(SRC_REP_T) < sizeof(int), the subtraction result is promoted
// to (signed) int. To avoid that, explicitly cast to SRC_REP_T.
if ((SRC_REP_T)(aAbs - srcMinNormal) < srcInfinity - srcMinNormal) {
// a is a normal number.
// Extend to the destination type by shifting the significand and
// exponent into the proper position and rebiasing the exponent.
absResult = (DST_REP_T)aAbs << (DST_SIG_BITS - SRC_SIG_BITS);
absResult += (DST_REP_T)(dstExpBias - srcExpBias) << DST_SIG_BITS;
}
else if (aAbs >= srcInfinity) {
// a is NaN or infinity.
// Conjure the result by beginning with infinity, then setting the qNaN
// bit (if needed) and right-aligning the rest of the trailing NaN
// payload field.
absResult = (DST_REP_T)dstInfExp << DST_SIG_BITS;
absResult |= (DST_REP_T)(aAbs & srcQNaN) << (DST_SIG_BITS - SRC_SIG_BITS);
absResult |= (DST_REP_T)(aAbs & srcNaNCode) << (DST_SIG_BITS - SRC_SIG_BITS);
} else if (aAbs) {
// a is denormal.
// renormalize the significand and clear the leading bit, then insert
// the correct adjusted exponent in the destination type.
const int scale = __clz(aAbs) - __clz(srcMinNormal);
absResult = (DST_REP_T)aAbs << (DST_SIG_BITS - SRC_SIG_BITS + scale);
absResult ^= dstMinNormal;
const int resultExponent = dstExpBias - srcExpBias - scale + 1;
absResult |= (DST_REP_T)resultExponent << DST_SIG_BITS;
} else {
// a is zero.
absResult = 0;
}
// Apply the signbit to (DST_T)abs(a).
const DST_REP_T result = absResult | (DST_REP_T)sign << (dstBits - srcBits);
union DstExchangeType {
DST_T f;
DST_REP_T i;
};
DstExchangeType dst_rep;
dst_rep.i = result;
return dst_rep.f;
}
#endif // COMPILER_RT_BUILTIN_FP16_H_
# Compiled Object files
*.slo
*.lo
*.o
*.obj
# Precompiled Headers
*.gch
*.pch
# Compiled Dynamic libraries
*.so
*.dylib
*.dll
# Fortran module files
*.mod
*.smod
# Compiled Static libraries
*.lai
*.la
*.a
*.lib
# Executables
*.exe
*.out
*.app
*~
build
bin
sudo: false
dist: trusty
language: cpp
os:
- linux
- osx
osx_image: xcode8
env:
# code analysis
- TASK=all_test
branches:
only:
- master
matrix:
exclude:
- os: osx
env: TASK=lint
# dependent apt packages
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- doxygen
- wget
- graphviz
- unzip
install:
- source tests/travis/setup.sh
script:
- tests/travis/run_test.sh
cache:
directories:
- ${HOME}/.cache/usr
###
# Set minimum version of CMake. Since command 'project' use
# VERSION sub-option we need at least 3.0.
# Note: If you use 2.6 or 2.4, God kills a kitten. Seriously.
cmake_minimum_required(VERSION 3.2 FATAL_ERROR)
####
# Set variables:
# * PROJECT_NAME
# * PROJECT_VERSION
project(dlpack VERSION 0.1.0 LANGUAGES C CXX)
#####
# Change the default build type from Debug to Release, while still
# supporting overriding the build type.
#
# The CACHE STRING logic here and elsewhere is needed to force CMake
# to pay attention to the value of these variables.
if(NOT CMAKE_BUILD_TYPE)
message(STATUS "No build type specified; defaulting to CMAKE_BUILD_TYPE=Release.")
set(CMAKE_BUILD_TYPE Release CACHE STRING
"Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel."
FORCE)
else(NOT CMAKE_BUILD_TYPE)
if(CMAKE_BUILD_TYPE STREQUAL "Debug")
message("==========================================================================================")
message(STATUS "Build type: Debug. Performance will be terrible!")
message(STATUS "Add -DCMAKE_BUILD_TYPE=Release to the CMake command line to get an optimized build.")
message("==========================================================================================")
endif(CMAKE_BUILD_TYPE STREQUAL "Debug")
endif(NOT CMAKE_BUILD_TYPE)
####
# Setup the compiler options
# set c++ standard to c++11.
# Note: not working on CMake 2.8. We assume that user has
# a compiler with C++11 support.
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
message(STATUS "C++11 support has been enabled by default.")
option(BUILD_DOCS "Set to ON to build documentation" OFF)
option(BUILD_MOCK "Build mock executable" ON)
if(BUILD_DOCS)
add_subdirectory(docs)
endif(BUILD_DOCS)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/bin)
if(BUILD_MOCK)
set(DLPACK_MOCK_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/mock_main.cc
${CMAKE_CURRENT_SOURCE_DIR}/contrib/mock_c.c)
add_executable(mock ${DLPACK_MOCK_SRCS})
endif()
add_library(dlpack INTERFACE)
add_library(${PROJECT_NAME}::dlpack ALIAS dlpack)
target_include_directories(
dlpack
INTERFACE
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/contrib>
)
if(BUILD_MOCK)
target_link_libraries(mock PRIVATE dlpack)
endif()
# Installation (https://github.com/forexample/package-example) {
# Introduce variables:
# * CMAKE_INSTALL_LIBDIR
# * CMAKE_INSTALL_BINDIR
# * CMAKE_INSTALL_INCLUDEDIR
include(GNUInstallDirs)
set(config_install_dir "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}")
set(generated_dir "${CMAKE_CURRENT_BINARY_DIR}/generated")
set(version_config "${generated_dir}/${PROJECT_NAME}ConfigVersion.cmake")
set(project_config "${generated_dir}/${PROJECT_NAME}Config.cmake")
set(TARGETS_EXPORT_NAME "${PROJECT_NAME}Targets")
set(namespace "${PROJECT_NAME}::")
include(CMakePackageConfigHelpers)
# Use:
# * PROJECT_VERSION
write_basic_package_version_file(
"${version_config}" COMPATIBILITY SameMajorVersion
)
# Use:
# * TARGETS_EXPORT_NAME
# * PROJECT_NAME
configure_package_config_file(
"cmake/template/Config.cmake.in"
"${project_config}"
INSTALL_DESTINATION "${config_install_dir}"
)
install(
TARGETS dlpack
EXPORT "${TARGETS_EXPORT_NAME}"
INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
)
install(
DIRECTORY include/
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
)
install(
FILES "${project_config}" "${version_config}"
DESTINATION "${config_install_dir}"
)
install(
EXPORT "${TARGETS_EXPORT_NAME}"
NAMESPACE "${namespace}"
DESTINATION "${config_install_dir}"
)
# }
.PHONY: clean all test doc lint
all: bin/mock
LDFLAGS =
CFLAGS = -Wall -O3 -Iinclude -Icontrib
CXXFLAGS = -std=c++11 $(CFLAGS)
SRC = $(wildcard contrib/*.cc contrib/*.c)
ALL_CXX_OBJ = $(patsubst contrib/%.cc, build/%.o, $(SRC))
ALL_C_OBJ = $(patsubst contrib/%.c, build/%.o, $(SRC))
ALL_OBJ = $(ALL_CC_OBJ) $(ALL_CXX_OBJ)
doc:
doxygen docs/Doxyfile
lint:
./tests/scripts/task_lint.sh
build/%.o: contrib/%.cc
@mkdir -p $(@D)
$(CXX) $(CXXFLAGS) -MM -MT build/$*.o $< >build/$*.d
$(CXX) -c $(CXXFLAGS) -c $< -o $@
build/%.o: contrib/%.c
@mkdir -p $(@D)
$(CC) $(CFLAGS) -MM -MT build/$*.o $< >build/$*.d
$(CC) -c $(CFLAGS) -c $< -o $@
bin/mock: $(ALL_OBJ)
@mkdir -p $(@D)
$(CXX) $(CXXFLAGS) -o $@ $(filter %.o %.a, $^) $(LDFLAGS)
clean:
$(RM) -rf build */*/*/*~ */*.o */*/*.o */*/*/*.o */*.d */*/*.d */*/*/*.d
DLPack Change Log
=================
This file records the changes in DLPack in reverse chronological order.
## v0.2
- New device types
- kDLMetal for Apple Metal device
- kDLVPI for verilog simulator memory
- kDLROCM for AMD GPUs
- Add prefix DL to all enum constant values
- This requires dependent frameworks to upgrade their reference to these constant
- The ABI is compatible, as it is only change of constant name.
- Add DLManagedTensor structure for borrowing tensors
## v0.1
- Finalize DLTensor structure
# DLPack: Open In Memory Tensor Structure
[![Build Status](https://travis-ci.org/dmlc/dlpack.svg?branch=master)](https://travis-ci.org/dmlc/dlpack)
DLPack is an open in-memory tensor structure to for sharing tensor among frameworks. DLPack enables
- Easier sharing of operators between deep learning frameworks.
- Easier wrapping of vendor level operator implementations, allowing collaboration when introducing new devices/ops.
- Quick swapping of backend implementations, like different version of BLAS
- For final users, this could bring more operators, and possibility of mixing usage between frameworks.
We do not intend to implement of Tensor and Ops, but instead use this as common bridge
to reuse tensor and ops across frameworks.
## Proposal Procedure
RFC proposals are opened as issues. The major release will happen as a vote issue to make
sure the participants agree on the changes.
## Project Structure
There are two major components so far
- include: stabilized headers
- contrib: in progress unstable libraries
## People
Here are list of people who have been involved in DLPack RFC design proposals:
@soumith @piiswrong @Yangqing @naibaf7 @bhack @edgarriba @tqchen @prigoyal @zdevito
all: numpy_dlpack.c
gcc -I../../include -shared -o libmain.so -fPIC numpy_dlpack.c
from __future__ import print_function
import numpy as np
import gc
import ctypes
libmain = ctypes.cdll.LoadLibrary("./libmain.so")
class DLContext(ctypes.Structure):
_fields_ = [("device_type", ctypes.c_int),
("device_id", ctypes.c_int)]
class DLDataType(ctypes.Structure):
_fields_ = [("type_code", ctypes.c_uint8),
("bits", ctypes.c_uint8),
("lanes", ctypes.c_uint16)]
TYPE_MAP = {
"bool": (1, 1, 1),
"int32": (0, 32, 1),
"int64": (0, 64, 1),
"uint32": (1, 32, 1),
"uint64": (1, 64, 1),
"float32": (2, 32, 1),
"float64": (2, 64, 1),
}
class DLTensor(ctypes.Structure):
_fields_ = [("data", ctypes.c_void_p),
("ctx", DLContext),
("ndim", ctypes.c_int),
("dtype", DLDataType),
("shape", ctypes.POINTER(ctypes.c_int64)),
("strides", ctypes.POINTER(ctypes.c_int64)),
("byte_offset", ctypes.c_uint64)]
class DLManagedTensor(ctypes.Structure):
pass
DLManagedTensorHandle = ctypes.POINTER(DLManagedTensor)
DeleterFunc = ctypes.CFUNCTYPE(None, DLManagedTensorHandle)
DLManagedTensor._fields_ = [("dl_tensor", DLTensor),
("manager_ctx", ctypes.c_void_p),
("deleter", DeleterFunc)]
def display(array):
print("data =", hex(array.ctypes.data_as(ctypes.c_void_p).value))
print("dtype =", array.dtype)
print("ndim =", array.ndim)
print("shape =", array.shape)
print("strides =", array.strides)
def make_manager_ctx(obj):
pyobj = ctypes.py_object(obj)
void_p = ctypes.c_void_p.from_buffer(pyobj)
ctypes.pythonapi.Py_IncRef(pyobj)
return void_p
# N.B.: In practice, one should ensure that this function
# is not destructed before the numpy array is destructed.
@DeleterFunc
def dl_managed_tensor_deleter(dl_managed_tensor_handle):
void_p = dl_managed_tensor_handle.contents.manager_ctx
pyobj = ctypes.cast(void_p, ctypes.py_object)
print("Deleting manager_ctx:")
display(pyobj.value)
ctypes.pythonapi.Py_DecRef(pyobj)
print("Deleter self...")
libmain.FreeHandle()
print("Done")
def make_dl_tensor(array):
# You may check array.flags here, e.g. array.flags['C_CONTIGUOUS']
dl_tensor = DLTensor()
dl_tensor.data = array.ctypes.data_as(ctypes.c_void_p)
dl_tensor.ctx = DLContext(1, 0)
dl_tensor.ndim = array.ndim
dl_tensor.dtype = DLDataType.TYPE_MAP[str(array.dtype)]
# For 0-dim ndarrays, strides and shape will be NULL
dl_tensor.shape = array.ctypes.shape_as(ctypes.c_int64)
dl_tensor.strides = array.ctypes.strides_as(ctypes.c_int64)
for i in range(array.ndim):
dl_tensor.strides[i] //= array.itemsize
dl_tensor.byte_offset = 0
return dl_tensor
def main():
array = np.random.rand(3, 1, 30).astype("float32")
print("Created:")
display(array)
c_obj = DLManagedTensor()
c_obj.dl_tensor = make_dl_tensor(array)
c_obj.manager_ctx = make_manager_ctx(array)
c_obj.deleter = dl_managed_tensor_deleter
print("-------------------------")
del array
gc.collect()
libmain.Give(c_obj)
print("-------------------------")
del c_obj
gc.collect()
libmain.Finalize()
print("-------------------------")
if __name__ == "__main__":
main()
#include <stdio.h>
#include <stdlib.h>
#include <dlpack/dlpack.h>
DLManagedTensor *given = NULL;
void display(DLManagedTensor a) {
puts("On C side:");
int i;
int ndim = a.dl_tensor.ndim;
printf("data = %p\n", a.dl_tensor.data);
printf("ctx = (device_type = %d, device_id = %d)\n",
(int) a.dl_tensor.ctx.device_type,
(int) a.dl_tensor.ctx.device_id);
printf("dtype = (code = %d, bits = %d, lanes = %d)\n",
(int) a.dl_tensor.dtype.code,
(int) a.dl_tensor.dtype.bits,
(int) a.dl_tensor.dtype.lanes);
printf("ndim = %d\n",
(int) a.dl_tensor.ndim);
printf("shape = (");
for (i = 0; i < ndim; ++i) {
if (i != 0) {
printf(", ");
}
printf("%d", (int) a.dl_tensor.shape[i]);
}
printf(")\n");
printf("strides = (");
for (i = 0; i < ndim; ++i) {
if (i != 0) {
printf(", ");
}
printf("%d", (int) a.dl_tensor.strides[i]);
}
printf(")\n");
}
void Give(DLManagedTensor dl_managed_tensor) {
display(dl_managed_tensor);
given = (DLManagedTensor *) malloc(sizeof(DLManagedTensor));
*given = dl_managed_tensor;
}
void Finalize() {
given->deleter(given);
}
void FreeHandle() {
free(given);
given = NULL;
}
@PACKAGE_INIT@
include("${CMAKE_CURRENT_LIST_DIR}/@TARGETS_EXPORT_NAME@.cmake")
check_required_components("@PROJECT_NAME@")
/*!
* Copyright (c) 2017 by Contributors
* \file dlpackcpp.h
* \brief Example C++ wrapper of DLPack
*/
#ifndef DLPACK_DLPACKCPP_H_
#define DLPACK_DLPACKCPP_H_
#include <dlpack/dlpack.h>
#include <cstdint> // for int64_t etc
#include <cstdlib> // for free()
#include <functional> // for std::multiplies
#include <memory>
#include <numeric>
#include <vector>
namespace dlpack {
// Example container wrapping of DLTensor.
class DLTContainer {
public:
DLTContainer() {
// default to float32
handle_.data = nullptr;
handle_.dtype.code = kDLFloat;
handle_.dtype.bits = 32U;
handle_.dtype.lanes = 1U;
handle_.ctx.device_type = kDLCPU;
handle_.ctx.device_id = 0;
handle_.shape = nullptr;
handle_.strides = nullptr;
handle_.byte_offset = 0;
}
~DLTContainer() {
if (origin_ == nullptr) {
free(handle_.data);
}
}
operator DLTensor() {
return handle_;
}
operator DLTensor*() {
return &(handle_);
}
void Reshape(const std::vector<int64_t>& shape) {
shape_ = shape;
int64_t sz = std::accumulate(std::begin(shape), std::end(shape),
int64_t(1), std::multiplies<int64_t>());
int ret = posix_memalign(&handle_.data, 256, sz);
if (ret != 0) throw std::bad_alloc();
handle_.shape = &shape_[0];
handle_.ndim = static_cast<uint32_t>(shape.size());
}
private:
DLTensor handle_;
std::vector<int64_t> shape_;
std::vector<int64_t> strides_;
// original space container, if
std::shared_ptr<DLTContainer> origin_;
};
} // namespace dlpack
#endif // DLPACK_DLPACKCPP_H_
// Copyright by contributors
// This file is used to make sure the package is C compatible
#include <dlpack/dlpack.h>
int GetNDim(DLTensor *t) {
return t->ndim;
}
// Copyright by contributors
#include <dlpack/dlpack.h>
#include <dlpack/dlpackcpp.h>
int main() {
dlpack::DLTContainer c;
return 0;
}
find_package(Doxygen QUIET)
if(NOT DOXYGEN_FOUND)
message(FATAL_ERROR "Doxygen is needed to build the documentation.")
endif()
# TODO: add config file
set(doxyfile_in ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in)
set(doxyfile ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile)
configure_file(${doxyfile_in} ${doxyfile} @ONLY)
add_custom_target(docs
COMMAND ${DOXYGEN_EXECUTABLE} ${doxyfile}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
COMMENT "Generating API documentation with Doxygen"
VERBATIM)
install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/html DESTINATION share/${PROJECT_NAME}/docs)
\ No newline at end of file
This source diff could not be displayed because it is too large. You can view the blob instead.
PROJECT_NAME = @PROJECT_NAME@
PROJECT_NUMBER = @PROJECT_VERSION@
PROJECT_BRIEF = "RFC for common tensor and operator guideline in deep learning system"
STRIP_FROM_PATH = @PROJECT_SOURCE_DIR@ \
@PROJECT_BINARY_DIR@
OUTPUT_LANGUAGE = English
FILE_PATTERNS = *.h *.md
RECURSIVE = YES
IMAGE_PATH = @CMAKE_SOURCE_DIR@/docs
USE_MDFILE_AS_MAINPAGE = @CMAKE_SOURCE_DIR@/docs/readme.md
JAVADOC_AUTOBRIEF = YES
GENERATE_HTML = YES
GENERATE_LATEX = NO
OUTPUT_DIRECTORY = @CMAKE_BINARY_DIR@/docs/doxygen
\ No newline at end of file
/*!
* Copyright (c) 2017 by Contributors
* \file dlpack.h
* \brief The common header of DLPack.
*/
#ifndef DLPACK_DLPACK_H_
#define DLPACK_DLPACK_H_
#ifdef __cplusplus
#define DLPACK_EXTERN_C extern "C"
#else
#define DLPACK_EXTERN_C
#endif
/*! \brief The current version of dlpack */
#define DLPACK_VERSION 020
/*! \brief DLPACK_DLL prefix for windows */
#ifdef _WIN32
#ifdef DLPACK_EXPORTS
#define DLPACK_DLL __declspec(dllexport)
#else
#define DLPACK_DLL __declspec(dllimport)
#endif
#else
#define DLPACK_DLL
#endif
#include <stdint.h>
#include <stddef.h>
#ifdef __cplusplus
extern "C" {
#endif
/*!
* \brief The device type in DLContext.
*/
typedef enum {
/*! \brief CPU device */
kDLCPU = 1,
/*! \brief CUDA GPU device */
kDLGPU = 2,
/*!
* \brief Pinned CUDA GPU device by cudaMallocHost
* \note kDLCPUPinned = kDLCPU | kDLGPU
*/
kDLCPUPinned = 3,
/*! \brief OpenCL devices. */
kDLOpenCL = 4,
/*! \brief Vulkan buffer for next generation graphics. */
kDLVulkan = 7,
/*! \brief Metal for Apple GPU. */
kDLMetal = 8,
/*! \brief Verilog simulator buffer */
kDLVPI = 9,
/*! \brief ROCm GPUs for AMD GPUs */
kDLROCM = 10,
/*!
* \brief Reserved extension device type,
* used for quickly test extension device
* The semantics can differ depending on the implementation.
*/
kDLExtDev = 12,
} DLDeviceType;
/*!
* \brief A Device context for Tensor and operator.
*/
typedef struct {
/*! \brief The device type used in the device. */
DLDeviceType device_type;
/*! \brief The device index */
int device_id;
} DLContext;
/*!
* \brief The type code options DLDataType.
*/
typedef enum {
kDLInt = 0U,
kDLUInt = 1U,
kDLFloat = 2U,
kDLBfloat = 4U,
} DLDataTypeCode;
/*!
* \brief The data type the tensor can hold.
*
* Examples
* - float: type_code = 2, bits = 32, lanes=1
* - float4(vectorized 4 float): type_code = 2, bits = 32, lanes=4
* - int8: type_code = 0, bits = 8, lanes=1
*/
typedef struct {
/*!
* \brief Type code of base types.
* We keep it uint8_t instead of DLDataTypeCode for minimal memory
* footprint, but the value should be one of DLDataTypeCode enum values.
* */
uint8_t code;
/*!
* \brief Number of bits, common choices are 8, 16, 32.
*/
uint8_t bits;
/*! \brief Number of lanes in the type, used for vector types. */
uint16_t lanes;
} DLDataType;
/*!
* \brief Plain C Tensor object, does not manage memory.
*/
typedef struct {
/*!
* \brief The opaque data pointer points to the allocated data. This will be
* CUDA device pointer or cl_mem handle in OpenCL. This pointer is always
* aligned to 256 bytes as in CUDA.
*
* For given DLTensor, the size of memory required to store the contents of
* data is calculated as follows:
*
* \code{.c}
* static inline size_t GetDataSize(const DLTensor* t) {
* size_t size = 1;
* for (tvm_index_t i = 0; i < t->ndim; ++i) {
* size *= t->shape[i];
* }
* size *= (t->dtype.bits * t->dtype.lanes + 7) / 8;
* return size;
* }
* \endcode
*/
void* data;
/*! \brief The device context of the tensor */
DLContext ctx;
/*! \brief Number of dimensions */
int ndim;
/*! \brief The data type of the pointer*/
DLDataType dtype;
/*! \brief The shape of the tensor */
int64_t* shape;
/*!
* \brief strides of the tensor (in number of elements, not bytes)
* can be NULL, indicating tensor is compact and row-majored.
*/
int64_t* strides;
/*! \brief The offset in bytes to the beginning pointer to data */
uint64_t byte_offset;
} DLTensor;
/*!
* \brief C Tensor object, manage memory of DLTensor. This data structure is
* intended to facilitate the borrowing of DLTensor by another framework. It is
* not meant to transfer the tensor. When the borrowing framework doesn't need
* the tensor, it should call the deleter to notify the host that the resource
* is no longer needed.
*/
typedef struct DLManagedTensor {
/*! \brief DLTensor which is being memory managed */
DLTensor dl_tensor;
/*! \brief the context of the original host framework of DLManagedTensor in
* which DLManagedTensor is used in the framework. It can also be NULL.
*/
void * manager_ctx;
/*! \brief Destructor signature void (*)(void*) - this should be called
* to destruct manager_ctx which holds the DLManagedTensor. It can be NULL
* if there is no way for the caller to provide a reasonable destructor.
* The destructors deletes the argument self as well.
*/
void (*deleter)(struct DLManagedTensor * self);
} DLManagedTensor;
#ifdef __cplusplus
} // DLPACK_EXTERN_C
#endif
#endif // DLPACK_DLPACK_H_
#!/bin/bash
make || exit -1
mkdir -p build
cd build
cmake .. || exit -1
make || exit -1
#!/bin/bash
mkdir -p bin
if [ ! -f bin/lint.py ]; then
echo "Grab linter ..."
wget https://raw.githubusercontent.com/dmlc/dmlc-core/master/scripts/lint.py
mv lint.py bin/lint.py
fi
echo "Check codestyle of c++ code..."
python bin/lint.py dlpack cpp include contrib
echo "Check doxygen generation..."
make doc 2>log.txt
(cat log.txt| grep -v ENABLE_PREPROCESSING |grep -v "unsupported tag") > logclean.txt
echo "---------Error Log----------"
cat logclean.txt
echo "----------------------------"
(cat logclean.txt|grep warning) && exit -1
(cat logclean.txt|grep error) && exit -1
rm logclean.txt
rm log.txt
echo "All checks passed..."
#!/bin/bash
if [ ${TASK} == "lint" ] || [ ${TASK} == "all_test" ]; then
if [ ! ${TRAVIS_OS_NAME} == "osx" ]; then
./tests/scripts/task_lint.sh || exit -1
fi
fi
if [ ${TASK} == "build" ] || [ ${TASK} == "all_test" ]; then
./tests/scripts/task_build.sh || exit -1
fi
echo "All travis test passed.."
if [ ${TASK} == "lint" ] || [ ${TASK} == "all_test" ]; then
if [ ! ${TRAVIS_OS_NAME} == "osx" ]; then
pip install --user cpplint
fi
fi
\ No newline at end of file
root = true
[*]
charset=utf-8
indent_style = space
indent_size = 2
insert_final_newline = true
\ No newline at end of file
name: continuous build
on: [push, pull_request]
jobs:
tests:
runs-on: ${{ matrix.os }}
strategy:
matrix:
task: ["unittest_gtest", "cmake_test", "sanitizer_test", "s390x_test"]
os: [macos-10.15, ubuntu-18.04]
exclude:
# excludes sanitizer_test and s390x_test on macOS
- os: macos-10.15
task: "sanitizer_test"
- os: macos-10.15
task: "s390x_test"
steps:
- name: Checkout repository
uses: actions/checkout@v2
- name: Install Dependencies
run: |
if [[ $(uname) != "Darwin" ]]; then
sudo apt install doxygen libcurl4-openssl-dev gcc-4.8 g++-4.8
else
brew install libomp doxygen
fi
python3 -m pip install --user --upgrade pip setuptools
python3 -m pip install --user flake8 pylint cpplint
- name: Lint
run: |
TASK=lint ./scripts/test_script.sh
- name: googletest
run: |
if [[ $(uname) != "Darwin" ]]; then
docker run --rm --privileged multiarch/qemu-user-static:register --reset
fi
TASK=${{ matrix.task }} ./scripts/test_script.sh
# Compiled Object files
*.slo
*.lo
*.o
*.obj
# Precompiled Headers
*.gch
*.pch
# Compiled Dynamic libraries
*.so
*.dylib
*.dll
# Fortran module files
*.mod
# Compiled Static libraries
*.lai
*.la
*.a
*.lib
# Executables
*.exe
*.out
*.app
*~
config.mk
*.pyc
# Vim
*.swp
*.swo
*.swn
*.csv
.vimrc
# Emacs
.clang_complete
deps
recommonmark
build
# CLion
.idea
cmake-build-*
cmake_minimum_required(VERSION 3.2)
project(dmlc VERSION 0.3 LANGUAGES C CXX)
if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/build/private/local_config.cmake)
include(${CMAKE_CURRENT_SOURCE_DIR}/build/private/local_config.cmake)
endif()
set(CMAKE_LOCAL "${PROJECT_SOURCE_DIR}/cmake")
list(APPEND CMAKE_MODULE_PATH ${CMAKE_LOCAL}/Modules)
include(CheckCXXSymbolExists)
include(cmake/Utils.cmake)
# Options
dmlccore_option(USE_HDFS "Build with HDFS support" OFF)
dmlccore_option(USE_AZURE "Build with AZURE support" OFF)
dmlccore_option(USE_S3 "Build with S3 support" OFF)
dmlccore_option(USE_OPENMP "Build with OpenMP" ON)
dmlccore_option(USE_CXX14_IF_AVAILABLE "Build with C++14 if the compiler supports it" OFF)
dmlccore_option(GOOGLE_TEST "Build google tests" OFF)
dmlccore_option(INSTALL_DOCUMENTATION "Install documentation" OFF)
dmlccore_option(DMLC_FORCE_SHARED_CRT "Build with dynamic CRT on Windows (/MD)" OFF)
dmlccore_option(DMLC_USE_SANITIZER "Use santizer flags; to specify a custom path for sanitizers, set this variable a value that's not ON or OFF" OFF)
set(DMLC_ENABLED_SANITIZERS "address" "leak" CACHE STRING
"Semicolon separated list of sanitizer names. E.g 'address;leak'. Supported sanitizers are
address, leak and thread.")
include(CheckCXXCompilerFlag)
if(USE_CXX14_IF_AVAILABLE)
check_cxx_compiler_flag("-std=c++14" SUPPORT_CXX14)
endif()
if(SUPPORT_CXX14)
set(CMAKE_CXX_STANDARD 14)
else()
set(CMAKE_CXX_STANDARD 11)
endif()
set(CMAKE_CXX_EXTENSIONS OFF)
FILE(GLOB SOURCE "src/*.cc")
FILE(GLOB_RECURSE SOURCE_INCLUDE "include/*")
list(APPEND SOURCE ${SOURCE_INCLUDE})
list(APPEND SOURCE "src/io/line_split.cc")
list(APPEND SOURCE "src/io/recordio_split.cc")
list(APPEND SOURCE "src/io/indexed_recordio_split.cc")
list(APPEND SOURCE "src/io/input_split_base.cc")
list(APPEND SOURCE "src/io/filesys.cc")
list(APPEND SOURCE "src/io/local_filesys.cc")
if(USE_HDFS)
list(APPEND SOURCE "src/io/hdfs_filesys.cc")
endif()
if(USE_S3)
list(APPEND SOURCE "src/io/s3_filesys.cc")
endif()
if(USE_AZURE)
list(APPEND SOURCE "src/io/azure_filesys.cc")
endif()
add_library(dmlc ${SOURCE})
# Sanitizer
if (DMLC_USE_SANITIZER)
# Older CMake versions have had troubles with Sanitizer
cmake_minimum_required(VERSION 3.12)
include(cmake/Sanitizer.cmake)
enable_sanitizers("${DMLC_ENABLED_SANITIZERS}")
endif (DMLC_USE_SANITIZER)
# HDFS configurations
if(USE_HDFS)
find_package(HDFS REQUIRED)
find_package(JNI REQUIRED)
target_include_directories(dmlc PRIVATE ${HDFS_INCLUDE_DIR})
target_link_libraries(dmlc PRIVATE ${HDFS_STATIC_LIB} ${JAVA_JVM_LIBRARY})
target_compile_definitions(dmlc PRIVATE -DDMLC_USE_HDFS=1)
else()
target_compile_definitions(dmlc PRIVATE -DDMLC_USE_HDFS=0)
endif()
# S3 configurations
if(USE_S3)
find_package(CURL REQUIRED)
target_include_directories(dmlc SYSTEM PRIVATE ${CURL_INCLUDE_DIR})
target_link_libraries(dmlc PRIVATE ${CURL_LIBRARY})
find_package(OpenSSL REQUIRED)
target_include_directories(dmlc SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR})
target_link_libraries(dmlc PRIVATE ${OPENSSL_LIBRARY} ${OPENSSL_LIBRARIES} ${OPENSSL_CRYPTO_LIBRARY})
target_compile_definitions(dmlc PRIVATE -DDMLC_USE_S3=1)
else()
target_compile_definitions(dmlc PRIVATE -DDMLC_USE_S3=0)
endif()
# Azure configurations
if(USE_AZURE)
target_compile_definitions(dmlc PRIVATE -DDMLC_USE_AZURE=1)
else()
target_compile_definitions(dmlc PRIVATE -DDMLC_USE_AZURE=0)
endif()
# OpenMP
if(USE_OPENMP)
if(APPLE AND (NOT CMAKE_COMPILER_IS_GNUCC))
# Require CMake 3.16+ for Mac to ensure that OpenMP can be located
# (Exception: it's okay if Homebrew GCC is used)
cmake_minimum_required(VERSION 3.16)
endif()
find_package(OpenMP REQUIRED)
# For CMake < 3.9, we need to make target OpenMP::OpenMP_CXX ourselves
if(NOT TARGET OpenMP::OpenMP_CXX)
find_package(Threads REQUIRED)
add_library(OpenMP::OpenMP_CXX IMPORTED INTERFACE)
set_property(TARGET OpenMP::OpenMP_CXX
PROPERTY INTERFACE_COMPILE_OPTIONS ${OpenMP_CXX_FLAGS})
set_property(TARGET OpenMP::OpenMP_CXX
PROPERTY INTERFACE_LINK_LIBRARIES ${OpenMP_CXX_FLAGS} Threads::Threads)
endif()
target_link_libraries(dmlc PRIVATE OpenMP::OpenMP_CXX)
endif()
if(WIN32 AND (NOT MSVC)) # On Windows, link Shlwapi.lib for non-MSVC compilers
target_link_libraries(dmlc PRIVATE Shlwapi)
endif()
# Check location of clock_gettime; if it's in librt, link it
include(CheckLibraryExists)
CHECK_LIBRARY_EXISTS(rt clock_gettime "time.h" HAVE_CLOCK_GETTIME_IN_LIBRT)
if(HAVE_CLOCK_GETTIME_IN_LIBRT)
target_link_libraries(dmlc PRIVATE rt)
endif()
# Check headers and symbols
include(CheckSymbolExists)
include(CheckIncludeFile)
include(CheckIncludeFileCXX)
check_symbol_exists(fopen64 stdio.h DMLC_FOPEN_64_PRESENT)
check_include_file_cxx(cxxabi.h DMLC_CXXABI_H_PRESENT)
check_symbol_exists(nanosleep time.h DMLC_NANOSLEEP_PRESENT)
# Check existence of backtrace(3)
find_package(Backtrace)
if(Backtrace_FOUND)
set(DMLC_EXECINFO_H_PRESENT 1)
set(DMLC_EXECINFO_H ${Backtrace_HEADER})
target_include_directories(dmlc SYSTEM PRIVATE ${Backtrace_INCLUDE_DIRS})
target_link_libraries(dmlc PRIVATE ${Backtrace_LIBRARIES})
else()
set(DMLC_EXECINFO_H_PRESENT 0)
endif()
# Check endianness
include(TestBigEndian)
test_big_endian(BIG_ENDIAN)
if(BIG_ENDIAN)
set(DMLC_CMAKE_LITTLE_ENDIAN 0)
else()
set(DMLC_CMAKE_LITTLE_ENDIAN 1)
endif()
message(STATUS "${CMAKE_LOCAL}/build_config.h.in -> include/dmlc/build_config.h")
configure_file("cmake/build_config.h.in" "include/dmlc/build_config.h")
target_include_directories(dmlc PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/include>
$<INSTALL_INTERFACE:${CMAKE_INSTALL_PREFIX}/include>)
target_compile_definitions(dmlc PRIVATE -D_XOPEN_SOURCE=700
-D_POSIX_SOURCE -D_POSIX_C_SOURCE=200809L -D_DARWIN_C_SOURCE)
# Older stdc++ enable c++11 items
target_compile_definitions(dmlc PUBLIC -D__USE_XOPEN2K8)
# DMLC_CORE_USE_CMAKE macro constant indicates the use of CMake
target_compile_definitions(dmlc PUBLIC -DDMLC_CORE_USE_CMAKE)
# compiler flags
if(MSVC)
target_compile_definitions(dmlc PUBLIC -DDMLC_USE_CXX11=1)
if(NOT BUILD_SHARED_LIBS AND NOT DMLC_FORCE_SHARED_CRT)
foreach(flag_var
CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
if(${flag_var} MATCHES "/MD")
string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
endif(${flag_var} MATCHES "/MD")
endforeach(flag_var)
endif()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc")
else()
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
check_cxx_compiler_flag("-msse2" SUPPORT_MSSE2)
if(SUPPORT_MSSE2)
target_compile_options(dmlc PRIVATE -msse2)
endif()
target_compile_options(dmlc PRIVATE -Wall -Wno-unknown-pragmas -fPIC)
if(CMAKE_BUILD_TYPE STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE STREQUAL "Debug")
target_compile_options(dmlc PRIVATE -g -O0)
else()
target_compile_options(dmlc PRIVATE -O3)
endif()
target_compile_definitions(dmlc PUBLIC -DDMLC_USE_CXX11=1)
if(SUPPORT_CXX14)
target_compile_definitions(dmlc PUBLIC -DDMLC_USE_CXX14=1)
endif()
endif()
include(GNUInstallDirs)
# ---[ Install Includes
install(DIRECTORY ${PROJECT_SOURCE_DIR}/include/dmlc
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/include/dmlc/build_config.h
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/dmlc)
# ---[ Install the archive static lib and header files
install(TARGETS dmlc
EXPORT DMLCTargets
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
install(EXPORT DMLCTargets
FILE DMLCTargets.cmake
NAMESPACE dmlc::
EXPORT_LINK_INTERFACE_LIBRARIES
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/dmlc)
# ---[ Install documentation
if(INSTALL_DOCUMENTATION)
install(DIRECTORY doc DESTINATION ${CMAKE_INSTALL_DATADIR})
endif()
# ---[ Package configurations
include(CMakePackageConfigHelpers)
configure_package_config_file(
${CMAKE_LOCAL}/dmlc-config.cmake.in
${CMAKE_BINARY_DIR}/cmake/dmlc-config.cmake
INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/dmlc)
write_basic_package_version_file(
${CMAKE_BINARY_DIR}/cmake/dmlc-config-version.cmake
VERSION ${DMLC_VERSION}
COMPATIBILITY AnyNewerVersion)
install(
FILES
${CMAKE_BINARY_DIR}/cmake/dmlc-config.cmake
${CMAKE_BINARY_DIR}/cmake/dmlc-config-version.cmake
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/dmlc)
# ---[ Linter target
if(MSVC)
find_package(PythonInterp)
set(PYTHON_EXECUTABLE ${PYTHON_EXECUTABLE} CACHE FILEPATH "Path to the python 2.x executable")
endif()
set(LINT_DIRS include src scripts)
add_custom_target(dmlc_lint COMMAND ${CMAKE_COMMAND} -DMSVC=${MSVC} -DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE} -DPROJECT_SOURCE_DIR=${PROJECT_SOURCE_DIR} -DLINT_DIRS=${LINT_DIRS} -DPROJECT_NAME=dmlc -P ${PROJECT_SOURCE_DIR}/cmake/lint.cmake)
# Setup testing
if(GOOGLE_TEST)
include(CTest)
add_subdirectory(test/unittest)
endif()
ifndef config
ifneq ("$(wildcard ./config.mk)","")
config = config.mk
else
config = make/config.mk
endif
endif
# use customized config file
include $(config)
include make/dmlc.mk
NOLINT_FILES = --exclude_path include/dmlc/concurrentqueue.h include/dmlc/blockingconcurrentqueue.h
# this is the common build script for dmlc lib
export LDFLAGS= -pthread -lm
export CFLAGS = -O3 -Wall -Wno-unknown-pragmas -Iinclude
ifeq ($(USE_GNU11), 1)
CFLAGS += -std=gnu++11
else
CFLAGS += -std=c++0x
endif
LDFLAGS+= $(DMLC_LDFLAGS) $(ADD_LDFLAGS)
CFLAGS+= $(DMLC_CFLAGS) $(ADD_CFLAGS)
ifndef USE_SSE
USE_SSE = 1
endif
ifeq ($(USE_SSE), 1)
CFLAGS += -msse2
endif
ifdef DEPS_PATH
CFLAGS+= -I$(DEPS_PATH)/include
LDFLAGS+= -L$(DEPS_PATH)/lib
endif
.PHONY: clean all test lint doc example pylint
OBJ=line_split.o indexed_recordio_split.o recordio_split.o input_split_base.o io.o filesys.o local_filesys.o data.o recordio.o config.o
ifeq ($(USE_HDFS), 1)
OBJ += hdfs_filesys.o
endif
ifeq ($(USE_S3), 1)
OBJ += s3_filesys.o
endif
ifeq ($(USE_AZURE), 1)
OBJ += azure_filesys.o
endif
ifndef LINT_LANG
LINT_LANG="all"
endif
ALIB=libdmlc.a
all: $(ALIB) test
include test/dmlc_test.mk
include example/dmlc_example.mk
ifeq ($(BUILD_TEST), 1)
test: $(ALL_TEST)
endif
example: $(ALL_EXAMPLE)
line_split.o: src/io/line_split.cc
recordio_split.o: src/io/recordio_split.cc
indexed_recordio_split.o: src/io/indexed_recordio_split.cc
input_split_base.o: src/io/input_split_base.cc
filesys.o: src/io/filesys.cc
hdfs_filesys.o: src/io/hdfs_filesys.cc
s3_filesys.o: src/io/s3_filesys.cc
azure_filesys.o: src/io/azure_filesys.cc
local_filesys.o: src/io/local_filesys.cc
io.o: src/io.cc
data.o: src/data.cc
recordio.o: src/recordio.cc
config.o: src/config.cc
libdmlc.a: $(OBJ)
$(BIN) :
$(CXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c %.cc %.a, $^) $(LDFLAGS)
$(OBJ) :
$(CXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c %.cc, $^) )
$(ALIB):
$(AR) cr $@ $+
lint:
scripts/lint.py dmlc ${LINT_LANG} include src scripts $(NOLINT_FILES)
pylint:
scripts/lint.py dmlc ${LINT_LANG} tracker/dmlc_tracker
doxygen:
doxygen doc/Doxyfile
clean:
$(RM) $(OBJ) $(BIN) $(ALIB) $(ALL_TEST) $(ALL_TEST_OBJ) *~ src/*~ src/*/*~ include/dmlc/*~ test/*~
Distributed Machine Learning Common Codebase
============================================
[![Build Status](https://github.com/dmlc/dmlc-core/workflows/continuous%20build/badge.svg)](https://github.com/dmlc/dmlc-core/actions)
[![Documentation Status](https://readthedocs.org/projects/dmlc-core/badge/?version=latest)](http://dmlc-core.readthedocs.org/en/latest/)
[![GitHub license](http://dmlc.github.io/img/apache2.svg)](./LICENSE)
DMLC-Core is the backbone library to support all DMLC projects, offers the bricks to build efficient and scalable distributed machine learning libraries.
Developer Channel [![Join the chat at https://gitter.im/dmlc/dmlc-core](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/dmlc/dmlc-core?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
What's New
----------
* [Note on Parameter Module for Machine Learning](http://dmlc-core.readthedocs.org/en/latest/parameter.html)
Contents
--------
* [Documentation and Tutorials](http://dmlc-core.readthedocs.org/en/latest/)
* [Contributing](#contributing)
Known Issues
------------
* RecordIO format is not portable across different processor endians. So it is not possible to save RecordIO file on a x86 machine and then load it on a SPARC machine, because x86 is little endian while SPARC is big endian.
Contributing
------------
Contributing to dmlc-core is welcomed! dmlc-core follows google's C style guide. If you are interested in contributing, take a look at [feature wishlist](https://github.com/dmlc/dmlc-core/labels/feature%20wishlist) and open a new issue if you like to add something.
* Use of c++11 is allowed, given that the code is macro guarded with ```DMLC_USE_CXX11```
* Try to introduce minimum dependency when possible
### CheckList before submit code
* Type ```make lint``` and fix all the style problems.
* Type ```make doc``` and fix all the warnings.
NOTE
----
deps:
libcurl4-openssl-dev
environment:
matrix:
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
target: msvc
ver: 2019
generator: "Visual Studio 16 2019"
configuration: Debug
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
target: msvc
ver: 2019
generator: "Visual Studio 16 2019"
configuration: Release
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
target: mingw32
generator: "MinGW Makefiles"
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
target: cygwin
generator: "Unix Makefiles"
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
target: mingw
generator: "MinGW Makefiles"
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
target: msys2
generator: "Unix Makefiles"
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
target: msvc
ver: 2013
generator: "Visual Studio 12 2013 Win64"
configuration: Release
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
target: msvc
ver: 2015
generator: "Visual Studio 14 2015 Win64"
configuration: Debug
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
target: msvc
ver: 2015
generator: "Visual Studio 14 2015 Win64"
configuration: Release
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
target: msvc
ver: 2017
generator: "Visual Studio 15 2017 Win64"
configuration: Debug
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
target: msvc
ver: 2017
generator: "Visual Studio 15 2017 Win64"
configuration: Release
matrix:
fast_finish: true
platform:
- x64
install:
- git submodule update --init --recursive
# Set PATH
- if /i "%target%" == "msys2" set PATH=C:\msys64\mingw64\bin;C:\msys64\usr\bin;%PATH%
- if /i "%target%" == "mingw32" set PATH=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin;%PATH:C:\Program Files\Git\usr\bin;=%
- if /i "%target%" == "mingw" set PATH=C:\MinGW-w64\x86_64-7.3.0-posix-seh-rt_v5-rev0\mingw64\bin;%PATH:C:\Program Files\Git\usr\bin;=%
- if /i "%target%" == "cygwin" set PATH=C:\cygwin64\bin;C:\cygwin64\usr\bin;%PATH%
# Install packages and show information
- if /i "%target%" == "msys2" (
gcc -v
)
- if /i "%target%" == "mingw" (
gcc -v
)
- if /i "%target%" == "cygwin" (
gcc -v
)
build_script:
- cd %APPVEYOR_BUILD_FOLDER%
- if /i "%target%" == "msvc" (
mkdir build_msvc%ver% &&
cd build_msvc%ver% &&
if /i "%generator%" == "Visual Studio 12 2013 Win64" (
cmake .. -G"%generator%" -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_CONFIGURATION_TYPES="Release;Debug;" &&
msbuild dmlc.sln
) else if /i "%generator%" == "Visual Studio 16 2019" (
cmake .. -G"%generator%" -A x64 -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_CONFIGURATION_TYPES="Release;Debug;" -DGOOGLE_TEST=ON &&
msbuild dmlc.sln
) else (
cmake .. -G"%generator%" -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_CONFIGURATION_TYPES="Release;Debug;" -DGOOGLE_TEST=ON &&
msbuild dmlc.sln
)
)
- if /i "%target%" == "msys2" (
mkdir build_msys2 &&
cd build_msys2 &&
cmake .. -G"%generator%" -DCMAKE_VERBOSE_MAKEFILE=ON -DGOOGLE_TEST=ON &&
cmake --build . -- -j2
)
- if /i "%target%" == "mingw32" (
mkdir build_mingw32 &&
cd build_mingw32 &&
cmake .. -G"%generator%" -DCMAKE_VERBOSE_MAKEFILE=ON -DGOOGLE_TEST=ON &&
cmake --build . -- -j2
)
- if /i "%target%" == "mingw" (
mkdir build_mingw &&
cd build_mingw &&
cmake .. -G"%generator%" -DCMAKE_VERBOSE_MAKEFILE=ON -DGOOGLE_TEST=ON &&
cmake --build . -- -j2
)
- if /i "%target%" == "cygwin" (
mkdir build_cygwin &&
cd build_cygwin &&
cmake .. -G"%generator%" -DCMAKE_VERBOSE_MAKEFILE=ON -DGOOGLE_TEST=ON -DOpenMP_gomp_LIBRARY:FILEPATH=-lgomp &&
cmake --build . -- -j2
)
test_script:
- cd %APPVEYOR_BUILD_FOLDER%
- if /i "%target%" == "msvc" (
if /i not "%generator%" == "Visual Studio 12 2013 Win64" (
.\build_msvc%ver%\test\unittest\%configuration%\dmlc_unit_tests.exe
)
)
- if /i "%target%" == "msys2" (
.\build_msys2\test\unittest\dmlc_unit_tests.exe
)
- if /i "%target%" == "mingw32" (
.\build_mingw32\test\unittest\dmlc_unit_tests.exe
)
- if /i "%target%" == "mingw" (
.\build_mingw\test\unittest\dmlc_unit_tests.exe
)
- if /i "%target%" == "cygwin" (
.\build_cygwin\test\unittest\dmlc_unit_tests.exe
)
set(ASan_LIB_NAME ASan)
find_library(ASan_LIBRARY
NAMES libasan.so libasan.so.4 libasan.so.3 libasan.so.2 libasan.so.1 libasan.so.0
PATHS ${DMLC_USE_SANITIZER} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(ASan DEFAULT_MSG
ASan_LIBRARY)
mark_as_advanced(
ASan_LIBRARY
ASan_LIB_NAME)
# DerivedFrom: https://github.com/cloudera/Impala/blob/cdh5-trunk/cmake_modules/FindHDFS.cmake
# - Find HDFS (hdfs.h and libhdfs.so)
# This module defines
# Hadoop_VERSION, version string of ant if found
# HDFS_INCLUDE_DIR, directory containing hdfs.h
# HDFS_LIBRARIES, location of libhdfs.so
# HDFS_FOUND, whether HDFS is found.
# hdfs_static, imported static hdfs library.
exec_program(hadoop ARGS version OUTPUT_VARIABLE Hadoop_VERSION
RETURN_VALUE Hadoop_RETURN)
# currently only looking in HADOOP_HOME
find_path(HDFS_INCLUDE_DIR hdfs.h PATHS
$ENV{HADOOP_HDFS_HOME}/include/
# make sure we don't accidentally pick up a different version
NO_DEFAULT_PATH
)
if ("${CMAKE_SIZEOF_VOID_P}" STREQUAL "8")
set(arch_hint "x64")
elseif ("$ENV{LIB}" MATCHES "(amd64|ia64)")
set(arch_hint "x64")
else ()
set(arch_hint "x86")
endif()
message(STATUS "Architecture: ${arch_hint}")
if ("${arch_hint}" STREQUAL "x64")
set(HDFS_LIB_PATHS $ENV{HADOOP_HDFS_HOME}/lib/native)
else ()
set(HDFS_LIB_PATHS $ENV{HADOOP_HDFS_HOME}/lib/native)
endif ()
message(STATUS "HDFS_LIB_PATHS: ${HDFS_LIB_PATHS}")
find_library(HDFS_LIB NAMES hdfs PATHS
${HDFS_LIB_PATHS}
# make sure we don't accidentally pick up a different version
NO_DEFAULT_PATH
)
if (HDFS_LIB)
set(HDFS_FOUND TRUE)
set(HDFS_LIBRARIES ${HDFS_LIB})
set(HDFS_STATIC_LIB ${HDFS_LIB_PATHS}/${CMAKE_STATIC_LIBRARY_PREFIX}hdfs${CMAKE_STATIC_LIBRARY_SUFFIX})
add_library(hdfs_static STATIC IMPORTED)
set_target_properties(hdfs_static PROPERTIES IMPORTED_LOCATION ${HDFS_STATIC_LIB})
else ()
set(HDFS_FOUND FALSE)
endif ()
if (HDFS_FOUND)
if (NOT HDFS_FIND_QUIETLY)
message(STATUS "${Hadoop_VERSION}")
message(STATUS "HDFS_INCLUDE_DIR: ${HDFS_INCLUDE_DIR}")
message(STATUS "HDFS_LIBRARIES: ${HDFS_LIBRARIES}")
message(STATUS "hdfs_static: ${HDFS_STATIC_LIB}")
endif ()
else ()
message(FATAL_ERROR "HDFS includes and libraries NOT found."
"(${HDFS_INCLUDE_DIR}, ${HDFS_LIB})")
endif ()
mark_as_advanced(
HDFS_LIBRARIES
HDFS_INCLUDE_DIR
hdfs_static
)
set(LSan_LIB_NAME lsan)
find_library(LSan_LIBRARY
NAMES liblsan.so liblsan.so.0 liblsan.so.0.0.0
PATHS ${DMLC_USE_SANITIZER} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(LSan DEFAULT_MSG
LSan_LIBRARY)
mark_as_advanced(
LSan_LIBRARY
LSan_LIB_NAME)
set(TSan_LIB_NAME tsan)
find_library(TSan_LIBRARY
NAMES libtsan.so libtsan.so.0 libtsan.so.0.0.0
PATHS ${DMLC_USE_SANITIZER} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(TSan DEFAULT_MSG
TSan_LIBRARY)
mark_as_advanced(
TSan_LIBRARY
TSan_LIB_NAME)
set(UBSan_LIB_NAME UBSan)
find_library(UBSan_LIBRARY
NAMES libubsan.so libubsan.so.5 libubsan.so.4 libubsan.so.3 libubsan.so.2 libubsan.so.1 libubsan.so.0
PATHS ${SANITIZER_PATH} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(UBSan DEFAULT_MSG
UBSan_LIBRARY)
mark_as_advanced(
UBSan_LIBRARY
UBSan_LIB_NAME)
# Set appropriate compiler and linker flags for sanitizers.
#
# Usage of this module:
# enable_sanitizers("address;leak")
# Add flags
macro(enable_sanitizer sanitizer)
if(${sanitizer} MATCHES "address")
find_package(ASan REQUIRED)
set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=address")
link_libraries(${ASan_LIBRARY})
elseif(${sanitizer} MATCHES "thread")
find_package(TSan REQUIRED)
set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=thread")
link_libraries(${TSan_LIBRARY})
elseif(${sanitizer} MATCHES "leak")
find_package(LSan REQUIRED)
set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=leak")
link_libraries(${LSan_LIBRARY})
elseif(${sanitizer} MATCHES "undefined")
find_package(UBSan REQUIRED)
set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=undefined -fno-sanitize-recover=undefined")
link_libraries(${UBSan_LIBRARY})
else()
message(FATAL_ERROR "Santizer ${sanitizer} not supported.")
endif()
endmacro()
macro(enable_sanitizers SANITIZERS)
# Check sanitizers compatibility.
# Idealy, we should use if(san IN_LIST SANITIZERS) ... endif()
# But I haven't figure out how to make it work.
foreach ( _san ${SANITIZERS} )
string(TOLOWER ${_san} _san)
if (_san MATCHES "thread")
if (${_use_other_sanitizers})
message(FATAL_ERROR
"thread sanitizer is not compatible with ${_san} sanitizer.")
endif()
set(_use_thread_sanitizer 1)
else ()
if (${_use_thread_sanitizer})
message(FATAL_ERROR
"${_san} sanitizer is not compatible with thread sanitizer.")
endif()
set(_use_other_sanitizers 1)
endif()
endforeach()
message("Sanitizers: ${SANITIZERS}")
foreach( _san ${SANITIZERS} )
string(TOLOWER ${_san} _san)
enable_sanitizer(${_san})
endforeach()
message("Sanitizers compile flags: ${SAN_COMPILE_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_COMPILE_FLAGS}")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_COMPILE_FLAGS}")
endmacro()
#ifndef DMLC_BUILD_CONFIG_H_
#define DMLC_BUILD_CONFIG_H_
#cmakedefine DMLC_FOPEN_64_PRESENT
#if !defined(DMLC_FOPEN_64_PRESENT) && DMLC_USE_FOPEN64
#define fopen64 std::fopen
#endif
#cmakedefine DMLC_CXXABI_H_PRESENT
#cmakedefine DMLC_EXECINFO_H_PRESENT
#if (defined DMLC_CXXABI_H_PRESENT) && (defined DMLC_EXECINFO_H_PRESENT)
#ifndef DMLC_LOG_STACK_TRACE
#define DMLC_LOG_STACK_TRACE 1
#endif
#ifndef DMLC_LOG_STACK_TRACE_SIZE
#define DMLC_LOG_STACK_TRACE_SIZE 10
#endif
#cmakedefine DMLC_EXECINFO_H <${DMLC_EXECINFO_H}>
#endif
#cmakedefine DMLC_NANOSLEEP_PRESENT
#define DMLC_CMAKE_LITTLE_ENDIAN ${DMLC_CMAKE_LITTLE_ENDIAN}
#endif // DMLC_BUILD_CONFIG_H_
@PACKAGE_INIT@
if(NOT TARGET dmlc::dmlc)
include(${CMAKE_CURRENT_LIST_DIR}/DMLCTargets.cmake)
endif()
cmake_minimum_required(VERSION 2.8.2)
project(googletest-download NONE)
include(ExternalProject)
ExternalProject_Add(googletest
GIT_REPOSITORY https://github.com/google/googletest.git
GIT_TAG release-1.10.0
SOURCE_DIR "${CMAKE_BINARY_DIR}/googletest-src"
BINARY_DIR "${CMAKE_BINARY_DIR}/googletest-build"
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
TEST_COMMAND ""
)
get_filename_component(CMAKE_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/.." ABSOLUTE)
if(NOT MSVC)
set(LINT_COMMAND ${CMAKE_SOURCE_DIR}/scripts/lint.py)
else()
if((NOT PYTHON_EXECUTABLE))
message(FATAL_ERROR "Cannot lint without python")
endif()
# format output so VS can bring us to the offending file/line
set(LINT_COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/scripts/lint.py)
endif()
cmake_policy(SET CMP0009 NEW) # suppress cmake warning
string(REPLACE " " ";" LINT_DIRS ${LINT_DIRS})
string(REPLACE " " ";" EXCLUDE_PATH ${EXCLUDE_PATH})
execute_process(
COMMAND ${LINT_COMMAND} ${PROJECT_NAME} all ${LINT_DIRS} --exclude_path=${EXCLUDE_PATH}
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
ERROR_VARIABLE LINT_OUTPUT
ERROR_STRIP_TRAILING_WHITESPACE
)
message(STATUS ${LINT_OUTPUT})
\ No newline at end of file
# Makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
PAPER =
BUILDDIR = _build
# User-friendly check for sphinx-build
ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
endif
# Internal variables.
PAPEROPT_a4 = -D latex_paper_size=a4
PAPEROPT_letter = -D latex_paper_size=letter
ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
# the i18n builder cannot share the environment and doctrees with the others
I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext
help:
@echo "Please use \`make <target>' where <target> is one of"
@echo " html to make standalone HTML files"
@echo " dirhtml to make HTML files named index.html in directories"
@echo " singlehtml to make a single large HTML file"
@echo " pickle to make pickle files"
@echo " json to make JSON files"
@echo " htmlhelp to make HTML files and a HTML help project"
@echo " qthelp to make HTML files and a qthelp project"
@echo " applehelp to make an Apple Help Book"
@echo " devhelp to make HTML files and a Devhelp project"
@echo " epub to make an epub"
@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
@echo " latexpdf to make LaTeX files and run them through pdflatex"
@echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
@echo " text to make text files"
@echo " man to make manual pages"
@echo " texinfo to make Texinfo files"
@echo " info to make Texinfo files and run them through makeinfo"
@echo " gettext to make PO message catalogs"
@echo " changes to make an overview of all changed/added/deprecated items"
@echo " xml to make Docutils-native XML files"
@echo " pseudoxml to make pseudoxml-XML files for display purposes"
@echo " linkcheck to check all external links for integrity"
@echo " doctest to run all doctests embedded in the documentation (if enabled)"
@echo " coverage to run coverage check of the documentation (if enabled)"
clean:
rm -rf $(BUILDDIR)/*
html:
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
dirhtml:
$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
singlehtml:
$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
@echo
@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
pickle:
$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
@echo
@echo "Build finished; now you can process the pickle files."
json:
$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
@echo
@echo "Build finished; now you can process the JSON files."
htmlhelp:
$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
@echo
@echo "Build finished; now you can run HTML Help Workshop with the" \
".hhp project file in $(BUILDDIR)/htmlhelp."
qthelp:
$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
@echo
@echo "Build finished; now you can run "qcollectiongenerator" with the" \
".qhcp project file in $(BUILDDIR)/qthelp, like this:"
@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/rabit.qhcp"
@echo "To view the help file:"
@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/rabit.qhc"
applehelp:
$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
@echo
@echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
@echo "N.B. You won't be able to view it unless you put it in" \
"~/Library/Documentation/Help or install it in your application" \
"bundle."
devhelp:
$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
@echo
@echo "Build finished."
@echo "To view the help file:"
@echo "# mkdir -p $$HOME/.local/share/devhelp/rabit"
@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/rabit"
@echo "# devhelp"
epub:
$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
@echo
@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
latex:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo
@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
@echo "Run \`make' in that directory to run these through (pdf)latex" \
"(use \`make latexpdf' here to do that automatically)."
latexpdf:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through pdflatex..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
latexpdfja:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through platex and dvipdfmx..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
text:
$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
@echo
@echo "Build finished. The text files are in $(BUILDDIR)/text."
man:
$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
@echo
@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
texinfo:
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo
@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
@echo "Run \`make' in that directory to run these through makeinfo" \
"(use \`make info' here to do that automatically)."
info:
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo "Running Texinfo files through makeinfo..."
make -C $(BUILDDIR)/texinfo info
@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
gettext:
$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
@echo
@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
changes:
$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
@echo
@echo "The overview file is in $(BUILDDIR)/changes."
linkcheck:
$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
@echo
@echo "Link check complete; look for any errors in the above output " \
"or in $(BUILDDIR)/linkcheck/output.txt."
doctest:
$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
@echo "Testing of doctests in the sources finished, look at the " \
"results in $(BUILDDIR)/doctest/output.txt."
coverage:
$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
@echo "Testing of coverage in the sources finished, look at the " \
"results in $(BUILDDIR)/coverage/python.txt."
xml:
$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
@echo
@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
pseudoxml:
$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
@echo
@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
This document is generated by sphinx.
Make sure you cloned the following repos in the root.
- https://github.com/tqchen/recommonmark
Type make html in doc folder.
Using dmlc-core with CMake
==========================
dmlc defines a exported CMake target which can be used by `find_package` command.
For example, if you have a simple C++ project that contains only a main.cc file,
which uses dmlc-core as dependency, the CMakeLists.txt for your project can be
defined as follow:
``` cmake
project(demo)
cmake_minimum_required(VERSION 3.2)
find_package(dmlc REQUIRED)
add_executable(demo main.cc)
target_link_libraries(demo dmlc::dmlc)
```
\ No newline at end of file
# -*- coding: utf-8 -*-
#
# documentation build configuration file, created by
# sphinx-quickstart on Thu Jul 23 19:40:08 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
import sys
import os, subprocess
import shlex
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
sys.path.insert(0, curr_path)
from sphinx_util import MarkdownParser, AutoStructify
# -- General configuration ------------------------------------------------
# General information about the project.
project = u'dmlc-core'
copyright = u'2015, dmlc-core developers'
author = u'dmlc-core developers'
github_doc_root = 'https://github.com/dmlc-core/dmlc-core/tree/master/doc/'
# add markdown parser
MarkdownParser.github_doc_root = github_doc_root
source_parsers = {
'.md': MarkdownParser,
}
# Version information.
version = "0.5.0"
release = version
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.napoleon',
'sphinx.ext.mathjax',
'breathe',
]
# Use breathe to include doxygen documents
breathe_projects = {'dmlc-core' : 'doxygen/xml/'}
breathe_default_project = 'dmlc-core'
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = ['.rst', '.md']
# The encoding of source files.
#source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = 'index'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']
# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []
# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False
# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False
# -- Options for HTML output ----------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
# html_theme = 'alabaster'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
# Output file base name for HTML help builder.
htmlhelp_basename = project + 'doc'
# -- Options for LaTeX output ---------------------------------------------
latex_elements = {
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'dmlc-core.tex', project,
author, 'manual'),
]
# hook for doxygen
def run_doxygen(folder):
"""Run the doxygen make command in the designated folder."""
try:
retcode = subprocess.call("cd %s; make doxygen" % folder, shell=True)
retcode = subprocess.call("rm -rf _build/html/doxygen", shell=True)
retcode = subprocess.call("mkdir _build", shell=True)
retcode = subprocess.call("mkdir _build/html", shell=True)
retcode = subprocess.call("cp -rf doxygen/html _build/html/doxygen", shell=True)
if retcode < 0:
sys.stderr.write("doxygen terminated by signal %s" % (-retcode))
except OSError as e:
sys.stderr.write("doxygen execution failed: %s" % e)
def generate_doxygen_xml(app):
"""Run the doxygen make commands if we're on the ReadTheDocs server"""
run_doxygen('..')
def setup(app):
# Add hook for building doxygen xml when needed
app.connect("builder-inited", generate_doxygen_xml)
app.add_config_value('recommonmark_config', {
'url_resolver': lambda url: github_doc_root + url,
}, True)
app.add_transform(AutoStructify)
DMLC-Core Documentation
=======================
DMLC Core contains common codebase to help us build machine learning toolkits easier.
Contents
--------
* [Using dmlc-core with CMake](build.md)
* [Parameter Structure for Machine Learning](parameter.md)
* [Doxygen C++ API Reference](https://dmlc-core.readthedocs.org/en/latest/doxygen)
Indices and tables
------------------
```eval_rst
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
```
Parameter Structure for Machine Learning
========================================
One of the most important ingredients of machine learning projects are the parameters.
Parameters act as a way of communication between users and the library. In this article, we will introduce the parameter module of DMLC, a lightweight C++ module that is designed to support
general machine learning libraries. It comes with the following nice properties:
- Easy declaration of typed fields, default values and constraints.
- Auto checking of constraints and throw exceptions when constraint is not met.
- Auto generation of human readable docstrings on parameters.
- Serialization and de-serialization into JSON and ```std::map<std::string, std::string>```.
Use Parameter Module
--------------------
### Declare the Parameter
In the dmlc parameter module, every parameter can be declared as a structure.
This means you can easily access these fields as they normally are efficiently.
For example, it is very common to write
```c++
weight -= param.learning_rate * gradient;
```
The only difference between a normal structure is that we will need to declare
all the fields, as well as their default value and constraints.
The following code gives an example of declaring parameter structure ```MyParam```.
```c++
#include <dmlc/parameter.h>
// declare the parameter, normally put it in header file.
struct MyParam : public dmlc::Parameter<MyParam> {
float learning_rate;
int num_hidden;
int activation;
std::string name;
// declare parameters
DMLC_DECLARE_PARAMETER(MyParam) {
DMLC_DECLARE_FIELD(num_hidden).set_range(0, 1000)
.describe("Number of hidden unit in the fully connected layer.");
DMLC_DECLARE_FIELD(learning_rate).set_default(0.01f)
.describe("Learning rate of SGD optimization.");
DMLC_DECLARE_FIELD(activation).add_enum("relu", 1).add_enum("sigmoid", 2)
.describe("Activation function type.");
DMLC_DECLARE_FIELD(name).set_default("layer")
.describe("Name of the net.");
}
};
// register the parameter, this is normally in a cc file.
DMLC_REGISTER_PARAMETER(MyParam);
```
We can find that the only difference is the lines after ```DMLC_DECLARE_PARAMETER(MyParam)```,
where all the fields are declared. In this example, we have declared parameters of ```float,int,string``` types.
Here are some highlights in this example:
- For the numeric parameters, it is possible to set a range constraints via ```.set_range(begin, end)```.
- It is possible to define enumeration types, in this case activation.
User is only allowed to set ```sigmoid``` or ```relu``` into the activation field, and they will be mapped into 1 and 2 separately.
- The ```describe``` function adds a description on the field, which is used to generate human readable docstring.
### Set the Parameters
After we declared the parameters, we can declare this structure as normal structure.
Except that the ```MyParam``` structure now comes with a few member functions
to make parameter manipulation easy.
To set the parameters from external data source, we can use the ```Init``` function.
```c++
int main() {
MyParam param;
std::vector<std::pair<std::string, std::string> > param_data = {
{"num_hidden", "100"},
{"activation", "relu"},
{"name", "myname"}
};
// set the parameters
param.Init(param_data);
return 0;
}
```
After the ```Init``` function is called, the ```param``` will be filled with the specified key values in ```param_data```.
More importantly, the ```Init``` function will do automatic checking of parameter range and throw an ```dmlc::ParamError```
with detailed error message if things went wrong.
### Generate Human Readable Docstrings
Another useful feature of the parameter module is to get an human readable docstring of the parameter.
This is helpful when we are creating language binding such as python and R, and we can use it to generate docstring of
foreign language interface.
The following code obtains the dostring of ```MyParam```.
```c++
std::string docstring = MyParam::__DOC__();
```
We also provide a more structured way to access the detail of the fields(name, default value, detailed description) via
```c++
std::vector<dmlc::ParamFieldInfo> fields = MyParam::__FIELDS__();
```
### Serialization of Parameters
One of the most common way to serialize the parameter is to convert it back to representation of ```std::map<string, string>```
by using the following code.
```c++
std::map<string, string> dict = param.__DICT__();
```
The ```std::map<string, string>``` can further be serialized easily. This way of serialization is more device and platform(32/64 bit) agnostic.
However, this is not very compact, and recommended only used to serialize the general parameters set by the user.
Direct serialization and loading of JSON format is also support.
### Play with an Example
We provide an example program [parameter.cc](https://github.com/dmlc/dmlc-core/blob/master/example/parameter.cc), to
demonstrate the usage mentioned above, and allow you to play with it and get sense of what is going on.
How does it work
----------------
Hope you like the parameter module so far. In this section, we will explain how does it work. Making such parameter module
in ```C++``` is not easy. Because this basically means some way of reflection -- getting the information of fields in a
structure out, which is not supported by ```C++```.
Consider the following program, how do the Init function know the location of ```num_hidden```, and set it correctly
in ```Init``` function?
```c++
#include <vector>
#include <string>
#include <dmlc/parameter.h>
// declare the parameter, normally put it in header file.
struct MyParam : public dmlc::Parameter<MyParam> {
float learning_rate;
int num_hidden;
// declare parameters
DMLC_DECLARE_PARAMETER(MyParam) {
DMLC_DECLARE_FIELD(num_hidden);
DMLC_DECLARE_FIELD(learning_rate).set_default(0.01f);
}
};
// register the parameter, this is normally in a cc file.
DMLC_REGISTER_PARAMETER(MyParam);
int main(int argc, char *argv[]) {
MyParam param;
std::vector<std::pair<std::string, std::string> > param_data = {
{"num_hidden", "100"},
};
param.Init(param_data);
return 0;
}
```
The secret lies in the function ```DMLC_DECLARE_PARAMETER(MyParam)```, this is a macro defined in the parameter module.
If we expand the macro, the code roughly becomes the following code.
```c++
struct Parameter<MyParam> {
template<typename ValueType>
inline FieldEntry<ValueType>&
DECLARE(ParamManagerSingleton<MyParam> *manager,
const std::string& key,
ValueType& ref){
// offset gives a generic way to access the address of the field
// from beginning of the structure.
size_t offset = ((char*)&ref - (char*)this);
parameter::FieldEntry<ValueType> *e =
new parameter::FieldEntry<ValueType>(key, offset);
manager->AddEntry(key, e);
return *e;
}
};
struct MyParam : public dmlc::Parameter<MyParam> {
float learning_rate;
int num_hidden;
// declare parameters
inline void __DECLARE__(ParamManagerSingleton<MyParam> *manager) {
this->DECLARE(manager, "num_hidden", num_hidden);
this->DECLARE(manager, "learning_rate", learning_rate).set_default(0.01f);
}
};
// This code is only used to show the general idea.
// This code will only run once, the real code is done via singleton declaration pattern.
{
static ParamManagerSingleton<MyParam> manager;
MyParam tmp;
tmp->__DECLARE__(&manager);
}
```
This is not the actual code that runs, but generally shows the idea on how it works.
The key is that the structure layout is fixed for all the instances of objects.
To figure out how to access each of the field, we can
- Create an instance of MyParam, call the ```__DECLARE__``` function.
- The relative position of the field against the head of the structure is recorded into a global singleton.
- When we call ```Init```, we can get the ```offset``` from the singleton, and access the address of the field via ```(ValueType*)((char*)this + offset)```.
You are welcomed to check out the real details in [dmlc/parameter.h](https://github.com/dmlc/dmlc-core/blob/master/include/dmlc/parameter.h).
By using the generic template programming in C++, we have created a simple and useful parameter module for machine learning libraries.
This module is used extensively by DMLC projects. Hope you will find it useful as well :).
# -*- coding: utf-8 -*-
"""Helper utilty function for customization."""
import sys
import os
import docutils
import subprocess
if os.environ.get('READTHEDOCS', None) == 'True':
subprocess.call('cd ..; rm -rf recommonmark;' +
'git clone https://github.com/tqchen/recommonmark', shell=True)
sys.path.insert(0, os.path.abspath('../recommonmark/'))
from recommonmark import parser, transform
MarkdownParser = parser.CommonMarkParser
AutoStructify = transform.AutoStructify
ALL_EXAMPLE=example/parameter
example/parameter: example/parameter.cc libdmlc.a
$(ALL_EXAMPLE) :
$(CXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c %.cc %.a, $^) $(LDFLAGS)
// This is an example program showing usage of parameter module
// Build, on root folder, type
//
// make example
//
// Example usage:
//
// example/parameter num_hidden=100 name=aaa activation=relu
//
#include <dmlc/parameter.h>
struct MyParam : public dmlc::Parameter<MyParam> {
float learning_rate;
int num_hidden;
int activation;
std::string name;
// declare parameters in header file
DMLC_DECLARE_PARAMETER(MyParam) {
DMLC_DECLARE_FIELD(num_hidden).set_range(0, 1000)
.describe("Number of hidden unit in the fully connected layer.");
DMLC_DECLARE_FIELD(learning_rate).set_default(0.01f)
.describe("Learning rate of SGD optimization.");
DMLC_DECLARE_FIELD(activation).add_enum("relu", 1).add_enum("sigmoid", 2)
.describe("Activation function type.");
DMLC_DECLARE_FIELD(name).set_default("mnet")
.describe("Name of the net.");
// user can also set nhidden besides num_hidden
DMLC_DECLARE_ALIAS(num_hidden, nhidden);
DMLC_DECLARE_ALIAS(activation, act);
}
};
// register it in cc file
DMLC_REGISTER_PARAMETER(MyParam);
int main(int argc, char *argv[]) {
if (argc == 1) {
printf("Usage: [key=value] ...\n");
return 0;
}
MyParam param;
std::map<std::string, std::string> kwargs;
for (int i = 0; i < argc; ++i) {
char name[256], val[256];
if (sscanf(argv[i], "%[^=]=%[^\n]", name, val) == 2) {
kwargs[name] = val;
}
}
printf("Docstring\n---------\n%s", MyParam::__DOC__().c_str());
printf("start to set parameters ...\n");
param.Init(kwargs);
printf("-----\n");
printf("param.num_hidden=%d\n", param.num_hidden);
printf("param.learning_rate=%f\n", param.learning_rate);
printf("param.name=%s\n", param.name.c_str());
printf("param.activation=%d\n", param.activation);
return 0;
}
/*!
* Copyright (c) 2016 by Contributors
* \file array_view.h
* \brief Read only data structure to reference array
*/
#ifndef DMLC_ARRAY_VIEW_H_
#define DMLC_ARRAY_VIEW_H_
#include <vector>
#include <array>
namespace dmlc {
/*!
* \brief Read only data structure to reference continuous memory region of array.
* Provide unified view for vector, array and C style array.
* This data structure do not guarantee aliveness of referenced array.
*
* Make sure do not use array_view to record data in async function closures.
* Also do not use array_view to create reference to temporary data structure.
*
* \tparam ValueType The value
*
* \code
* std::vector<int> myvec{1,2,3};
* dmlc::array_view<int> view(myvec);
* // indexed visit to the view.
* LOG(INFO) << view[0];
*
* for (int v : view) {
* // visit each element in the view
* }
* \endcode
*/
template<typename ValueType>
class array_view {
public:
/*! \brief default constructor */
array_view() = default;
/*!
* \brief default copy constructor
* \param other another array view.
*/
array_view(const array_view<ValueType> &other) = default; // NOLINT(*)
#ifndef _MSC_VER
/*!
* \brief default move constructor
* \param other another array view.
*/
array_view(array_view<ValueType>&& other) = default; // NOLINT(*)
#else
/*!
* \brief default move constructor
* \param other another array view.
*/
array_view(array_view<ValueType>&& other) { // NOLINT(*)
begin_ = other.begin_;
size_ = other.size_;
other.begin_ = nullptr;
}
#endif
/*!
* \brief default assign constructor
* \param other another array view.
* \return self.
*/
array_view<ValueType>& operator=(const array_view<ValueType>& other) = default; // NOLINT(*)
/*!
* \brief construct array view std::vector
* \param other vector container
*/
array_view(const std::vector<ValueType>& other) { // NOLINT(*)
if (other.size() != 0) {
begin_ = &other[0]; size_ = other.size();
}
}
/*!
* \brief construct array std::array
* \param other another array view.
*/
template<std::size_t size>
array_view(const std::array<ValueType, size>& other) { // NOLINT(*)
if (size != 0) {
begin_ = &other[0]; size_ = size;
}
}
/*!
* \brief construct array view from continuous segment
* \param begin beginning pointre
* \param end end pointer
*/
array_view(const ValueType* begin, const ValueType* end) {
if (begin < end) {
begin_ = begin;
size_ = end - begin;
}
}
/*! \return size of the array */
inline size_t size() const {
return size_;
}
/*! \return begin of the array */
inline const ValueType* begin() const {
return begin_;
}
/*! \return end point of the array */
inline const ValueType* end() const {
return begin_ + size_;
}
/*!
* \brief get i-th element from the view
* \param i The index.
* \return const reference to i-th element.
*/
inline const ValueType& operator[](size_t i) const {
return begin_[i];
}
private:
/*! \brief the begin of the view */
const ValueType* begin_{nullptr};
/*! \brief The size of the view */
size_t size_{0};
};
} // namespace dmlc
#endif // DMLC_ARRAY_VIEW_H_
/*!
* Copyright (c) 2015 by Contributors
* \file base.h
* \brief defines configuration macros
*/
#ifndef DMLC_BASE_H_
#define DMLC_BASE_H_
/*! \brief whether use glog for logging */
#ifndef DMLC_USE_GLOG
#define DMLC_USE_GLOG 0
#endif
/*!
* \brief whether throw dmlc::Error instead of
* directly calling abort when FATAL error occured
* NOTE: this may still not be perfect.
* do not use FATAL and CHECK in destructors
*/
#ifndef DMLC_LOG_FATAL_THROW
#define DMLC_LOG_FATAL_THROW 1
#endif
/*!
* \brief whether always log a message before throw
* This can help identify the error that cannot be catched.
*/
#ifndef DMLC_LOG_BEFORE_THROW
#define DMLC_LOG_BEFORE_THROW 0
#endif
/*!
* \brief Whether to use customized logger,
* whose output can be decided by other libraries.
*/
#ifndef DMLC_LOG_CUSTOMIZE
#define DMLC_LOG_CUSTOMIZE 0
#endif
/*!
* \brief Whether to enable debug logging feature.
*/
#ifndef DMLC_LOG_DEBUG
#ifdef NDEBUG
#define DMLC_LOG_DEBUG 0
#else
#define DMLC_LOG_DEBUG 1
#endif
#endif
/*!
* \brief Whether to disable date message on the log.
*/
#ifndef DMLC_LOG_NODATE
#define DMLC_LOG_NODATE 0
#endif
/*! \brief whether compile with hdfs support */
#ifndef DMLC_USE_HDFS
#define DMLC_USE_HDFS 0
#endif
/*! \brief whether compile with s3 support */
#ifndef DMLC_USE_S3
#define DMLC_USE_S3 0
#endif
/*! \brief whether or not use parameter server */
#ifndef DMLC_USE_PS
#define DMLC_USE_PS 0
#endif
/*! \brief whether or not use c++11 support */
#ifndef DMLC_USE_CXX11
#if defined(__GXX_EXPERIMENTAL_CXX0X__) || defined(_MSC_VER)
#define DMLC_USE_CXX11 1
#else
#define DMLC_USE_CXX11 (__cplusplus >= 201103L)
#endif
#endif
/*! \brief strict CXX11 support */
#ifndef DMLC_STRICT_CXX11
#if defined(_MSC_VER)
#define DMLC_STRICT_CXX11 1
#else
#define DMLC_STRICT_CXX11 (__cplusplus >= 201103L)
#endif
#endif
/*! \brief Whether cxx11 thread local is supported */
#ifndef DMLC_CXX11_THREAD_LOCAL
#if defined(_MSC_VER)
#define DMLC_CXX11_THREAD_LOCAL (_MSC_VER >= 1900)
#elif defined(__clang__)
#define DMLC_CXX11_THREAD_LOCAL (__has_feature(cxx_thread_local))
#else
#define DMLC_CXX11_THREAD_LOCAL (__cplusplus >= 201103L)
#endif
#endif
/*! \brief Whether to use modern thread local construct */
#ifndef DMLC_MODERN_THREAD_LOCAL
#define DMLC_MODERN_THREAD_LOCAL 1
#endif
/*! \brief whether RTTI is enabled */
#ifndef DMLC_ENABLE_RTTI
#define DMLC_ENABLE_RTTI 1
#endif
/*! \brief whether use fopen64 */
#ifndef DMLC_USE_FOPEN64
#define DMLC_USE_FOPEN64 1
#endif
/// check if g++ is before 4.6
#if DMLC_USE_CXX11 && defined(__GNUC__) && !defined(__clang_version__)
#if __GNUC__ == 4 && __GNUC_MINOR__ < 6
#pragma message("Will need g++-4.6 or higher to compile all" \
"the features in dmlc-core, " \
"compile without c++0x, some features may be disabled")
#undef DMLC_USE_CXX11
#define DMLC_USE_CXX11 0
#endif
#endif
/*!
* \brief Use little endian for binary serialization
* if this is set to 0, use big endian.
*/
#ifndef DMLC_IO_USE_LITTLE_ENDIAN
#define DMLC_IO_USE_LITTLE_ENDIAN 1
#endif
/*!
* \brief Enable std::thread related modules,
* Used to disable some module in mingw compile.
*/
#ifndef DMLC_ENABLE_STD_THREAD
#define DMLC_ENABLE_STD_THREAD DMLC_USE_CXX11
#endif
/*! \brief whether enable regex support, actually need g++-4.9 or higher*/
#ifndef DMLC_USE_REGEX
#define DMLC_USE_REGEX DMLC_STRICT_CXX11
#endif
/*! \brief helper macro to supress unused warning */
#if defined(__GNUC__)
#define DMLC_ATTRIBUTE_UNUSED __attribute__((unused))
#else
#define DMLC_ATTRIBUTE_UNUSED
#endif
/*! \brief helper macro to supress Undefined Behavior Sanitizer for a specific function */
#if defined(__clang__)
#define DMLC_SUPPRESS_UBSAN __attribute__((no_sanitize("undefined")))
#elif defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 409)
#define DMLC_SUPPRESS_UBSAN __attribute__((no_sanitize_undefined))
#else
#define DMLC_SUPPRESS_UBSAN
#endif
/*! \brief helper macro to generate string concat */
#define DMLC_STR_CONCAT_(__x, __y) __x##__y
#define DMLC_STR_CONCAT(__x, __y) DMLC_STR_CONCAT_(__x, __y)
/*!
* \brief Disable copy constructor and assignment operator.
*
* If C++11 is supported, both copy and move constructors and
* assignment operators are deleted explicitly. Otherwise, they are
* only declared but not implemented. Place this macro in private
* section if C++11 is not available.
*/
#ifndef DISALLOW_COPY_AND_ASSIGN
# if DMLC_USE_CXX11
# define DISALLOW_COPY_AND_ASSIGN(T) \
T(T const&) = delete; \
T(T&&) = delete; \
T& operator=(T const&) = delete; \
T& operator=(T&&) = delete
# else
# define DISALLOW_COPY_AND_ASSIGN(T) \
T(T const&); \
T& operator=(T const&)
# endif
#endif
#ifdef __APPLE__
# define off64_t off_t
#endif
#ifdef _MSC_VER
#if _MSC_VER < 1900
// NOTE: sprintf_s is not equivalent to snprintf,
// they are equivalent when success, which is sufficient for our case
#define snprintf sprintf_s
#define vsnprintf vsprintf_s
#endif
#else
#ifdef _FILE_OFFSET_BITS
#if _FILE_OFFSET_BITS == 32
#pragma message("Warning: FILE OFFSET BITS defined to be 32 bit")
#endif
#endif
extern "C" {
#include <sys/types.h>
}
#endif
#ifdef _MSC_VER
//! \cond Doxygen_Suppress
typedef signed char int8_t;
typedef __int16 int16_t;
typedef __int32 int32_t;
typedef __int64 int64_t;
typedef unsigned char uint8_t;
typedef unsigned __int16 uint16_t;
typedef unsigned __int32 uint32_t;
typedef unsigned __int64 uint64_t;
//! \endcond
#else
#include <inttypes.h>
#endif
#include <string>
#include <vector>
#if defined(_MSC_VER) && _MSC_VER < 1900
#define noexcept_true throw ()
#define noexcept_false
#define noexcept(a) noexcept_##a
#endif
#if defined(_MSC_VER)
#define DMLC_NO_INLINE __declspec(noinline)
#else
#define DMLC_NO_INLINE __attribute__((noinline))
#endif
#if DMLC_USE_CXX11
#define DMLC_THROW_EXCEPTION noexcept(false)
#define DMLC_NO_EXCEPTION noexcept(true)
#else
#define DMLC_THROW_EXCEPTION
#define DMLC_NO_EXCEPTION
#endif
/*! \brief namespace for dmlc */
namespace dmlc {
/*!
* \brief safely get the beginning address of a vector
* \param vec input vector
* \return beginning address of a vector
*/
template<typename T>
inline T *BeginPtr(std::vector<T> &vec) { // NOLINT(*)
if (vec.size() == 0) {
return NULL;
} else {
return &vec[0];
}
}
/*!
* \brief get the beginning address of a const vector
* \param vec input vector
* \return beginning address of a vector
*/
template<typename T>
inline const T *BeginPtr(const std::vector<T> &vec) {
if (vec.size() == 0) {
return NULL;
} else {
return &vec[0];
}
}
/*!
* \brief get the beginning address of a string
* \param str input string
* \return beginning address of a string
*/
inline char* BeginPtr(std::string &str) { // NOLINT(*)
if (str.length() == 0) return NULL;
return &str[0];
}
/*!
* \brief get the beginning address of a const string
* \param str input string
* \return beginning address of a string
*/
inline const char* BeginPtr(const std::string &str) {
if (str.length() == 0) return NULL;
return &str[0];
}
} // namespace dmlc
#if defined(_MSC_VER) && _MSC_VER < 1900
#define constexpr const
#define alignof __alignof
#endif
/* If fopen64 is not defined by current machine,
replace fopen64 with std::fopen. Also determine ability to print stack trace
for fatal error and define DMLC_LOG_STACK_TRACE if stack trace can be
produced. Always keep this include directive at the bottom of dmlc/base.h */
#ifdef DMLC_CORE_USE_CMAKE
#include <dmlc/build_config.h>
#else
#include <dmlc/build_config_default.h>
#endif
#endif // DMLC_BASE_H_
/*!
* Copyright (c) 2018 by Contributors
* \file build_config_default.h
* \brief Default detection logic for fopen64 and other symbols.
* May be overriden by CMake
* \author KOLANICH
*/
#ifndef DMLC_BUILD_CONFIG_DEFAULT_H_
#define DMLC_BUILD_CONFIG_DEFAULT_H_
/* default logic for fopen64 */
#if DMLC_USE_FOPEN64 && \
(!defined(__GNUC__) || (defined __ANDROID__) || (defined __FreeBSD__) \
|| (defined __APPLE__) || ((defined __MINGW32__) && !(defined __MINGW64__)) \
|| (defined __CYGWIN__) )
#define fopen64 std::fopen
#endif
/* default logic for stack trace */
#if (defined(__GNUC__) && !defined(__MINGW32__)\
&& !defined(__sun) && !defined(__SVR4)\
&& !(defined __MINGW64__) && !(defined __ANDROID__))\
&& !defined(__CYGWIN__) && !defined(__EMSCRIPTEN__)\
&& !defined(__RISCV__) && !defined(__hexagon__)
#ifndef DMLC_LOG_STACK_TRACE
#define DMLC_LOG_STACK_TRACE 1
#endif
#ifndef DMLC_LOG_STACK_TRACE_SIZE
#define DMLC_LOG_STACK_TRACE_SIZE 10
#endif
#define DMLC_EXECINFO_H <execinfo.h>
#endif
/* default logic for detecting existence of nanosleep() */
#if !(defined _WIN32) || (defined __CYGWIN__)
#define DMLC_NANOSLEEP_PRESENT
#endif
#endif // DMLC_BUILD_CONFIG_DEFAULT_H_
/*!
* Copyright (c) 2015 by Contributors
* \file common.h
* \brief defines some common utility function.
*/
#ifndef DMLC_COMMON_H_
#define DMLC_COMMON_H_
#include <vector>
#include <string>
#include <sstream>
#include <mutex>
#include <utility>
#include "./logging.h"
namespace dmlc {
/*!
* \brief Split a string by delimiter
* \param s String to be splitted.
* \param delim The delimiter.
* \return a splitted vector of strings.
*/
inline std::vector<std::string> Split(const std::string& s, char delim) {
std::string item;
std::istringstream is(s);
std::vector<std::string> ret;
while (std::getline(is, item, delim)) {
ret.push_back(item);
}
return ret;
}
/*!
* \brief hash an object and combines the key with previous keys
*/
template<typename T>
inline size_t HashCombine(size_t key, const T& value) {
std::hash<T> hash_func;
return key ^ (hash_func(value) + 0x9e3779b9 + (key << 6) + (key >> 2));
}
/*!
* \brief specialize for size_t
*/
template<>
inline size_t HashCombine<size_t>(size_t key, const size_t& value) {
return key ^ (value + 0x9e3779b9 + (key << 6) + (key >> 2));
}
/*!
* \brief OMP Exception class catches, saves and rethrows exception from OMP blocks
*/
class OMPException {
private:
// exception_ptr member to store the exception
std::exception_ptr omp_exception_;
// mutex to be acquired during catch to set the exception_ptr
std::mutex mutex_;
public:
/*!
* \brief Parallel OMP blocks should be placed within Run to save exception
*/
template <typename Function, typename... Parameters>
void Run(Function f, Parameters... params) {
try {
f(params...);
} catch (dmlc::Error &ex) {
std::lock_guard<std::mutex> lock(mutex_);
if (!omp_exception_) {
omp_exception_ = std::current_exception();
}
} catch (std::exception &ex) {
std::lock_guard<std::mutex> lock(mutex_);
if (!omp_exception_) {
omp_exception_ = std::current_exception();
}
}
}
/*!
* \brief should be called from the main thread to rethrow the exception
*/
void Rethrow() {
if (this->omp_exception_) std::rethrow_exception(this->omp_exception_);
}
};
} // namespace dmlc
#endif // DMLC_COMMON_H_
/*!
* Copyright (c) 2015 by Contributors
* \file concurrency.h
* \brief thread-safe data structures.
* \author Yutian Li
*/
#ifndef DMLC_CONCURRENCY_H_
#define DMLC_CONCURRENCY_H_
// this code depends on c++11
#if DMLC_USE_CXX11
#include <atomic>
#include <deque>
#include <queue>
#include <mutex>
#include <vector>
#include <utility>
#include <condition_variable>
#include "dmlc/base.h"
namespace dmlc {
/*!
* \brief Simple userspace spinlock implementation.
*/
class Spinlock {
public:
#ifdef _MSC_VER
Spinlock() {
lock_.clear();
}
#else
#if defined(__clang__)
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wbraced-scalar-init"
#endif // defined(__clang__)
Spinlock() : lock_(ATOMIC_FLAG_INIT) {
}
#if defined(__clang__)
#pragma clang diagnostic pop
#endif // defined(__clang__)
#endif
~Spinlock() = default;
/*!
* \brief Acquire lock.
*/
inline void lock() noexcept(true);
/*!
* \brief Release lock.
*/
inline void unlock() noexcept(true);
private:
std::atomic_flag lock_;
/*!
* \brief Disable copy and move.
*/
DISALLOW_COPY_AND_ASSIGN(Spinlock);
};
/*! \brief type of concurrent queue */
enum class ConcurrentQueueType {
/*! \brief FIFO queue */
kFIFO,
/*! \brief queue with priority */
kPriority
};
/*!
* \brief Cocurrent blocking queue.
*/
template <typename T,
ConcurrentQueueType type = ConcurrentQueueType::kFIFO>
class ConcurrentBlockingQueue {
public:
ConcurrentBlockingQueue();
~ConcurrentBlockingQueue() = default;
/*!
* \brief Push element to the end of the queue.
* \param e Element to push into.
* \param priority the priority of the element, only used for priority queue.
* The higher the priority is, the better.
* \tparam E the element type
*
* It will copy or move the element into the queue, depending on the type of
* the parameter.
*/
template <typename E>
void Push(E&& e, int priority = 0);
/*!
* \brief Push element to the front of the queue. Only works for FIFO queue.
* For priority queue it is the same as Push.
* \param e Element to push into.
* \param priority the priority of the element, only used for priority queue.
* The higher the priority is, the better.
* \tparam E the element type
*
* It will copy or move the element into the queue, depending on the type of
* the parameter.
*/
template <typename E>
void PushFront(E&& e, int priority = 0);
/*!
* \brief Pop element from the queue.
* \param rv Element popped.
* \return On false, the queue is exiting.
*
* The element will be copied or moved into the object passed in.
*/
bool Pop(T* rv);
/*!
* \brief Signal the queue for destruction.
*
* After calling this method, all blocking pop call to the queue will return
* false.
*/
void SignalForKill();
/*!
* \brief Get the size of the queue.
* \return The size of the queue.
*/
size_t Size();
private:
struct Entry {
T data;
int priority;
inline bool operator<(const Entry &b) const {
return priority < b.priority;
}
};
std::mutex mutex_;
std::condition_variable cv_;
std::atomic<bool> exit_now_;
int nwait_consumer_;
// a priority queue
std::vector<Entry> priority_queue_;
// a FIFO queue
std::deque<T> fifo_queue_;
/*!
* \brief Disable copy and move.
*/
DISALLOW_COPY_AND_ASSIGN(ConcurrentBlockingQueue);
};
inline void Spinlock::lock() noexcept(true) {
while (lock_.test_and_set(std::memory_order_acquire)) {
}
}
inline void Spinlock::unlock() noexcept(true) {
lock_.clear(std::memory_order_release);
}
template <typename T, ConcurrentQueueType type>
ConcurrentBlockingQueue<T, type>::ConcurrentBlockingQueue()
: exit_now_{false}, nwait_consumer_{0} {}
template <typename T, ConcurrentQueueType type>
template <typename E>
void ConcurrentBlockingQueue<T, type>::Push(E&& e, int priority) {
static_assert(std::is_same<typename std::remove_cv<
typename std::remove_reference<E>::type>::type,
T>::value,
"Types must match.");
bool notify;
{
std::lock_guard<std::mutex> lock{mutex_};
if (type == ConcurrentQueueType::kFIFO) {
fifo_queue_.emplace_back(std::forward<E>(e));
notify = nwait_consumer_ != 0;
} else {
Entry entry;
entry.data = std::move(e);
entry.priority = priority;
priority_queue_.push_back(std::move(entry));
std::push_heap(priority_queue_.begin(), priority_queue_.end());
notify = nwait_consumer_ != 0;
}
}
if (notify) cv_.notify_one();
}
template <typename T, ConcurrentQueueType type>
template <typename E>
void ConcurrentBlockingQueue<T, type>::PushFront(E&& e, int priority) {
static_assert(std::is_same<typename std::remove_cv<
typename std::remove_reference<E>::type>::type,
T>::value,
"Types must match.");
bool notify;
{
std::lock_guard<std::mutex> lock{mutex_};
if (type == ConcurrentQueueType::kFIFO) {
fifo_queue_.emplace_front(std::forward<E>(e));
notify = nwait_consumer_ != 0;
} else {
Entry entry;
entry.data = std::move(e);
entry.priority = priority;
priority_queue_.push_back(std::move(entry));
std::push_heap(priority_queue_.begin(), priority_queue_.end());
notify = nwait_consumer_ != 0;
}
}
if (notify) cv_.notify_one();
}
template <typename T, ConcurrentQueueType type>
bool ConcurrentBlockingQueue<T, type>::Pop(T* rv) {
std::unique_lock<std::mutex> lock{mutex_};
if (type == ConcurrentQueueType::kFIFO) {
++nwait_consumer_;
cv_.wait(lock, [this] {
return !fifo_queue_.empty() || exit_now_.load();
});
--nwait_consumer_;
if (!exit_now_.load()) {
*rv = std::move(fifo_queue_.front());
fifo_queue_.pop_front();
return true;
} else {
return false;
}
} else {
++nwait_consumer_;
cv_.wait(lock, [this] {
return !priority_queue_.empty() || exit_now_.load();
});
--nwait_consumer_;
if (!exit_now_.load()) {
std::pop_heap(priority_queue_.begin(), priority_queue_.end());
*rv = std::move(priority_queue_.back().data);
priority_queue_.pop_back();
return true;
} else {
return false;
}
}
}
template <typename T, ConcurrentQueueType type>
void ConcurrentBlockingQueue<T, type>::SignalForKill() {
{
std::lock_guard<std::mutex> lock{mutex_};
exit_now_.store(true);
}
cv_.notify_all();
}
template <typename T, ConcurrentQueueType type>
size_t ConcurrentBlockingQueue<T, type>::Size() {
std::lock_guard<std::mutex> lock{mutex_};
if (type == ConcurrentQueueType::kFIFO) {
return fifo_queue_.size();
} else {
return priority_queue_.size();
}
}
} // namespace dmlc
#endif // DMLC_USE_CXX11
#endif // DMLC_CONCURRENCY_H_
This source diff could not be displayed because it is too large. You can view the blob instead.
/*!
* Copyright (c) 2015 by Contributors
* \file config.h
* \brief defines config parser class
*/
#ifndef DMLC_CONFIG_H_
#define DMLC_CONFIG_H_
#include <cstring>
#include <iostream>
#include <iterator>
#include <map>
#include <vector>
#include <utility>
#include <string>
#include <sstream>
/*! \brief namespace for dmlc */
namespace dmlc {
/*!
* \brief class for config parser
*
* Two modes are supported:
* 1. non-multi value mode: if two same keys in the configure file, the later one will replace the
* ealier one; when using iterator, the order will be the "last effective insersion" order
* 2. multi value mode: multiple values with the same key could co-exist; when using iterator, the
* order will be the insersion order.
*
* [Basic usage]
*
* Config cfg(file_input_stream);
* for(Config::ConfigIterator iter = cfg.begin(); iter != cfg.end(); ++iter) {
* ConfigEntry ent = *iter;
* std::string key = ent.first;
* std::string value = ent.second;
* do_something_with(key, value);
* }
*/
class Config {
public:
/*!
* \brief type when extracting from iterator
*/
typedef std::pair<std::string, std::string> ConfigEntry;
/*!
* \brief iterator class
*/
class ConfigIterator;
/*!
* \brief create empty config
* \param multi_value whether the config supports multi value
*/
explicit Config(bool multi_value = false);
/*!
* \brief create config and load content from the given stream
* \param is input stream
* \param multi_value whether the config supports multi value
*/
explicit Config(std::istream& is, bool multi_value = false); // NOLINT(*)
/*!
* \brief clear all the values
*/
void Clear(void);
/*!
* \brief load the contents from the stream
* \param is the stream as input
*/
void LoadFromStream(std::istream& is); // NOLINT(*)
/*!
* \brief set a key-value pair into the config; if the key already exists in the configure file,
* it will either replace the old value with the given one (in non-multi value mode) or
* store it directly (in multi-value mode);
* \param key key
* \param value value
* \param is_string whether the value should be wrapped by quotes in proto string
*/
template<class T>
void SetParam(const std::string& key, const T& value, bool is_string = false);
/*!
* \brief get the config under the key; if multiple values exist for the same key,
* return the last inserted one.
* \param key key
* \return config value
*/
const std::string& GetParam(const std::string& key) const;
/*!
* \brief check whether the configure value given by the key should be wrapped by quotes
* \param key key
* \return whether the configure value is represented by string
*/
bool IsGenuineString(const std::string& key) const;
/*!
* \brief transform all the configuration into string recognizable to protobuf
* \return string that could be parsed directly by protobuf
*/
std::string ToProtoString(void) const;
/*!
* \brief get begin iterator
* \return begin iterator
*/
ConfigIterator begin() const;
/*!
* \brief get end iterator
* \return end iterator
*/
ConfigIterator end() const;
public:
/*!
* \brief iterator class
*/
class ConfigIterator : public std::iterator< std::input_iterator_tag, ConfigEntry > {
friend class Config;
public:
/*!
* \brief copy constructor
*/
ConfigIterator(const ConfigIterator& other);
/*!
* \brief uni-increment operators
* \return the reference of current config
*/
ConfigIterator& operator++();
/*!
* \brief uni-increment operators
* \return the reference of current config
*/
ConfigIterator operator++(int); // NOLINT(*)
/*!
* \brief compare operators
* \param rhs the other config to compare against
* \return the compared result
*/
bool operator == (const ConfigIterator& rhs) const;
/*!
* \brief compare operators not equal
* \param rhs the other config to compare against
* \return the compared result
*/
bool operator != (const ConfigIterator& rhs) const;
/*!
* \brief retrieve value from operator
*/
ConfigEntry operator * () const;
private:
ConfigIterator(size_t index, const Config* config);
void FindNextIndex();
private:
size_t index_;
const Config* config_;
};
private:
struct ConfigValue {
std::vector<std::string> val;
std::vector<size_t> insert_index;
bool is_string;
};
void Insert(const std::string& key, const std::string& value, bool is_string);
private:
std::map<std::string, ConfigValue> config_map_;
std::vector<std::pair<std::string, size_t> > order_;
const bool multi_value_;
};
template<class T>
void Config::SetParam(const std::string& key, const T& value, bool is_string) {
std::ostringstream oss;
oss << value;
Insert(key, oss.str(), is_string);
}
} // namespace dmlc
#endif // DMLC_CONFIG_H_
/*!
* Copyright (c) 2017 by Contributors
* \file endian.h
* \brief Endian testing, need c++11
*/
#ifndef DMLC_ENDIAN_H_
#define DMLC_ENDIAN_H_
#include "./base.h"
#ifdef DMLC_CMAKE_LITTLE_ENDIAN
// If compiled with CMake, use CMake's endian detection logic
#define DMLC_LITTLE_ENDIAN DMLC_CMAKE_LITTLE_ENDIAN
#else
#if defined(__APPLE__) || defined(_WIN32)
#define DMLC_LITTLE_ENDIAN 1
#elif defined(__GLIBC__) || defined(__GNU_LIBRARY__) \
|| defined(__ANDROID__) || defined(__RISCV__)
#include <endian.h>
#define DMLC_LITTLE_ENDIAN (__BYTE_ORDER == __LITTLE_ENDIAN)
#elif defined(__FreeBSD__) || defined(__OpenBSD__)
#include <sys/endian.h>
#define DMLC_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN)
#elif defined(__EMSCRIPTEN__) || defined(__hexagon__)
#define DMLC_LITTLE_ENDIAN 1
#elif defined(__sun) || defined(sun)
#include <sys/isa_defs.h>
#if defined(_LITTLE_ENDIAN)
#define DMLC_LITTLE_ENDIAN 1
#else
#define DMLC_LITTLE_ENDIAN 0
#endif
#else
#error "Unable to determine endianness of your machine; use CMake to compile"
#endif
#endif
/*! \brief whether serialize using little endian */
#define DMLC_IO_NO_ENDIAN_SWAP (DMLC_LITTLE_ENDIAN == DMLC_IO_USE_LITTLE_ENDIAN)
namespace dmlc {
/*!
* \brief A generic inplace byte swapping function.
* \param data The data pointer.
* \param elem_bytes The number of bytes of the data elements
* \param num_elems Number of elements in the data.
* \note Always try pass in constant elem_bytes to enable
* compiler optimization
*/
inline void ByteSwap(void* data, size_t elem_bytes, size_t num_elems) {
for (size_t i = 0; i < num_elems; ++i) {
uint8_t* bptr = reinterpret_cast<uint8_t*>(data) + elem_bytes * i;
for (size_t j = 0; j < elem_bytes / 2; ++j) {
uint8_t v = bptr[elem_bytes - 1 - j];
bptr[elem_bytes - 1 - j] = bptr[j];
bptr[j] = v;
}
}
}
} // namespace dmlc
#endif // DMLC_ENDIAN_H_
/*!
* Copyright (c) 2018 by Contributors
* \file filesystem.h
* \brief Utilities to manipulate files
* \author Hyunsu Philip Cho
*/
#ifndef DMLC_FILESYSTEM_H_
#define DMLC_FILESYSTEM_H_
#include <dmlc/logging.h>
#include <dmlc/io.h>
#include <algorithm>
#include <string>
#include <vector>
#include <random>
/* platform specific headers */
#ifdef _WIN32
#define NOMINMAX
#include <windows.h>
#include <Shlwapi.h>
#pragma comment(lib, "Shlwapi.lib")
#else // _WIN32
#include <unistd.h>
#include <sys/stat.h>
#include <sys/types.h>
#endif // _WIN32
namespace dmlc {
/*!
* \brief Manager class for temporary directories. Whenever a new
* TemporaryDirectory object is constructed, a temporary directory is
* created. The directory is deleted when the object is deleted or goes
* out of scope. Note: no symbolic links are allowed inside the
* temporary directory.
*
* Usage example:
* \code
*
* void foo() {
* dmlc::TemporaryDirectory tempdir;
* // Create a file my_file.txt inside the temporary directory
* std::ofstream of(tempdir.path + "/my_file.txt");
* // ... write to my_file.txt ...
*
* // ... use my_file.txt
*
* // When tempdir goes out of scope, the temporary directory is deleted
* }
*
* \endcode
*/
class TemporaryDirectory {
public:
/*!
* \brief Default constructor.
* Creates a new temporary directory with a unique name.
* \param verbose whether to emit extra messages
*/
explicit TemporaryDirectory(bool verbose = false)
: verbose_(verbose) {
#if _WIN32
/* locate the root directory of temporary area */
char tmproot[MAX_PATH] = {0};
const DWORD dw_retval = GetTempPathA(MAX_PATH, tmproot);
if (dw_retval > MAX_PATH || dw_retval == 0) {
LOG(FATAL) << "TemporaryDirectory(): "
<< "Could not create temporary directory";
}
/* generate a unique 8-letter alphanumeric string */
const std::string letters = "abcdefghijklmnopqrstuvwxyz0123456789_";
std::string uniqstr(8, '\0');
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_int_distribution<int> dis(0, letters.length() - 1);
std::generate(uniqstr.begin(), uniqstr.end(),
[&dis, &gen, &letters]() -> char {
return letters[dis(gen)];
});
/* combine paths to get the name of the temporary directory */
char tmpdir[MAX_PATH] = {0};
PathCombineA(tmpdir, tmproot, uniqstr.c_str());
if (!CreateDirectoryA(tmpdir, NULL)) {
LOG(FATAL) << "TemporaryDirectory(): "
<< "Could not create temporary directory";
}
path = std::string(tmpdir);
#else // _WIN32
std::string tmproot; /* root directory of temporary area */
std::string dirtemplate; /* template for temporary directory name */
/* Get TMPDIR env variable or fall back to /tmp/ */
{
const char* tmpenv = getenv("TMPDIR");
if (tmpenv) {
tmproot = std::string(tmpenv);
// strip trailing forward slashes
while (tmproot.length() != 0 && tmproot[tmproot.length() - 1] == '/') {
tmproot.resize(tmproot.length() - 1);
}
} else {
tmproot = "/tmp";
}
}
dirtemplate = tmproot + "/tmpdir.XXXXXX";
std::vector<char> dirtemplate_buf(dirtemplate.begin(), dirtemplate.end());
dirtemplate_buf.push_back('\0');
char* tmpdir = mkdtemp(&dirtemplate_buf[0]);
if (!tmpdir) {
LOG(FATAL) << "TemporaryDirectory(): "
<< "Could not create temporary directory";
}
path = std::string(tmpdir);
#endif // _WIN32
if (verbose_) {
LOG(INFO) << "Created temporary directory " << path;
}
}
/*! \brief Destructor. Will perform recursive deletion via RecursiveDelete() */
~TemporaryDirectory() {
RecursiveDelete(path);
}
/*! \brief Full path of the temporary directory */
std::string path;
private:
/*! \brief Whether to emit extra messages */
bool verbose_;
/*!
* \brief Determine whether a given path is a symbolic link
* \param path String representation of path
*/
inline bool IsSymlink(const std::string& path) {
#ifdef _WIN32
DWORD attr = GetFileAttributesA(path.c_str());
CHECK_NE(attr, INVALID_FILE_ATTRIBUTES)
<< "dmlc::TemporaryDirectory::IsSymlink(): Unable to read file attributes";
return attr & FILE_ATTRIBUTE_REPARSE_POINT;
#else // _WIN32
struct stat sb;
CHECK_EQ(lstat(path.c_str(), &sb), 0)
<< "dmlc::TemporaryDirectory::IsSymlink(): Unable to read file attributes";
return S_ISLNK(sb.st_mode);
#endif // _WIN32
}
/*!
* \brief Delete a directory recursively, along with sub-directories and files.
* \param path String representation of path. It must refer to a directory.
*/
void RecursiveDelete(const std::string& path);
};
} // namespace dmlc
#endif // DMLC_FILESYSTEM_H_
/*!
* Copyright (c) 2016 by Contributors
* \file input_split_shuffle.h
* \brief base class to construct input split with global shuffling
* \author Yifeng Geng
*/
#ifndef DMLC_INPUT_SPLIT_SHUFFLE_H_
#define DMLC_INPUT_SPLIT_SHUFFLE_H_
#include <cstdio>
#include <cstring>
#include <vector>
#include <string>
#include <algorithm>
#include <memory>
namespace dmlc {
/*! \brief class to construct input split with global shuffling */
class InputSplitShuffle : public InputSplit {
public:
// destructor
virtual ~InputSplitShuffle(void) { source_.reset(); }
// implement BeforeFirst
virtual void BeforeFirst(void) {
if (num_shuffle_parts_ > 1) {
std::shuffle(shuffle_indexes_.begin(), shuffle_indexes_.end(), trnd_);
int idx = shuffle_indexes_[0] + part_index_ * num_shuffle_parts_;
source_->ResetPartition(idx, num_parts_ * num_shuffle_parts_);
cur_shuffle_idx_ = 0;
} else {
source_->BeforeFirst();
}
}
virtual void HintChunkSize(size_t chunk_size) {
source_->HintChunkSize(chunk_size);
}
virtual size_t GetTotalSize(void) {
return source_->GetTotalSize();
}
// implement next record
virtual bool NextRecord(Blob *out_rec) {
if (num_shuffle_parts_ > 1) {
if (!source_->NextRecord(out_rec)) {
if (cur_shuffle_idx_ == num_shuffle_parts_ - 1) {
return false;
}
++cur_shuffle_idx_;
int idx =
shuffle_indexes_[cur_shuffle_idx_] + part_index_ * num_shuffle_parts_;
source_->ResetPartition(idx, num_parts_ * num_shuffle_parts_);
return NextRecord(out_rec);
} else {
return true;
}
} else {
return source_->NextRecord(out_rec);
}
}
// implement next chunk
virtual bool NextChunk(Blob* out_chunk) {
if (num_shuffle_parts_ > 1) {
if (!source_->NextChunk(out_chunk)) {
if (cur_shuffle_idx_ == num_shuffle_parts_ - 1) {
return false;
}
++cur_shuffle_idx_;
int idx =
shuffle_indexes_[cur_shuffle_idx_] + part_index_ * num_shuffle_parts_;
source_->ResetPartition(idx, num_parts_ * num_shuffle_parts_);
return NextChunk(out_chunk);
} else {
return true;
}
} else {
return source_->NextChunk(out_chunk);
}
}
// implement ResetPartition.
virtual void ResetPartition(unsigned rank, unsigned nsplit) {
CHECK(nsplit == num_parts_) << "num_parts is not consistent!";
int idx = shuffle_indexes_[0] + rank * num_shuffle_parts_;
source_->ResetPartition(idx, nsplit * num_shuffle_parts_);
cur_shuffle_idx_ = 0;
}
/*!
* \brief constructor
* \param uri the uri of the input, can contain hdfs prefix
* \param part_index the part id of current input
* \param num_parts total number of splits
* \param type type of record
* List of possible types: "text", "recordio"
* - "text":
* text file, each line is treated as a record
* input split will split on '\\n' or '\\r'
* - "recordio":
* binary recordio file, see recordio.h
* \param num_shuffle_parts number of shuffle chunks for each split
* \param shuffle_seed shuffle seed for chunk shuffling
*/
InputSplitShuffle(const char* uri,
unsigned part_index,
unsigned num_parts,
const char* type,
unsigned num_shuffle_parts,
int shuffle_seed)
: part_index_(part_index),
num_parts_(num_parts),
num_shuffle_parts_(num_shuffle_parts),
cur_shuffle_idx_(0) {
for (unsigned i = 0; i < num_shuffle_parts_; i++) {
shuffle_indexes_.push_back(i);
}
trnd_.seed(kRandMagic_ + part_index_ + num_parts_ + num_shuffle_parts_ +
shuffle_seed);
std::shuffle(shuffle_indexes_.begin(), shuffle_indexes_.end(), trnd_);
int idx = shuffle_indexes_[cur_shuffle_idx_] + part_index_ * num_shuffle_parts_;
source_.reset(
InputSplit::Create(uri, idx , num_parts_ * num_shuffle_parts_, type));
}
/*!
* \brief factory function:
* create input split with chunk shuffling given a uri
* \param uri the uri of the input, can contain hdfs prefix
* \param part_index the part id of current input
* \param num_parts total number of splits
* \param type type of record
* List of possible types: "text", "recordio"
* - "text":
* text file, each line is treated as a record
* input split will split on '\\n' or '\\r'
* - "recordio":
* binary recordio file, see recordio.h
* \param num_shuffle_parts number of shuffle chunks for each split
* \param shuffle_seed shuffle seed for chunk shuffling
* \return a new input split
* \sa InputSplit::Type
*/
static InputSplit* Create(const char* uri,
unsigned part_index,
unsigned num_parts,
const char* type,
unsigned num_shuffle_parts,
int shuffle_seed) {
CHECK(num_shuffle_parts > 0) << "number of shuffle parts should be greater than zero!";
return new InputSplitShuffle(
uri, part_index, num_parts, type, num_shuffle_parts, shuffle_seed);
}
private:
// magic nyumber for seed
static const int kRandMagic_ = 666;
/*! \brief random engine */
std::mt19937 trnd_;
/*! \brief inner inputsplit */
std::unique_ptr<InputSplit> source_;
/*! \brief part index */
unsigned part_index_;
/*! \brief number of parts */
unsigned num_parts_;
/*! \brief the number of block for shuffling*/
unsigned num_shuffle_parts_;
/*! \brief current shuffle block index */
unsigned cur_shuffle_idx_;
/*! \brief shuffled indexes */
std::vector<int> shuffle_indexes_;
};
} // namespace dmlc
#endif // DMLC_INPUT_SPLIT_SHUFFLE_H_
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This source diff could not be displayed because it is too large. You can view the blob instead.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment