diff options
-rw-r--r-- | .SRCINFO | 35 | ||||
-rw-r--r-- | 010-caffe2-fix-include-system-path.patch | 11 | ||||
-rw-r--r-- | 020-caffe2-use-system-libuv.patch | 22 | ||||
-rw-r--r-- | PKGBUILD | 166 |
4 files changed, 166 insertions, 68 deletions
@@ -1,7 +1,7 @@ pkgbase = caffe2-git pkgdesc = A new lightweight, modular, and scalable deep learning framework - pkgver = 0.8.2.r18962.gd8de69d621 - pkgrel = 2 + pkgver = 0.8.2.r28489.gc5b4f60fc24 + pkgrel = 1 epoch = 1 url = https://caffe2.ai/ arch = x86_64 @@ -13,8 +13,10 @@ pkgbase = caffe2-git makedepends = cuda makedepends = cudnn makedepends = nccl + makedepends = pybind11 depends = google-glog depends = protobuf + depends = openmp depends = python depends = python-numpy depends = python-protobuf @@ -22,6 +24,10 @@ pkgbase = caffe2-git depends = blas depends = lapack depends = gflags + depends = numactl + depends = intel-mkl + depends = opencv + depends = libuv optdepends = python-flask optdepends = graphviz optdepends = python-hypothesis @@ -60,7 +66,7 @@ pkgbase = caffe2-git source = git+https://github.com/benjaminp/six source = git+https://github.com/onnx/onnx source = git+https://github.com/onnx/onnx-tensorrt - source = sleef-zdevito::git+https://github.com/zdevito/sleef + source = git+https://github.com/shibatch/sleef source = git+https://github.com/intel/ideep source = git+https://github.com/NVIDIA/nccl.git source = git+https://github.com/google/gemmlowp @@ -69,6 +75,15 @@ pkgbase = caffe2-git source = git+https://github.com/pytorch/fbgemm source = git+https://github.com/houseroad/foxi source = git+https://github.com/01org/tbb + source = git+https://github.com/facebookincubator/fbjni + source = git+https://github.com/google/XNNPACK + source = git+https://github.com/fmtlib/fmt + source = git+https://github.com/pytorch/tensorpipe + source = git+https://github.com/asmjit/asmjit.git + source = git+https://github.com/01org/mkl-dnn.git + source = git+https://github.com/emil-e/rapidcheck.git + source = 010-caffe2-fix-include-system-path.patch + source = 020-caffe2-use-system-libuv.patch sha256sums = SKIP sha256sums = SKIP sha256sums = SKIP @@ -99,6 +114,15 @@ pkgbase = caffe2-git sha256sums = SKIP sha256sums = SKIP sha256sums = SKIP + sha256sums = SKIP + sha256sums = SKIP + sha256sums = SKIP + sha256sums = SKIP + sha256sums = SKIP + sha256sums = SKIP + sha256sums = SKIP + sha256sums = 85cee13362fb5c7b0e9b087a78a35afa82ae3e2e644be4a89f6e5261ed4e15ef + sha256sums = 2ca3a48a563d2523740609bd3fa83de2e325291ab58ebf6563b540302b074753 pkgname = caffe2-git pkgdesc = A new lightweight, modular, and scalable deep learning framework (cpu only, git version) @@ -113,6 +137,7 @@ pkgname = caffe2-cuda-git pkgdesc = A new lightweight, modular, and scalable deep learning framework (with cuda support, git version) depends = google-glog depends = protobuf + depends = openmp depends = python depends = python-numpy depends = python-protobuf @@ -120,6 +145,10 @@ pkgname = caffe2-cuda-git depends = blas depends = lapack depends = gflags + depends = numactl + depends = intel-mkl + depends = opencv + depends = libuv depends = cuda depends = cudnn depends = nccl diff --git a/010-caffe2-fix-include-system-path.patch b/010-caffe2-fix-include-system-path.patch new file mode 100644 index 000000000000..094e7366ef39 --- /dev/null +++ b/010-caffe2-fix-include-system-path.patch @@ -0,0 +1,11 @@ +--- a/torch/utils/cpp_extension.py ++++ b/torch/utils/cpp_extension.py +@@ -1574,7 +1574,7 @@ def _write_ninja_file_to_build_library(p + common_cflags = ['-DTORCH_EXTENSION_NAME={}'.format(name)] + common_cflags.append('-DTORCH_API_INCLUDE_EXTENSION_H') + common_cflags += ['-I{}'.format(include) for include in user_includes] +- common_cflags += ['-isystem {}'.format(include) for include in system_includes] ++ common_cflags += ['-I{}'.format(include) for include in system_includes] + + common_cflags += ['-D_GLIBCXX_USE_CXX11_ABI=' + str(int(torch._C._GLIBCXX_USE_CXX11_ABI))] + diff --git a/020-caffe2-use-system-libuv.patch b/020-caffe2-use-system-libuv.patch new file mode 100644 index 000000000000..d55cb0af30af --- /dev/null +++ b/020-caffe2-use-system-libuv.patch @@ -0,0 +1,22 @@ +--- a/cmake/Dependencies.cmake ++++ b/cmake/Dependencies.cmake +@@ -1312,7 +1312,7 @@ if(USE_DISTRIBUTED AND USE_TENSORPIPE) + set(__BUILD_TESTING ${BUILD_TESTING}) + set(BUILD_TESTING OFF) + set(TP_BUILD_PYTHON OFF) +- set(TP_BUILD_LIBUV ON) ++ set(TP_BUILD_LIBUV OFF) + + add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/tensorpipe) + +--- a/third_party/tensorpipe/cmake/pytorch.cmake ++++ b/third_party/tensorpipe/cmake/pytorch.cmake +@@ -138,7 +138,7 @@ endif() + + add_library(tensorpipe ${TENSORPIPE_SRC}) + +-set(TP_BUILD_LIBUV ON) ++set(TP_BUILD_LIBUV OFF) + find_package(uv REQUIRED) + target_link_libraries(tensorpipe PRIVATE uv::uv) + @@ -1,21 +1,25 @@ # Maintainer : Daniel Bermond <dbermond@archlinux.org> +# select the desired cuda architecture(s) +# default is to build for all (takes a very long time to build) +_cuda_archs='5.2;5.3;6.0;6.0+PTX;6.1;6.1+PTX;6.2;6.2+PTX;7.0;7.0+PTX;7.2;7.2+PTX;7.5;7.5+PTX;8.0;8.0+PTX' + pkgbase=caffe2-git pkgname=('caffe2-git' 'caffe2-cuda-git') -pkgver=0.8.2.r18962.gd8de69d621 -pkgrel=2 +pkgver=0.8.2.r28489.gc5b4f60fc24 +pkgrel=1 epoch=1 pkgdesc='A new lightweight, modular, and scalable deep learning framework' arch=('x86_64') url='https://caffe2.ai/' license=('BSD') -depends=('google-glog' 'protobuf' 'python' 'python-numpy' 'python-protobuf' - 'python-yaml' 'blas' 'lapack' 'gflags') +depends=('google-glog' 'protobuf' 'openmp' 'python' 'python-numpy' 'python-protobuf' + 'python-yaml' 'blas' 'lapack' 'gflags' 'numactl' 'intel-mkl' 'opencv' 'libuv') optdepends=('python-flask' 'graphviz' 'python-hypothesis' 'python-matplotlib' 'python-pydot' 'python-nvd3' 'python-yaml' 'python-requests' 'python-scikit-image' 'python-scipy' 'python-setuptools' 'python-future' 'python-tornado' 'python-six' 'python-lmdb') -makedepends=('git' 'cmake' 'gtest' 'snappy' 'cuda' 'cudnn' 'nccl') +makedepends=('git' 'cmake' 'gtest' 'snappy' 'cuda' 'cudnn' 'nccl' 'pybind11') conflicts=('python-pytorch') options=('!emptydirs') source=("git+https://github.com/pytorch/pytorch.git" @@ -39,7 +43,7 @@ source=("git+https://github.com/pytorch/pytorch.git" 'git+https://github.com/benjaminp/six' 'git+https://github.com/onnx/onnx' 'git+https://github.com/onnx/onnx-tensorrt' - 'sleef-zdevito'::'git+https://github.com/zdevito/sleef' + 'git+https://github.com/shibatch/sleef' 'git+https://github.com/intel/ideep' 'git+https://github.com/NVIDIA/nccl.git' 'git+https://github.com/google/gemmlowp' @@ -47,7 +51,16 @@ source=("git+https://github.com/pytorch/pytorch.git" 'git+https://github.com/intel/ARM_NEON_2_x86_SSE' 'git+https://github.com/pytorch/fbgemm' 'git+https://github.com/houseroad/foxi' - 'git+https://github.com/01org/tbb') + 'git+https://github.com/01org/tbb' + 'git+https://github.com/facebookincubator/fbjni' + 'git+https://github.com/google/XNNPACK' + 'git+https://github.com/fmtlib/fmt' + 'git+https://github.com/pytorch/tensorpipe' + 'git+https://github.com/asmjit/asmjit.git' + 'git+https://github.com/01org/mkl-dnn.git' + 'git+https://github.com/emil-e/rapidcheck.git' + '010-caffe2-fix-include-system-path.patch' + '020-caffe2-use-system-libuv.patch') sha256sums=('SKIP' 'SKIP' 'SKIP' @@ -77,14 +90,19 @@ sha256sums=('SKIP' 'SKIP' 'SKIP' 'SKIP' - 'SKIP') + 'SKIP' + 'SKIP' + 'SKIP' + 'SKIP' + 'SKIP' + 'SKIP' + 'SKIP' + 'SKIP' + '85cee13362fb5c7b0e9b087a78a35afa82ae3e2e644be4a89f6e5261ed4e15ef' + '2ca3a48a563d2523740609bd3fa83de2e325291ab58ebf6563b540302b074753') prepare() { - mkdir -p pytorch/{build-cpu-only,build-cuda} - - cd pytorch - - # submodules which clone dir coincide with the submodule name + # submodules which clone dir coincides with the submodule name local _submodule_list=('pybind11' 'cub' 'googletest' @@ -101,45 +119,64 @@ prepare() { 'QNNPACK' 'fbgemm' 'foxi' - 'tbb') + 'tbb' + 'XNNPACK' + 'fmt' + 'tensorpipe') + + git -C pytorch submodule init + local _submodule for _submodule in "${_submodule_list[@]}" do - git config --local "submodule.third_party/${_submodule}.url" "${srcdir}/${_submodule}" + git -C pytorch config --local "submodule.third_party/${_submodule}.url" "${srcdir}/${_submodule}" done # submodules which clone dir does not coincide with the submodule name - git config --local "submodule.third_party/eigen.url" "${srcdir}/eigen-git-mirror" - git config --local "submodule.third_party/protobuf.url" "${srcdir}/protobuf-protocolbuffers" - git config --local "submodule.third_party/NNPACK_deps/pthreadpool.url" "${srcdir}/pthreadpool" - git config --local "submodule.third_party/NNPACK_deps/FXdiv.url" "${srcdir}/FXdiv" - git config --local "submodule.third_party/NNPACK_deps/FP16.url" "${srcdir}/FP16" - git config --local "submodule.third_party/NNPACK_deps/psimd.url" "${srcdir}/psimd" - git config --local "submodule.third_party/cpuinfo.url" "${srcdir}/cpuinfo-pytorch" - git config --local "submodule.third_party/python-enum.url" "${srcdir}/enum34" - git config --local "submodule.third_party/python-peachpy.url" "${srcdir}/PeachPy" - git config --local "submodule.third_party/python-six.url" "${srcdir}/six" - git config --local "submodule.third_party/sleef.url" "${srcdir}/sleef-zdevito" - git config --local "submodule.third_party/neon2sse.url" "${srcdir}/ARM_NEON_2_x86_SSE" - git config --local "submodule.third_party/nccl/nccl.url" "${srcdir}/nccl" - git config --local "submodule.third_party/gemmlowp/gemmlowp.url" "${srcdir}/gemmlowp" + git -C pytorch config --local submodule.third_party/eigen.url "${srcdir}/eigen-git-mirror" + git -C pytorch config --local submodule.third_party/protobuf.url "${srcdir}/protobuf-protocolbuffers" + git -C pytorch config --local submodule.third_party/NNPACK_deps/pthreadpool.url "${srcdir}/pthreadpool" + git -C pytorch config --local submodule.third_party/NNPACK_deps/FXdiv.url "${srcdir}/FXdiv" + git -C pytorch config --local submodule.third_party/NNPACK_deps/FP16.url "${srcdir}/FP16" + git -C pytorch config --local submodule.third_party/NNPACK_deps/psimd.url "${srcdir}/psimd" + git -C pytorch config --local submodule.third_party/python-enum.url "${srcdir}/enum34" + git -C pytorch config --local submodule.third_party/python-peachpy.url "${srcdir}/PeachPy" + git -C pytorch config --local submodule.third_party/python-six.url "${srcdir}/six" + git -C pytorch config --local submodule.third_party/neon2sse.url "${srcdir}/ARM_NEON_2_x86_SSE" + git -C pytorch config --local submodule.third_party/nccl/nccl.url "${srcdir}/nccl" + git -C pytorch config --local submodule.third_party/gemmlowp/gemmlowp.url "${srcdir}/gemmlowp" + git -C pytorch config --local submodule.android/libs/fbjni.url "${srcdir}/fbjni" # special case (upstream uses third-party instead of third_party) - git config --local 'submodule.third-party/cpuinfo.url' "${srcdir}/cpuinfo-pytorch" + git -C pytorch config --local submodule.third-party/cpuinfo.url "${srcdir}/cpuinfo-pytorch" + + git -C pytorch submodule update + + # fbgemm submodules + git -C pytorch/third_party/fbgemm submodule init + git -C pytorch/third_party/fbgemm config --local submodule.third_party/asmjit.url "${srcdir}/asmjit" + git -C pytorch/third_party/fbgemm config --local submodule.third_party/cpuinfo.url "${srcdir}/cpuinfo-pytorch" + git -C pytorch/third_party/fbgemm config --local submodule.third_party/googletest.url "${srcdir}/googletest" + git -C pytorch/third_party/fbgemm submodule update - git submodule update + # ideep submodules + git -C pytorch/third_party/ideep submodule init + git -C pytorch/third_party/ideep config --local submodule.mkl-dnn.url "${srcdir}/mkl-dnn" + git -C pytorch/third_party/ideep config --local submodule.tests/googletest.url "${srcdir}/googletest" + git -C pytorch/third_party/ideep config --local submodule.tests/rapidcheck.url "${srcdir}/rapidcheck" + git -C pytorch/third_party/ideep submodule update + + patch -d pytorch -Np1 -i "${srcdir}/010-caffe2-fix-include-system-path.patch" + patch -d pytorch -Np1 -i "${srcdir}/020-caffe2-use-system-libuv.patch" } pkgver() { - cd pytorch - local _version local _revision local _shorthash - - _version="$(head -n1 caffe2/VERSION_NUMBER)" - _revision="$( git rev-list --count HEAD)" - _shorthash="$(git rev-parse --short HEAD)" + _version="$(head -n1 pytorch/caffe2/VERSION_NUMBER)" + _revision="$( git -C pytorch rev-list --count HEAD)" + _shorthash="$(git -C pytorch rev-parse --short HEAD)" printf '%s.r%s.g%s' "$_version" "$_revision" "$_shorthash" } @@ -160,54 +197,57 @@ build() { '-DC_AVX_FOUND:BOOL=FALSE' '-DUSE_GFLAGS:BOOL=ON' '-DUSE_GLOG:BOOL=ON' - '-DUSE_MKLDNN:BOOL=OFF' - '-DUSE_MPI:BOOL=OFF' - '-DUSE_NUMA:BOOL=OFF' - '-DUSE_OPENCV:BOOL=OFF' + '-DUSE_OPENCV:BOOL=ON' '-Wno-dev') # caffe2-cuda - cd pytorch/build-cuda - cmake \ + cmake -B build-cuda -S pytorch \ "${_common_opts[@]}" \ -DCMAKE_CXX_COMPILER:FILEPATH='/opt/cuda/bin/g++' \ -DCMAKE_C_COMPILER:FILEPATH='/opt/cuda/bin/gcc' \ -DCUDA_HOST_COMPILER:FILEPATH='/opt/cuda/bin/gcc' \ -DCUDA_NVCC_FLAGS:STRING='-Xfatbin -compress-all' \ - -DTORCH_CUDA_ARCH_LIST='3.0;3.2;3.5;3.7;5.0;5.2;5.3;6.0;6.1;6.2;7.0;7.2;7.5' \ + -DTORCH_CUDA_ARCH_LIST="$_cuda_archs" \ -DUSE_CUDA:BOOL='ON' \ -DUSE_CUDNN:BOOL='ON' \ -DUSE_METAL:BOOL='OFF' \ -DUSE_NCCL:BOOL='ON' \ -DUSE_SYSTEM_NCCL:BOOL='ON' ## fix: avoid a second compilation in package() - sed -i 's/^preinstall:[[:space:]]all/preinstall:/' Makefile - make + sed -i 's/^preinstall:[[:space:]]all/preinstall:/' build-cuda/Makefile + make -C build-cuda # caffe2 (cpu only, without cuda support) - cd "${srcdir}/pytorch/build-cpu-only" - cmake "${_common_opts[@]}" -DUSE_CUDA:BOOL='OFF' -DUSE_NCCL:BOOL='OFF' - make + cmake -B build-cpu-only -S pytorch "${_common_opts[@]}" -DUSE_CUDA:BOOL='OFF' -DUSE_NCCL:BOOL='OFF' + make -C build-cpu-only } _package_common() { - make DESTDIR="$pkgdir" install + make -C "$1" DESTDIR="$pkgdir" install # remove unneeded files - rm "$pkgdir"/usr/include/*.h - rm "$pkgdir"/usr/lib/{*.a,lib{foxi*,onnxifi*}.so*} - rm -r "$pkgdir"/usr/share/{ATen,cmake/{ATen,Gloo,ONNX}} - rm -r "$pkgdir"/torch + rm -f "$pkgdir"/usr/include/*.h* + rm -rf "$pkgdir"/usr/share/ATen local _dir + local _file while read -r -d '' _dir do - rm -rf "$_dir" - done < <(find "${pkgdir}/usr/include" -mindepth 1 -maxdepth 1 -type d ! -name 'caffe*' ! -name 'c10' -print0) + rm -r "$_dir" + done < <(find "${pkgdir}/usr"/{include,share/doc} -mindepth 1 -maxdepth 1 -type d ! -name 'caffe*' ! -name 'c10*' \ + ! -name 'TH*' ! -name 'torch' ! -name 'tensorpipe' -print0) + while read -r -d '' _dir + do + rm -r "$_dir" + done < <(find "${pkgdir}/usr"/{lib,share}/cmake -mindepth 1 -maxdepth 1 -type d ! -name 'Caffe*' ! -name 'Torch' \ + ! -name 'tensorpipe' -print0) + while read -r -d '' _file + do + rm "$_file" + done < <(find -L "${pkgdir}/usr/lib" -mindepth 1 -maxdepth 1 -type f ! -name '*c10*.so*' ! -name '*caffe*' \ + ! -name '*shm*' ! -name '*torch*.so*' ! -name '*tensorpipe*' -print0) # license - cd "${srcdir}/pytorch" - install -D -m644 LICENSE -t "${pkgdir}/usr/share/licenses/${pkgname}" - install -D -m644 NOTICE -t "${pkgdir}/usr/share/licenses/${pkgname}" + install -D -m644 pytorch/{LICENSE,NOTICE} -t "${pkgdir}/usr/share/licenses/${pkgname}" } package_caffe2-cuda-git() { @@ -216,9 +256,7 @@ package_caffe2-cuda-git() { provides=('caffe2' 'caffe2-git' 'caffe2-cuda') conflicts+=('caffe2') - cd pytorch/build-cuda - - _package_common + _package_common 'build-cuda' } package_caffe2-git() { @@ -227,7 +265,5 @@ package_caffe2-git() { conflicts+=('caffe2' 'caffe2-cpu-git') replaces=('caffe2-cpu-git') - cd pytorch/build-cpu-only - - _package_common + _package_common 'build-cpu-only' } |