diff options
Diffstat (limited to 'PKGBUILD')
-rw-r--r-- | PKGBUILD | 166 |
1 files changed, 101 insertions, 65 deletions
@@ -1,21 +1,25 @@ # Maintainer : Daniel Bermond <dbermond@archlinux.org> +# select the desired cuda architecture(s) +# default is to build for all (takes a very long time to build) +_cuda_archs='5.2;5.3;6.0;6.0+PTX;6.1;6.1+PTX;6.2;6.2+PTX;7.0;7.0+PTX;7.2;7.2+PTX;7.5;7.5+PTX;8.0;8.0+PTX' + pkgbase=caffe2-git pkgname=('caffe2-git' 'caffe2-cuda-git') -pkgver=0.8.2.r18962.gd8de69d621 -pkgrel=2 +pkgver=0.8.2.r28489.gc5b4f60fc24 +pkgrel=1 epoch=1 pkgdesc='A new lightweight, modular, and scalable deep learning framework' arch=('x86_64') url='https://caffe2.ai/' license=('BSD') -depends=('google-glog' 'protobuf' 'python' 'python-numpy' 'python-protobuf' - 'python-yaml' 'blas' 'lapack' 'gflags') +depends=('google-glog' 'protobuf' 'openmp' 'python' 'python-numpy' 'python-protobuf' + 'python-yaml' 'blas' 'lapack' 'gflags' 'numactl' 'intel-mkl' 'opencv' 'libuv') optdepends=('python-flask' 'graphviz' 'python-hypothesis' 'python-matplotlib' 'python-pydot' 'python-nvd3' 'python-yaml' 'python-requests' 'python-scikit-image' 'python-scipy' 'python-setuptools' 'python-future' 'python-tornado' 'python-six' 'python-lmdb') -makedepends=('git' 'cmake' 'gtest' 'snappy' 'cuda' 'cudnn' 'nccl') +makedepends=('git' 'cmake' 'gtest' 'snappy' 'cuda' 'cudnn' 'nccl' 'pybind11') conflicts=('python-pytorch') options=('!emptydirs') source=("git+https://github.com/pytorch/pytorch.git" @@ -39,7 +43,7 @@ source=("git+https://github.com/pytorch/pytorch.git" 'git+https://github.com/benjaminp/six' 'git+https://github.com/onnx/onnx' 'git+https://github.com/onnx/onnx-tensorrt' - 'sleef-zdevito'::'git+https://github.com/zdevito/sleef' + 'git+https://github.com/shibatch/sleef' 'git+https://github.com/intel/ideep' 'git+https://github.com/NVIDIA/nccl.git' 'git+https://github.com/google/gemmlowp' @@ -47,7 +51,16 @@ source=("git+https://github.com/pytorch/pytorch.git" 'git+https://github.com/intel/ARM_NEON_2_x86_SSE' 'git+https://github.com/pytorch/fbgemm' 'git+https://github.com/houseroad/foxi' - 'git+https://github.com/01org/tbb') + 'git+https://github.com/01org/tbb' + 'git+https://github.com/facebookincubator/fbjni' + 'git+https://github.com/google/XNNPACK' + 'git+https://github.com/fmtlib/fmt' + 'git+https://github.com/pytorch/tensorpipe' + 'git+https://github.com/asmjit/asmjit.git' + 'git+https://github.com/01org/mkl-dnn.git' + 'git+https://github.com/emil-e/rapidcheck.git' + '010-caffe2-fix-include-system-path.patch' + '020-caffe2-use-system-libuv.patch') sha256sums=('SKIP' 'SKIP' 'SKIP' @@ -77,14 +90,19 @@ sha256sums=('SKIP' 'SKIP' 'SKIP' 'SKIP' - 'SKIP') + 'SKIP' + 'SKIP' + 'SKIP' + 'SKIP' + 'SKIP' + 'SKIP' + 'SKIP' + 'SKIP' + '85cee13362fb5c7b0e9b087a78a35afa82ae3e2e644be4a89f6e5261ed4e15ef' + '2ca3a48a563d2523740609bd3fa83de2e325291ab58ebf6563b540302b074753') prepare() { - mkdir -p pytorch/{build-cpu-only,build-cuda} - - cd pytorch - - # submodules which clone dir coincide with the submodule name + # submodules which clone dir coincides with the submodule name local _submodule_list=('pybind11' 'cub' 'googletest' @@ -101,45 +119,64 @@ prepare() { 'QNNPACK' 'fbgemm' 'foxi' - 'tbb') + 'tbb' + 'XNNPACK' + 'fmt' + 'tensorpipe') + + git -C pytorch submodule init + local _submodule for _submodule in "${_submodule_list[@]}" do - git config --local "submodule.third_party/${_submodule}.url" "${srcdir}/${_submodule}" + git -C pytorch config --local "submodule.third_party/${_submodule}.url" "${srcdir}/${_submodule}" done # submodules which clone dir does not coincide with the submodule name - git config --local "submodule.third_party/eigen.url" "${srcdir}/eigen-git-mirror" - git config --local "submodule.third_party/protobuf.url" "${srcdir}/protobuf-protocolbuffers" - git config --local "submodule.third_party/NNPACK_deps/pthreadpool.url" "${srcdir}/pthreadpool" - git config --local "submodule.third_party/NNPACK_deps/FXdiv.url" "${srcdir}/FXdiv" - git config --local "submodule.third_party/NNPACK_deps/FP16.url" "${srcdir}/FP16" - git config --local "submodule.third_party/NNPACK_deps/psimd.url" "${srcdir}/psimd" - git config --local "submodule.third_party/cpuinfo.url" "${srcdir}/cpuinfo-pytorch" - git config --local "submodule.third_party/python-enum.url" "${srcdir}/enum34" - git config --local "submodule.third_party/python-peachpy.url" "${srcdir}/PeachPy" - git config --local "submodule.third_party/python-six.url" "${srcdir}/six" - git config --local "submodule.third_party/sleef.url" "${srcdir}/sleef-zdevito" - git config --local "submodule.third_party/neon2sse.url" "${srcdir}/ARM_NEON_2_x86_SSE" - git config --local "submodule.third_party/nccl/nccl.url" "${srcdir}/nccl" - git config --local "submodule.third_party/gemmlowp/gemmlowp.url" "${srcdir}/gemmlowp" + git -C pytorch config --local submodule.third_party/eigen.url "${srcdir}/eigen-git-mirror" + git -C pytorch config --local submodule.third_party/protobuf.url "${srcdir}/protobuf-protocolbuffers" + git -C pytorch config --local submodule.third_party/NNPACK_deps/pthreadpool.url "${srcdir}/pthreadpool" + git -C pytorch config --local submodule.third_party/NNPACK_deps/FXdiv.url "${srcdir}/FXdiv" + git -C pytorch config --local submodule.third_party/NNPACK_deps/FP16.url "${srcdir}/FP16" + git -C pytorch config --local submodule.third_party/NNPACK_deps/psimd.url "${srcdir}/psimd" + git -C pytorch config --local submodule.third_party/python-enum.url "${srcdir}/enum34" + git -C pytorch config --local submodule.third_party/python-peachpy.url "${srcdir}/PeachPy" + git -C pytorch config --local submodule.third_party/python-six.url "${srcdir}/six" + git -C pytorch config --local submodule.third_party/neon2sse.url "${srcdir}/ARM_NEON_2_x86_SSE" + git -C pytorch config --local submodule.third_party/nccl/nccl.url "${srcdir}/nccl" + git -C pytorch config --local submodule.third_party/gemmlowp/gemmlowp.url "${srcdir}/gemmlowp" + git -C pytorch config --local submodule.android/libs/fbjni.url "${srcdir}/fbjni" # special case (upstream uses third-party instead of third_party) - git config --local 'submodule.third-party/cpuinfo.url' "${srcdir}/cpuinfo-pytorch" + git -C pytorch config --local submodule.third-party/cpuinfo.url "${srcdir}/cpuinfo-pytorch" + + git -C pytorch submodule update + + # fbgemm submodules + git -C pytorch/third_party/fbgemm submodule init + git -C pytorch/third_party/fbgemm config --local submodule.third_party/asmjit.url "${srcdir}/asmjit" + git -C pytorch/third_party/fbgemm config --local submodule.third_party/cpuinfo.url "${srcdir}/cpuinfo-pytorch" + git -C pytorch/third_party/fbgemm config --local submodule.third_party/googletest.url "${srcdir}/googletest" + git -C pytorch/third_party/fbgemm submodule update - git submodule update + # ideep submodules + git -C pytorch/third_party/ideep submodule init + git -C pytorch/third_party/ideep config --local submodule.mkl-dnn.url "${srcdir}/mkl-dnn" + git -C pytorch/third_party/ideep config --local submodule.tests/googletest.url "${srcdir}/googletest" + git -C pytorch/third_party/ideep config --local submodule.tests/rapidcheck.url "${srcdir}/rapidcheck" + git -C pytorch/third_party/ideep submodule update + + patch -d pytorch -Np1 -i "${srcdir}/010-caffe2-fix-include-system-path.patch" + patch -d pytorch -Np1 -i "${srcdir}/020-caffe2-use-system-libuv.patch" } pkgver() { - cd pytorch - local _version local _revision local _shorthash - - _version="$(head -n1 caffe2/VERSION_NUMBER)" - _revision="$( git rev-list --count HEAD)" - _shorthash="$(git rev-parse --short HEAD)" + _version="$(head -n1 pytorch/caffe2/VERSION_NUMBER)" + _revision="$( git -C pytorch rev-list --count HEAD)" + _shorthash="$(git -C pytorch rev-parse --short HEAD)" printf '%s.r%s.g%s' "$_version" "$_revision" "$_shorthash" } @@ -160,54 +197,57 @@ build() { '-DC_AVX_FOUND:BOOL=FALSE' '-DUSE_GFLAGS:BOOL=ON' '-DUSE_GLOG:BOOL=ON' - '-DUSE_MKLDNN:BOOL=OFF' - '-DUSE_MPI:BOOL=OFF' - '-DUSE_NUMA:BOOL=OFF' - '-DUSE_OPENCV:BOOL=OFF' + '-DUSE_OPENCV:BOOL=ON' '-Wno-dev') # caffe2-cuda - cd pytorch/build-cuda - cmake \ + cmake -B build-cuda -S pytorch \ "${_common_opts[@]}" \ -DCMAKE_CXX_COMPILER:FILEPATH='/opt/cuda/bin/g++' \ -DCMAKE_C_COMPILER:FILEPATH='/opt/cuda/bin/gcc' \ -DCUDA_HOST_COMPILER:FILEPATH='/opt/cuda/bin/gcc' \ -DCUDA_NVCC_FLAGS:STRING='-Xfatbin -compress-all' \ - -DTORCH_CUDA_ARCH_LIST='3.0;3.2;3.5;3.7;5.0;5.2;5.3;6.0;6.1;6.2;7.0;7.2;7.5' \ + -DTORCH_CUDA_ARCH_LIST="$_cuda_archs" \ -DUSE_CUDA:BOOL='ON' \ -DUSE_CUDNN:BOOL='ON' \ -DUSE_METAL:BOOL='OFF' \ -DUSE_NCCL:BOOL='ON' \ -DUSE_SYSTEM_NCCL:BOOL='ON' ## fix: avoid a second compilation in package() - sed -i 's/^preinstall:[[:space:]]all/preinstall:/' Makefile - make + sed -i 's/^preinstall:[[:space:]]all/preinstall:/' build-cuda/Makefile + make -C build-cuda # caffe2 (cpu only, without cuda support) - cd "${srcdir}/pytorch/build-cpu-only" - cmake "${_common_opts[@]}" -DUSE_CUDA:BOOL='OFF' -DUSE_NCCL:BOOL='OFF' - make + cmake -B build-cpu-only -S pytorch "${_common_opts[@]}" -DUSE_CUDA:BOOL='OFF' -DUSE_NCCL:BOOL='OFF' + make -C build-cpu-only } _package_common() { - make DESTDIR="$pkgdir" install + make -C "$1" DESTDIR="$pkgdir" install # remove unneeded files - rm "$pkgdir"/usr/include/*.h - rm "$pkgdir"/usr/lib/{*.a,lib{foxi*,onnxifi*}.so*} - rm -r "$pkgdir"/usr/share/{ATen,cmake/{ATen,Gloo,ONNX}} - rm -r "$pkgdir"/torch + rm -f "$pkgdir"/usr/include/*.h* + rm -rf "$pkgdir"/usr/share/ATen local _dir + local _file while read -r -d '' _dir do - rm -rf "$_dir" - done < <(find "${pkgdir}/usr/include" -mindepth 1 -maxdepth 1 -type d ! -name 'caffe*' ! -name 'c10' -print0) + rm -r "$_dir" + done < <(find "${pkgdir}/usr"/{include,share/doc} -mindepth 1 -maxdepth 1 -type d ! -name 'caffe*' ! -name 'c10*' \ + ! -name 'TH*' ! -name 'torch' ! -name 'tensorpipe' -print0) + while read -r -d '' _dir + do + rm -r "$_dir" + done < <(find "${pkgdir}/usr"/{lib,share}/cmake -mindepth 1 -maxdepth 1 -type d ! -name 'Caffe*' ! -name 'Torch' \ + ! -name 'tensorpipe' -print0) + while read -r -d '' _file + do + rm "$_file" + done < <(find -L "${pkgdir}/usr/lib" -mindepth 1 -maxdepth 1 -type f ! -name '*c10*.so*' ! -name '*caffe*' \ + ! -name '*shm*' ! -name '*torch*.so*' ! -name '*tensorpipe*' -print0) # license - cd "${srcdir}/pytorch" - install -D -m644 LICENSE -t "${pkgdir}/usr/share/licenses/${pkgname}" - install -D -m644 NOTICE -t "${pkgdir}/usr/share/licenses/${pkgname}" + install -D -m644 pytorch/{LICENSE,NOTICE} -t "${pkgdir}/usr/share/licenses/${pkgname}" } package_caffe2-cuda-git() { @@ -216,9 +256,7 @@ package_caffe2-cuda-git() { provides=('caffe2' 'caffe2-git' 'caffe2-cuda') conflicts+=('caffe2') - cd pytorch/build-cuda - - _package_common + _package_common 'build-cuda' } package_caffe2-git() { @@ -227,7 +265,5 @@ package_caffe2-git() { conflicts+=('caffe2' 'caffe2-cpu-git') replaces=('caffe2-cpu-git') - cd pytorch/build-cpu-only - - _package_common + _package_common 'build-cpu-only' } |