build() and package() improvements. Update submodules and options.

2 files changed, 46 insertions, 27 deletions

diff --git a/.SRCINFO b/.SRCINFO
index 10ccb41df7e3..c2c540f643df 100644
--- a/.SRCINFO
+++ b/.SRCINFO
@@ -1,8 +1,8 @@
 pkgbase = caffe2-cuda-git
 	pkgdesc = A new lightweight, modular, and scalable deep learning framework (with cuda, git version)
-	pkgver = 0.8.2.r14124.gb4d0dc77be
+	pkgver = 0.8.2.r14502.g7978ba45ba
 	pkgrel = 1
-	url = http://caffe2.ai/
+	url = https://caffe2.ai/
 	arch = x86_64
 	license = BSD
 	makedepends = git
@@ -54,12 +54,10 @@ pkgbase = caffe2-cuda-git
 	conflicts = caffe2
 	options = !emptydirs
 	source = pytorch-git::git+https://github.com/pytorch/pytorch.git
-	source = caffe2-submodule-catch::git+https://github.com/catchorg/Catch2.git
 	source = caffe2-submodule-pybind11::git+https://github.com/pybind/pybind11.git
 	source = caffe2-submodule-cub::git+https://github.com/NVlabs/cub.git
 	source = caffe2-submodule-eigen::git+https://github.com/eigenteam/eigen-git-mirror.git
 	source = caffe2-submodule-googletest::git+https://github.com/google/googletest.git
-	source = caffe2-submodule-nervanagpu::git+https://github.com/NervanaSystems/nervanagpu.git
 	source = caffe2-submodule-benchmark::git+https://github.com/google/benchmark.git
 	source = caffe2-submodule-protobuf::git+https://github.com/google/protobuf.git
 	source = caffe2-submodule-ios-cmake::git+https://github.com/Yangqing/ios-cmake.git
@@ -76,10 +74,18 @@ pkgbase = caffe2-cuda-git
 	source = caffe2-submodule-python-six::git+https://github.com/benjaminp/six.git
 	source = caffe2-submodule-ComputeLibrary::git+https://github.com/ARM-software/ComputeLibrary.git
 	source = caffe2-submodule-onnx::git+https://github.com/onnx/onnx.git
-	source = caffe2-submodule-cereal::git+https://github.com/USCiLab/cereal
 	source = caffe2-submodule-onnx-tensorrt::git+https://github.com/onnx/onnx-tensorrt
 	source = caffe2-submodule-sleef::git+https://github.com/shibatch/sleef
 	source = caffe2-submodule-ideep::git+https://github.com/intel/ideep
+	source = caffe2-submodule-nccl::git+https://github.com/NVIDIA/nccl
+	source = caffe2-submodule-gemmlowp::git+https://github.com/google/gemmlowp.git
+	source = caffe2-submodule-QNNPACK::git+https://github.com/pytorch/QNNPACK
+	source = caffe2-submodule-neon2sse::git+https://github.com/intel/ARM_NEON_2_x86_SSE.git
+	source = caffe2-submodule-fbgemm::git+https://github.com/pytorch/fbgemm
+	source = asmjit-git::git+https://github.com/asmjit/asmjit.git
+	sha256sums = SKIP
+	sha256sums = SKIP
+	sha256sums = SKIP
 	sha256sums = SKIP
 	sha256sums = SKIP
 	sha256sums = SKIP
diff --git a/PKGBUILD b/PKGBUILD
index 1bee6e0912b6..4719ffd9a507 100644
--- a/PKGBUILD
+++ b/PKGBUILD
@@ -1,11 +1,11 @@
 # Maintainer : Daniel Bermond < gmail-com: danielbermond >
 
 pkgname=caffe2-cuda-git
-pkgver=0.8.2.r14124.gb4d0dc77be
+pkgver=0.8.2.r14502.g7978ba45ba
 pkgrel=1
 pkgdesc='A new lightweight, modular, and scalable deep learning framework (with cuda, git version)'
 arch=('x86_64')
-url='http://caffe2.ai/'
+url='https://caffe2.ai/'
 license=('BSD')
 depends=(
     # official repositories:
@@ -35,12 +35,10 @@ source=(
     # main source:
         'pytorch-git'::'git+https://github.com/pytorch/pytorch.git'
     # git submodules:
-        'caffe2-submodule-catch'::'git+https://github.com/catchorg/Catch2.git'
         'caffe2-submodule-pybind11'::'git+https://github.com/pybind/pybind11.git'
         'caffe2-submodule-cub'::'git+https://github.com/NVlabs/cub.git'
         'caffe2-submodule-eigen'::'git+https://github.com/eigenteam/eigen-git-mirror.git'
         'caffe2-submodule-googletest'::'git+https://github.com/google/googletest.git'
-        'caffe2-submodule-nervanagpu'::'git+https://github.com/NervanaSystems/nervanagpu.git'
         'caffe2-submodule-benchmark'::'git+https://github.com/google/benchmark.git'
         'caffe2-submodule-protobuf'::'git+https://github.com/google/protobuf.git'
         'caffe2-submodule-ios-cmake'::'git+https://github.com/Yangqing/ios-cmake.git'
@@ -57,10 +55,16 @@ source=(
         'caffe2-submodule-python-six'::'git+https://github.com/benjaminp/six.git'
         'caffe2-submodule-ComputeLibrary'::'git+https://github.com/ARM-software/ComputeLibrary.git'
         'caffe2-submodule-onnx'::'git+https://github.com/onnx/onnx.git'
-        'caffe2-submodule-cereal'::'git+https://github.com/USCiLab/cereal'
         'caffe2-submodule-onnx-tensorrt'::'git+https://github.com/onnx/onnx-tensorrt'
         'caffe2-submodule-sleef'::'git+https://github.com/shibatch/sleef'
         'caffe2-submodule-ideep'::'git+https://github.com/intel/ideep'
+        'caffe2-submodule-nccl'::'git+https://github.com/NVIDIA/nccl'
+        'caffe2-submodule-gemmlowp'::'git+https://github.com/google/gemmlowp.git'
+        'caffe2-submodule-QNNPACK'::'git+https://github.com/pytorch/QNNPACK'
+        'caffe2-submodule-neon2sse'::'git+https://github.com/intel/ARM_NEON_2_x86_SSE.git'
+        'caffe2-submodule-fbgemm'::'git+https://github.com/pytorch/fbgemm'
+    # others:
+        'asmjit-git'::'git+https://github.com/asmjit/asmjit.git'
 )
 sha256sums=('SKIP'
             'SKIP'
@@ -88,6 +92,9 @@ sha256sums=('SKIP'
             'SKIP'
             'SKIP'
             'SKIP'
+            'SKIP'
+            'SKIP'
+            'SKIP'
             'SKIP')
 
 prepare() {
@@ -96,11 +103,11 @@ prepare() {
     local _submodule
     local _submodule_dir
     
-    local _submodule_list="catch pybind11 cub eigen googletest nervanagpu benchmark \
-                           protobuf ios-cmake NNPACK gloo NNPACK_deps/pthreadpool \
-                           NNPACK_deps/FXdiv NNPACK_deps/FP16 NNPACK_deps/psimd zstd \
-                           python-enum python-peachpy python-six ComputeLibrary onnx
-                           cereal onnx-tensorrt sleef ideep"
+    local _submodule_list="pybind11 cub eigen googletest benchmark protobuf ios-cmake \
+                           NNPACK gloo NNPACK_deps/pthreadpool NNPACK_deps/FXdiv \
+                           NNPACK_deps/FP16 NNPACK_deps/psimd zstd python-enum \
+                           python-peachpy python-six ComputeLibrary onnx onnx-tensorrt \
+                           sleef ideep QNNPACK neon2sse fbgemm"
                            
     git submodule init
     
@@ -112,6 +119,14 @@ prepare() {
         git config --local "submodule.third_party/${_submodule}.url" "${srcdir}/${_submodule_dir}"
     done
     
+    _submodule_list='nccl gemmlowp' # upstream names this submodules as nccl/nccl and gemmlowp/gemmlowp
+    
+    for _submodule in $_submodule_list
+    do
+        _submodule_dir="caffe2-submodule-${_submodule}"
+        git config --local "submodule.third_party/${_submodule}/${_submodule}.url" "${srcdir}/${_submodule_dir}"
+    done
+    
     git submodule update
 }
 
@@ -167,6 +182,7 @@ build() {
         -DGLOO_STATIC_OR_SHARED:STRING='STATIC' \
         \
         -DOpenCV_DIR:PATH='/usr/share/OpenCV' \
+        -DASMJIT_SRC_DIR:STRING="${srcdir}/asmjit-git" \
         \
         -DPYTHON_EXECUTABLE:FILEPATH="/usr/bin/python${_pythonver}" \
         -DPYTHON_INCLUDE_DIR:PATH="/usr/include/python${_pythonver}m" \
@@ -177,32 +193,32 @@ build() {
         -DUSE_CUDA:BOOL='ON' \
         -DUSE_CUDNN:BOOL='ON' \
         -DUSE_DISTRIBUTED:BOOL='ON' \
+        -DUSE_FBGEMM:BOOL='ON' \
         -DUSE_FFMPEG:BOOL='ON' \
         -DUSE_GFLAGS:BOOL='ON' \
         -DUSE_GLOG:BOOL='ON' \
         -DUSE_GLOO:BOOL='ON' \
         -DUSE_GLOO_IBVERBS:BOOL='ON' \
         -DUSE_IBVERBS:BOOL='ON' \
-        -DUSE_IDEEP:BOOL='ON' \
         -DUSE_LEVELDB:BOOL='ON' \
         -DUSE_LITE_PROTO:BOOL='OFF' \
         -DUSE_LMDB:BOOL='ON' \
         -DUSE_METAL:BOOL='OFF' \
         -DUSE_MKLDNN:BOOL='OFF' \
-        -DUSE_MKLML:BOOL='OFF' \
         -DUSE_MOBILE_OPENGL:BOOL='OFF' \
         -DUSE_MPI:BOOL='ON' \
         -DUSE_NCCL:BOOL='ON' \
-        -DUSE_NERVANA_GPU:BOOL='OFF' \
         -DUSE_NNAPI:BOOL='OFF' \
         -DUSE_NNPACK:BOOL='ON' \
         -DUSE_NUMA:BOOL='ON' \
+        -DUSE_NUMPY:BOOL='ON' \
         -DUSE_NVRTC:BOOL='ON' \
         -DUSE_OBSERVERS:BOOL='ON' \
         -DUSE_OPENCL:BOOL='OFF' \
         -DUSE_OPENCV:BOOL='OFF' \
         -DUSE_OPENMP:BOOL='ON' \
         -DUSE_PROF:BOOL='OFF' \
+        -DUSE_QNNPACK:BOOL='ON' \
         -DUSE_REDIS:BOOL='ON' \
         -DUSE_ROCKSDB:BOOL='OFF' \
         -DUSE_ROCM:BOOL='OFF' \
@@ -216,13 +232,10 @@ build() {
         -Wno-dev \
         ..
         
-    # NOTE:
-    # The recommended approach of running make in build() and make install in
-    # package() produces two compilations (being the second one unnecessary).
-    # A workaround is to suppress make in build() and run only make install
-    # in package().
+    # fix: avoid a second compilation in package()
+    sed -i 's/^preinstall:[[:space:]]all/preinstall:/' Makefile
     
-    #make
+    make
 }
 
 package() {
@@ -234,8 +247,8 @@ package() {
     local _entry
     local _exclude_dirs
     local _exclude_libs
-    _exclude_dirs=($(find "${pkgdir}/usr/include" -mindepth 1 -maxdepth 1 -type d ! -name 'caffe*'))
-    _exclude_libs=($(find -L "${pkgdir}/usr/lib" -maxdepth 1 -type f ! -name 'libcaffe*'))
+    mapfile -t -d '' _exclude_dirs < <(find "${pkgdir}/usr/include" -mindepth 1 -maxdepth 1 -type d ! -name 'caffe*' -print0)
+    mapfile -t -d '' _exclude_libs < <(find -L "${pkgdir}/usr/lib" -maxdepth 1 -type f ! -name 'libcaffe*' -print0)
     rm -f  "$pkgdir"/usr/bin/{protoc,unzstd,zstd{cat,mt,}}
     rm -f  "$pkgdir"/usr/include/{*.h,*.py}
     rm -rf "$pkgdir"/usr/lib/cmake/protobuf
@@ -243,7 +256,7 @@ package() {
     rm -rf "$pkgdir"/usr/share/pkgconfig
     rm -rf "$pkgdir"/usr/share/{ATen,cmake/{ATen,ONNX}}
     rm -f  "$pkgdir"/usr/share/man/man1/{unzstd,zstd{cat,}}.1
-    for _entry in ${_exclude_dirs[@]} ${_exclude_libs[@]}
+    for _entry in "${_exclude_dirs[@]}" "${_exclude_libs[@]}"
     do
         rm -rf "$_entry"
     done