fix building for 1.3.1

2 files changed, 298 insertions, 90 deletions

diff --git a/.SRCINFO b/.SRCINFO
index 45b77039a3f2..20c63754fcfa 100644
--- a/.SRCINFO
+++ b/.SRCINFO
@@ -1,43 +1,90 @@
-# Generated by mksrcinfo v8
-# Fri Nov 17 20:02:34 UTC 2017
 pkgbase = python-pytorch-git
 	pkgdesc = Tensors and Dynamic neural networks in Python with strong GPU acceleration
-	pkgver = 0.1.10.r46.gbe6322e4
-	pkgrel = 4
-	url = https://github.com/pytorch/pytorch
+	pkgver = 1.3.1.r22820.1350b99de4
+	pkgrel = 1
+	url = https://pytorch.org
 	arch = x86_64
 	license = BSD
-	makedepends = python
-	makedepends = python-setuptools
-	makedepends = python2
-	makedepends = python2-setuptools
-	makedepends = python-yaml
-	makedepends = python2-yaml
-	makedepends = python-numpy
-	makedepends = python2-numpy
-	makedepends = gcc5
 	makedepends = cmake
+	makedepends = cuda
+	makedepends = cuda
+	makedepends = cudnn
+	makedepends = cudnn
+	makedepends = doxygen
 	makedepends = git
-	depends = cuda
-	depends = cudnn
-	source = git://github.com/pytorch/pytorch
-	sha256sums = SKIP
+	makedepends = magma
+	makedepends = nccl
+	makedepends = python-numpy
+	makedepends = python-setuptools
+	makedepends = python-yaml
+	depends = ffmpeg
+	depends = gflags
+	depends = google-glog
+	depends = leveldb
+	depends = lmdb
+	depends = opencv
+	depends = openmp
+	depends = protobuf
+	depends = pybind11
+	depends = python-future
+	depends = python-numpy
+	depends = python-yaml
+	depends = qt5-base
+	depends = zeromq
+	source = pytorch::git+https://github.com/pytorch/pytorch.git
+	sha512sums = SKIP
 
 pkgname = python-pytorch-git
+
+pkgname = python-pytorch-mkl-git
+	pkgdesc = Tensors and Dynamic neural networks in Python with strong GPU acceleration (with MKL-DNN)
+	provides = python-pytorch=1.3.1.r22820.1350b99de4
+	conflicts = python-pytorch
+
+pkgname = python-pytorch-cuda-git
+	pkgdesc = Tensors and Dynamic neural networks in Python with strong GPU acceleration (with CUDA)
+	depends = ffmpeg
+	depends = gflags
+	depends = google-glog
+	depends = leveldb
+	depends = lmdb
+	depends = opencv
+	depends = openmp
+	depends = protobuf
+	depends = pybind11
+	depends = python-future
+	depends = python-numpy
+	depends = python-yaml
+	depends = qt5-base
+	depends = zeromq
 	depends = cuda
 	depends = cudnn
-	depends = python
-	depends = python-yaml
-	depends = python-numpy
-	provides = python-pytorch
+	depends = magma
+	depends = nccl
+	provides = python-pytorch=1.3.1.r22820.1350b99de4
 	conflicts = python-pytorch
 
-pkgname = python2-pytorch-git
+pkgname = python-pytorch-mkl-cuda-git
+	pkgdesc = Tensors and Dynamic neural networks in Python with strong GPU acceleration (with CUDA and MKL-DNN)
+	depends = ffmpeg
+	depends = gflags
+	depends = google-glog
+	depends = leveldb
+	depends = lmdb
+	depends = opencv
+	depends = openmp
+	depends = protobuf
+	depends = pybind11
+	depends = python-future
+	depends = python-numpy
+	depends = python-yaml
+	depends = qt5-base
+	depends = zeromq
 	depends = cuda
 	depends = cudnn
-	depends = python2
-	depends = python2-yaml
-	depends = python2-numpy
-	provides = python2-pytorch
-	conflicts = python2-pytorch
+	depends = magma
+	depends = nccl
+	provides = python-pytorch=1.3.1.r22820.1350b99de4
+	provides = python-pytorch-cuda=1.3.1.r22820.1350b99de4
+	conflicts = python-pytorch
 
diff --git a/PKGBUILD b/PKGBUILD
index 1bd89f47d9b9..8b6a9b358da7 100644
--- a/PKGBUILD
+++ b/PKGBUILD
@@ -1,83 +1,244 @@
+# Maintainer: Butui Hu <hot123tea123@gmail.com>
+# Contributor: Sven-Hendrik Haase <sh@lutzhaase.com>
 # Contributor: Stephen Zhang <zsrkmyn at gmail dot com>
 
-pkgbase="python-pytorch-git"
-pkgname=("python-pytorch-git" "python2-pytorch-git")
-_pkgname="pytorch"
-pkgver=0.1.10.r46.gbe6322e4
-pkgrel=4
-pkgdesc="Tensors and Dynamic neural networks in Python with strong GPU acceleration"
+pkgname=('python-pytorch-git' 'python-pytorch-mkl-git' 'python-pytorch-cuda-git' 'python-pytorch-mkl-cuda-git')
+_pkgname='pytorch'
+_pkgver=1.3.1
+pkgver=1.3.1.r22820.1350b99de4
+pkgrel=1
+pkgdesc='Tensors and Dynamic neural networks in Python with strong GPU acceleration'
 arch=('x86_64')
-url="https://github.com/pytorch/pytorch"
+url='https://pytorch.org'
 license=('BSD')
-makedepends=('python' 'python-setuptools' 'python2' 'python2-setuptools'
-             'python-yaml' 'python2-yaml' 'python-numpy' 'python2-numpy'
-             'gcc5' 'cmake' 'git')
-depends=('cuda' 'cudnn')
-source=("git://github.com/pytorch/pytorch")
-sha256sums=('SKIP')
-
-pkgver () {
-  cd "${_pkgname}"
-  (
-    set -o pipefail
-    git describe --tag 2>/dev/null | sed -e 's/\([^-]*-g\)/r\1/;s/-/./g' -e 's/^v//g'
-  )
+depends=(
+  'ffmpeg'
+  'gflags'
+  'google-glog'
+  'leveldb'
+  'lmdb'
+  'opencv'
+  'openmp'
+  'protobuf'
+  'pybind11'
+  'python-future'
+  'python-numpy'
+  'python-yaml'
+  'qt5-base'
+  'zeromq'
+)
+makedepends=(
+  'cmake'
+  'cuda'
+  'cuda'
+  'cudnn'
+  'cudnn'
+  'doxygen'
+  'git'
+  'magma'
+  'nccl'
+  'python-numpy'
+  'python-setuptools'
+  'python-yaml'
+)
+source=("${_pkgname}::git+https://github.com/pytorch/pytorch.git")
+sha512sums=('SKIP')
+
+get_pyver() {
+  python -c 'import sys; print(str(sys.version_info[0]) + "." + str(sys.version_info[1]))'
 }
 
+pkgver() {
+  cd "${srcdir}/${_pkgname}"
+  ver=$(printf "r%s.%s" "$(git rev-list --count HEAD)" "$(git rev-parse --short HEAD)")
+  echo "${_pkgver}.${ver}"
+}
 
 prepare() {
-  cd "$srcdir/${_pkgname}"
-  git submodule update --init
-
-  cd "$srcdir"
-  cp -a "${_pkgname}" "${_pkgname}-py2"
   cd "${_pkgname}"
-  sed -e "s|#![ ]*/usr/bin/python$|#!/usr/bin/python2|" \
-      -e "s|#![ ]*/usr/bin/env python$|#!/usr/bin/env python2|" \
-      -e "s|#![ ]*/bin/env python$|#!/usr/bin/env python2|" \
-      -i $(find . -name '*.py')
+
+  # This is the lazy way since pytorch has sooo many submodules and they keep
+  # changing them around but we've run into more problems so far doing it the
+  # manual than the lazy way. This lazy way (not explicitly specifying all
+  # submodules) will make building inefficient but for now I'll take it.
+  # It will result in the same package, don't worry.
+  git submodule update --init --recursive
+
+  # https://github.com/pytorch/pytorch/issues/26555
+  sed -i 's#^  ${CMAKE_CURRENT_SOURCE_DIR}/tensor_iterator_test.cpp##g' aten/src/ATen/test/CMakeLists.txt
+
+  # Fix build with Python 3.8
+  # https://github.com/pytorch/pytorch/issues/28060
+  find -name '*.cpp' -exec sed -i '/tp_print/s/nullptr/0/' {} +
+
+  cd ..
+  cp -a "${_pkgname}" "${_pkgname}-git"
+  cp -a "${_pkgname}" "${_pkgname}-mkl-git"
+  cp -a "${_pkgname}" "${_pkgname}-cuda-git"
+  cp -a "${_pkgname}" "${_pkgname}-mkl-cuda-git"
+
+  # Check tools/setup_helpers/cmake.py, setup.py and CMakeLists.txt for a list of flags that can be set via env vars.
+  export BUILD_BINARY=ON
+  export BUILD_CAFFE2_OPS=ON
+  export BUILD_CUSTOM_PROTOBUF=OFF
+  export BUILD_DOCS=ON
+  export BUILDING_WITH_TORCH_LIBS=ON
+  export BUILD_JNI=OFF
+  export BUILD_NAMEDTENSOR=ON
+  export BUILD_SHARED_LIBS=ON
+  export BUILD_TEST=OFF
+  export CAFFE2_STATIC_LINK_CUDA=OFF
+  export CUDA_HOME=/opt/cuda
+  export CUDAHOSTCXX=/opt/cuda/bin/g++
+  export CUDNN_INCLUDE_DIR=/usr/include
+  export CUDNN_LIB_DIR=/usr/lib
+  export PYTORCH_BUILD_NUMBER=1
+  export PYTORCH_BUILD_VERSION="${_pkgver}"
+  # modify to your need, you don't need to compile for all GPU arch
+  export TORCH_CUDA_ARCH_LIST="3.2;3.5;3.7;5.0;5.2;5.3;6.0;6.0+PTX;6.1;6.1+PTX;6.2;6.2+PTX;7.0;7.0+PTX;7.2;7.2+PTX;7.5;7.5+PTX"
+  export TORCH_NVCC_FLAGS="-Xfatbin -compress-all"
+  export USE_CAFFE2_OPS=ON
+  export USE_DISTRIBUTED=ON
+  export USE_FBGEMM=OFF
+  export USE_FFMPEG=ON
+  export USE_GFLAGS=ON
+  export USE_GLOG=ON
+  export USE_GLOO=ON
+  export USE_LEVELDB=ON
+  export USE_LITE_PROTO=OFF
+  export USE_LMDB=ON
+  export USE_NATIVE_ARCH=ON
+  export USE_NNPACK=ON
+  export USE_NUMPY=ON
+  export USE_NVRTC=ON
+  export USE_OPENCV=ON
+  export USE_OPENMP=ON
+  export USE_QNNPACK=ON
+  export USE_ROCM=OFF
+  export USE_STATIC_NCCL=OFF
+  export USE_SYSTEM_EIGEN_INSTALL=ON
+  export USE_SYSTEM_NCCL=ON
+  export USE_TBB=ON
+  export USE_ZMQ=ON
+  export USE_ZSTD=ON
+  export VERBOSE=1
 }
 
 build() {
-  msg "Building Python 2"
-  cd "$srcdir/${_pkgname}-py2"
-  CC=gcc-5 \
-  CXX=g++-5 \
-  WITH_CUDA=1 \
-  CUDA_HOME=/opt/cuda \
-  WITH_CUDNN=1 \
-  CUDNN_LIB_DIR=/opt/cuda/lib64 \
-  CUDNN_INCLUDE_DIR=/opt/cuda/include \
-  python2 setup.py build
-
-  msg "Building Python 3"
-  cd "$srcdir/${_pkgname}"
-  CC=gcc-5 \
-  CXX=g++-5 \
-  WITH_CUDA=1 \
-  CUDA_HOME=/opt/cuda \
-  WITH_CUDNN=1 \
-  CUDNN_LIB_DIR=/opt/cuda/lib64 \
-  CUDNN_INCLUDE_DIR=/opt/cuda/include \
+  echo "Building without cuda and without MKL-DNN"
+  export USE_CUDA=OFF
+  export USE_CUDNN=OFF
+  export USE_MKLDNN_CBLAS=OFF
+  export USE_MKLDNN=OFF
+  export USE_NCCL=OFF
+
+  cd "${srcdir}/${_pkgname}-git"
+  python setup.py build
+
+
+  echo "Building without cuda and with MKL-DNN"
+  export USE_CUDA=OFF
+  export USE_CUDNN=OFF
+  export USE_MKLDNN_CBLAS=ON
+  export USE_MKLDNN=ON
+  export USE_NCCL=OFF
+  cd "${srcdir}/${_pkgname}-mkl-git"
+  python setup.py build
+
+
+  echo "Building with cuda and without MKL-DNN"
+  export USE_CUDA=ON
+  export USE_CUDNN=ON
+  export USE_MKLDNN_CBLAS=OFF
+  export USE_MKLDNN=OFF
+  export USE_NCCL=ON
+  cd "${srcdir}/${_pkgname}-cuda-git"
+  python setup.py build
+
+
+  echo "Building with cuda and with MKL-DNN"
+  export USE_CUDA=ON
+  export USE_CUDNN=ON
+  export USE_MKLDNN_CBLAS=ON
+  export USE_MKLDNN=ON
+  export USE_NCCL=ON
+  cd "${srcdir}/${_pkgname}-mkl-cuda-git"
   python setup.py build
 }
 
-package_python2-pytorch-git() {
-  conflicts=('python2-pytorch')
-  provides=('python2-pytorch')
-  depends+=('python2' 'python2-yaml' 'python2-numpy')
-  cd "$srcdir/${_pkgname}-py2"
-  python2 setup.py install --root="$pkgdir"/ --optimize=1 --skip-build
-  install -Dm644 LICENSE "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE.txt"
+_package() {
+  # Prevent setup.py from re-running CMake and rebuilding
+  sed -e 's/RUN_BUILD_DEPS = True/RUN_BUILD_DEPS = False/g' -i setup.py
+
+  python setup.py install --root="${pkgdir}"/ --optimize=1 --skip-build
+
+  install -Dm644 LICENSE "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE"
+
+  pytorchpath="usr/lib/python$(get_pyver)/site-packages/torch"
+  install -d "${pkgdir}/usr/lib"
+
+  # put CMake files in correct place
+  mv "${pkgdir}/${pytorchpath}/share/cmake" "${pkgdir}/usr/lib/cmake"
+
+  # put C++ API in correct place
+  mv "${pkgdir}/${pytorchpath}/include" "${pkgdir}/usr/include"
+  mv "${pkgdir}/${pytorchpath}/lib"/*.so* "${pkgdir}/usr/lib/"
+
+  # clean up duplicates
+  # TODO: move towards direct shared library dependecy of:
+  #   c10, caffe2, libcpuinfo, CUDA RT, gloo, GTest, Intel MKL,
+  #   NVRTC, ONNX, protobuf, libthreadpool, QNNPACK
+  rm -rf "${pkgdir}/usr/include/pybind11"
+
+  # python module is hardcoded to look there at runtime
+  ln -s /usr/include "${pkgdir}/${pytorchpath}/include"
+  find "${pkgdir}"/usr/lib -type f -name "*.so*" -print0 | while read -rd $'\0' _lib; do
+    ln -s ${_lib#"$pkgdir"} "${pkgdir}/${pytorchpath}/lib/"
+  done
 }
 
 package_python-pytorch-git() {
-  conflicts=('python-pytorch')
-  provides=('python-pytorch')
-  depends+=('python' 'python-yaml' 'python-numpy')
-  cd "$srcdir/${_pkgname}"
-  python setup.py install --root="$pkgdir"/ --optimize=1 --skip-build
-  install -Dm644 LICENSE "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE.txt"
+  cd "${srcdir}/${_pkgname}-git"
+  _package
+}
+
+package_python-pytorch-mkl-git() {
+  pkgdesc='Tensors and Dynamic neural networks in Python with strong GPU acceleration (with MKL-DNN)'
+  conflicts=(python-pytorch)
+  provides=(python-pytorch=${pkgver})
+
+  cd "${srcdir}/${_pkgname}-mkl-git"
+  _package
+}
+
+package_python-pytorch-cuda-git() {
+  pkgdesc='Tensors and Dynamic neural networks in Python with strong GPU acceleration (with CUDA)'
+  depends+=(
+    'cuda'
+    'cudnn'
+    'magma'
+    'nccl'
+)
+  conflicts=(python-pytorch)
+  provides=(python-pytorch=${pkgver})
+
+  cd "${srcdir}/${_pkgname}-cuda-git"
+  _package
+}
+
+package_python-pytorch-mkl-cuda-git() {
+  pkgdesc='Tensors and Dynamic neural networks in Python with strong GPU acceleration (with CUDA and MKL-DNN)'
+  depends+=(
+    'cuda'
+    'cudnn'
+    'magma'
+    'nccl'
+)
+  conflicts=(python-pytorch)
+  provides=(python-pytorch=${pkgver} python-pytorch-cuda=${pkgver})
+
+  cd "${srcdir}/${_pkgname}-mkl-cuda-git"
+  _package
 }
 
 # vim:set ts=2 sw=2 et: