summarylogtreecommitdiffstats
diff options
context:
space:
mode:
authorkyechou2017-08-01 14:08:41 +0800
committerkyechou2017-08-01 14:26:55 +0800
commitbd85e51045d8452b5c65d2e1b7ad81ad7edfa25d (patch)
tree85302cf9cee535a274a033eba8c31a56f77ec494
downloadaur-bd85e51045d8452b5c65d2e1b7ad81ad7edfa25d.tar.gz
shifter-slurm aur files
-rw-r--r--.SRCINFO62
-rw-r--r--PKGBUILD90
-rw-r--r--build_ssh.patch75
-rw-r--r--shifter-imagegw.conf1
-rw-r--r--shifter-imagegw.service11
-rw-r--r--shifter-runtime.conf3
-rw-r--r--shifter-slurm-imagegw.install32
-rw-r--r--shifter-slurm-runtime.install38
-rwxr-xr-xstart-imagegw.sh16
9 files changed, 328 insertions, 0 deletions
diff --git a/.SRCINFO b/.SRCINFO
new file mode 100644
index 00000000000..93348dbc2d2
--- /dev/null
+++ b/.SRCINFO
@@ -0,0 +1,62 @@
+pkgbase = shifter-slurm
+ pkgdesc = NERSC Shifter: Linux Containers for HPC (with SLURM support)
+ pkgver = 16.08.3
+ pkgrel = 2
+ url = https://github.com/NERSC/shifter
+ arch = x86_64
+ license = BSD
+ makedepends = gcc
+ makedepends = glibc
+ makedepends = make
+ makedepends = autoconf
+ makedepends = automake
+ makedepends = libtool
+ makedepends = libcap
+ makedepends = munge
+ makedepends = json-c
+ makedepends = slurm-llnl
+ source = https://github.com/NERSC/shifter/archive/16.08.3_2.tar.gz
+ source = build_ssh.patch
+ source = shifter-runtime.conf
+ source = shifter-imagegw.conf
+ source = shifter-imagegw.service
+ source = start-imagegw.sh
+ sha512sums = c5de6953909b3c33693a5957c797d3fe505f0a18507cf25e801c0ac8b02f87e8a17388e0768782a659c2c5ee7c879d98e8193ddd8ec0331b965884d2f8eee656
+ sha512sums = bcb8da45ba803f40a76902961b86783d2399732272d68554dd75e09115958c9d1cf5c8d9d6d1f0db406d36a16e404500c47fa40c70d4f17ab0c6b7d94cd28f06
+ sha512sums = 8df5449ea93988364e8c350a2652e661545d883fada2f589b944cc7a2bd0d34b494f31e0f63bb0aea9c82e2805b4c9393b2290575dd14497fcd873b286d51b36
+ sha512sums = c56ad3b92901abdb41cffaeceeff9a6c8dae3882aacf9a4b654a8898f1e26eeadda3ac3ac799b127b176cc31d397652f27a07bcdfbf7f4ede3c6826d08f7bd7d
+ sha512sums = daf8163025a5d4099dc64cab8bab57d0b1a4101f04c563b5b03521b386d87581565d0e03c2fb3a02d791daff16cfb139244cc6bd7a886661e3caa0d107a7cb0a
+ sha512sums = afd15fb203ccf76ebb2a3542cbbde216f9f988752808479f25292ef4d9949a793070eead996072b564fe19dec6c33da72be21c1732a0e7d5817b30e7306d65e3
+
+pkgname = shifter-slurm-runtime
+ install = shifter-slurm-runtime.install
+ depends = munge
+ depends = curl
+ depends = pam
+ depends = squashfs-tools
+ depends = json-c
+ depends = xfsprogs
+ depends = wget
+ depends = slurm-llnl
+ provides = shifter-runtime
+ conflicts = shifter-runtime
+
+pkgname = shifter-slurm-imagegw
+ install = shifter-slurm-imagegw.install
+ depends = munge
+ depends = curl
+ depends = pam
+ depends = squashfs-tools
+ depends = mongodb
+ depends = redis
+ depends = jq
+ depends = slurm-llnl
+ depends = python2-celery
+ depends = python2-pymongo
+ depends = python2-flask
+ depends = python2-redis
+ depends = gunicorn-python2
+ depends = python2-pylint
+ provides = shifter-imagegw
+ conflicts = shifter-imagegw
+
diff --git a/PKGBUILD b/PKGBUILD
new file mode 100644
index 00000000000..65fe445dc3b
--- /dev/null
+++ b/PKGBUILD
@@ -0,0 +1,90 @@
+# Maintainer: Kuan-Yen Chou <forendef2846 at gmail dot com>
+
+pkgbase=shifter-slurm
+pkgname=('shifter-slurm-runtime' 'shifter-slurm-imagegw')
+pkgver=16.08.3
+pkgrel=2
+pkgdesc='NERSC Shifter: Linux Containers for HPC (with SLURM support)'
+makedepends=('gcc' 'glibc' 'make' 'autoconf' 'automake' 'libtool' 'libcap' 'munge' 'json-c' 'slurm-llnl')
+arch=('x86_64')
+url="https://github.com/NERSC/shifter"
+license=('BSD')
+source=("https://github.com/NERSC/shifter/archive/${pkgver}_${pkgrel}.tar.gz"
+ 'build_ssh.patch'
+ 'shifter-runtime.conf'
+ 'shifter-imagegw.conf'
+ 'shifter-imagegw.service'
+ 'start-imagegw.sh')
+sha512sums=('c5de6953909b3c33693a5957c797d3fe505f0a18507cf25e801c0ac8b02f87e8a17388e0768782a659c2c5ee7c879d98e8193ddd8ec0331b965884d2f8eee656'
+ 'bcb8da45ba803f40a76902961b86783d2399732272d68554dd75e09115958c9d1cf5c8d9d6d1f0db406d36a16e404500c47fa40c70d4f17ab0c6b7d94cd28f06'
+ '8df5449ea93988364e8c350a2652e661545d883fada2f589b944cc7a2bd0d34b494f31e0f63bb0aea9c82e2805b4c9393b2290575dd14497fcd873b286d51b36'
+ 'c56ad3b92901abdb41cffaeceeff9a6c8dae3882aacf9a4b654a8898f1e26eeadda3ac3ac799b127b176cc31d397652f27a07bcdfbf7f4ede3c6826d08f7bd7d'
+ 'daf8163025a5d4099dc64cab8bab57d0b1a4101f04c563b5b03521b386d87581565d0e03c2fb3a02d791daff16cfb139244cc6bd7a886661e3caa0d107a7cb0a'
+ 'afd15fb203ccf76ebb2a3542cbbde216f9f988752808479f25292ef4d9949a793070eead996072b564fe19dec6c33da72be21c1732a0e7d5817b30e7306d65e3')
+
+prepare() {
+ cd "${srcdir}/shifter-${pkgver}_${pkgrel}"
+ patch -p0 < ../build_ssh.patch
+ sed -i -e 's/\/api\/expire\/<system>\/<imgtype>\/<tag>\//\/api\/expire\/<system>\/<imgtype>\/<path:tag>\//' imagegw/shifter_imagegw/api.py
+}
+
+build() {
+ cd "${srcdir}/shifter-${pkgver}_${pkgrel}"
+ export PYTHON=/usr/bin/python2
+ ./autogen.sh
+ ./configure \
+ --prefix=/usr \
+ --sbindir=/usr/bin \
+ --sysconfdir=/etc/shifter \
+ --with-json-c \
+ --with-libcurl \
+ --with-munge \
+ --with-slurm=/usr
+ make
+}
+
+package_shifter-slurm-runtime() {
+ depends=('munge' 'curl' 'pam' 'squashfs-tools'
+ 'json-c' 'xfsprogs' 'wget' 'slurm-llnl')
+ install="${pkgname}.install"
+ provides=('shifter-runtime')
+ conflicts=('shifter-runtime')
+ cd "${srcdir}/shifter-${pkgver}_${pkgrel}"
+ make DESTDIR="${pkgdir}" install
+ install -Dm 644 LICENSE "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE"
+ install -Dm 644 ../shifter-runtime.conf -t "${pkgdir}/etc/modules-load.d"
+ rm -rf "${pkgdir}/etc/shifter/shifter_etc_files/group"
+ rm -rf "${pkgdir}/etc/shifter/shifter_etc_files/passwd"
+ rm -rf "${pkgdir}/etc/shifter/imagemanager.json.example"
+ rm -rf "${pkgdir}/usr/lib"
+ rm -rf "${pkgdir}/usr/libexec/shifter/imagecli.py"
+ rm -rf "${pkgdir}/usr/libexec/shifter/imagegwapi.py"
+ rm -rf "${pkgdir}/usr/libexec/shifter/sitecustomize.py"
+ rm -rf "${pkgdir}/usr/share/shifter"
+}
+
+package_shifter-slurm-imagegw() {
+ depends=('munge' 'curl' 'pam' 'squashfs-tools'
+ 'mongodb' 'redis' 'jq' 'slurm-llnl'
+ 'python2-celery' 'python2-pymongo' 'python2-flask'
+ 'python2-redis' 'gunicorn-python2' 'python2-pylint')
+ install="${pkgname}.install"
+ provides=('shifter-imagegw')
+ conflicts=('shifter-imagegw')
+ cd "${srcdir}/shifter-${pkgver}_${pkgrel}"
+ make DESTDIR="${pkgdir}" install
+ install -Dm 644 LICENSE "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE"
+ install -Dm 644 imagegw/*{.py,.sh,.yml,.json} -t "${pkgdir}/opt/shifter/imagegw/"
+ install -Dm 755 ../start-imagegw.sh -t "${pkgdir}/opt/shifter/imagegw"
+ install -Dm 644 ../shifter-imagegw.service -t "${pkgdir}/usr/lib/systemd/system"
+ install -Dm 644 ../shifter-imagegw.conf -t "${pkgdir}/etc/modules-load.d"
+ rm -rf "${pkgdir}/etc/shifter/udiRoot.conf.example"
+ rm -rf "${pkgdir}/etc/shifter/shifter_etc_files"
+ rm -rf "${pkgdir}/usr/bin"
+ rm -rf "${pkgdir}/usr/libexec/shifter/mount"
+ rm -rf "${pkgdir}/usr/libexec/shifter/opt"
+ rm -rf "${pkgdir}/usr/libexec/shifter/shifter_slurm_dws_support"
+ rm -rf "${pkgdir}/usr/share/shifter"
+}
+
+# vim: set ts=4 sw=4 et :
diff --git a/build_ssh.patch b/build_ssh.patch
new file mode 100644
index 00000000000..23606bbb8d8
--- /dev/null
+++ b/build_ssh.patch
@@ -0,0 +1,75 @@
+--- dep/build_ssh.sh 2017-05-08 23:53:14.000000000 +0800
++++ dep/build_ssh.sh 2017-07-27 15:51:29.847977423 +0800
+@@ -8,7 +8,7 @@
+ INST_PREFIX=${INST_PREFIX:-/opt/udiImage}
+ SPRT_PREFIX=$( mktemp -d )
+ PREFIX=$( mktemp -d )
+-MUSL_VERSION=1.1.8
++MUSL_VERSION=latest
+ LIBRESSL_VERSION=2.1.6
+ ZLIB_VERSION=1.2.8
+ OPENSSH_VERSION=6.8p1
+@@ -23,12 +23,6 @@
+ exit 1
+ fi
+
+-if [[ ! -e "musl-${MUSL_VERSION}.tar.gz" && -n "$DEPTAR_DIR" && -e "$DEPTAR_DIR/musl-${MUSL_VERSION}.tar.gz" ]]; then
+- cp "$DEPTAR_DIR/musl-${MUSL_VERSION}.tar.gz" .
+-fi
+-if [[ ! -e "musl-${MUSL_VERSION}.tar.gz" ]]; then
+- curl -o "musl-${MUSL_VERSION}.tar.gz" "http://www.musl-libc.org/releases/musl-${MUSL_VERSION}.tar.gz"
+-fi
+ if [[ ! -e "libressl-${LIBRESSL_VERSION}.tar.gz" && -n "$DEPTAR_DIR" && -e "$DEPTAR_DIR/libressl-${LIBRESSL_VERSION}.tar.gz" ]]; then
+ cp "$DEPTAR_DIR/libressl-${LIBRESSL_VERSION}.tar.gz" .
+ fi
+@@ -48,31 +42,11 @@
+ curl -o "openssh-${OPENSSH_VERSION}.tar.gz" "http://mirrors.sonic.net/pub/OpenBSD/OpenSSH/portable/openssh-${OPENSSH_VERSION}.tar.gz"
+ fi
+
+-mkdir -p musl
+-tar xf "musl-${MUSL_VERSION}.tar.gz" -C musl --strip-components=1
+-cd musl
+-./configure "--prefix=${SPRT_PREFIX}" --enable-static --disable-shared
+-make
+-make install
+-cd "${builddir}"
+-
+-dirs="linux asm asm-generic x86_64-linux-gnu/asm"
+-for dir in $dirs; do
+- if [[ -e "/usr/include/$dir" ]]; then
+- if [[ -L "/usr/include/$dir" ]]; then
+- # SLES has symlinks for asm
+- realpath=$(readlink -f "/usr/include/$dir")
+- cp -rp "$realpath" "${SPRT_PREFIX}/include/"
+- fi
+- cp -rp "/usr/include/$dir" "${SPRT_PREFIX}/include/"
+- fi
+-done
+-
+ cd "${builddir}"
+ mkdir -p libressl
+ tar xf "libressl-${LIBRESSL_VERSION}.tar.gz" -C libressl --strip-components=1
+ cd libressl
+-CC="${SPRT_PREFIX}/bin/musl-gcc" ./configure "--prefix=${SPRT_PREFIX}" --enable-static --disable-shared
++./configure "--prefix=${SPRT_PREFIX}" --enable-static --disable-shared
+ make
+ make install
+
+@@ -80,7 +54,7 @@
+ mkdir -p zlib
+ tar xf "zlib-${ZLIB_VERSION}.tar.gz" -C zlib --strip-components=1
+ cd zlib
+-CC="${SPRT_PREFIX}/bin/musl-gcc" ./configure "--prefix=${SPRT_PREFIX}"
++./configure "--prefix=${SPRT_PREFIX}"
+ make
+ make install
+
+@@ -93,7 +67,7 @@
+ ## the image is not infected with all kinds of silly paths (sshd sets PATH to
+ ## very nearly the path it was built with)
+ export PATH="/usr/bin:/bin"
+-LDFLAGS="-L${SPRT_PREFIX}/lib -L${SPRT_PREFIX}/lib64" CC="${SPRT_PREFIX}/bin/musl-gcc" ./configure --without-pam "--with-ssl-dir=${SPRT_PREFIX}" --without-ssh1 --enable-static --disable-shared "--with-zlib=${SPRT_PREFIX}" "--prefix=${INST_PREFIX}"
++./configure --without-pam "--with-ssl-dir=${SPRT_PREFIX}" --without-ssh1 --enable-static --disable-shared "--with-zlib=${SPRT_PREFIX}" "--prefix=${INST_PREFIX}"
+ make
+ make install "DESTDIR=${PREFIX}"
+ cd "${builddir}"
diff --git a/shifter-imagegw.conf b/shifter-imagegw.conf
new file mode 100644
index 00000000000..7c1157fa31e
--- /dev/null
+++ b/shifter-imagegw.conf
@@ -0,0 +1 @@
+squashfs
diff --git a/shifter-imagegw.service b/shifter-imagegw.service
new file mode 100644
index 00000000000..14052dc2e3a
--- /dev/null
+++ b/shifter-imagegw.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Shifter Image Gateway
+Wants=network.target
+
+[Service]
+Type=forking
+ExecStart=/opt/shifter/imagegw/start-imagegw.sh
+KillMode=control-group
+
+[Install]
+WantedBy=multi-user.target
diff --git a/shifter-runtime.conf b/shifter-runtime.conf
new file mode 100644
index 00000000000..2aaf72f8aac
--- /dev/null
+++ b/shifter-runtime.conf
@@ -0,0 +1,3 @@
+ext4
+loop
+squashfs
diff --git a/shifter-slurm-imagegw.install b/shifter-slurm-imagegw.install
new file mode 100644
index 00000000000..537b7744386
--- /dev/null
+++ b/shifter-slurm-imagegw.install
@@ -0,0 +1,32 @@
+post_install() {
+ post_upgrade
+
+ echo -e "\e[39;1m- Please make sure that the Shifter runtimes and the image gateway has the same munge key.\e[0m"
+ echo -e "\e[39;1m- Check the configuration file (/etc/shifter/imagemanager.json) before starting the shifter-imagegw.service.\e[0m"
+}
+
+post_upgrade() {
+ systemctl daemon-reload
+
+ echo -e "\e[34;1m==>\e[39;1m Enabling mongodb.service... \e[0m"
+ systemctl is-enabled mongodb.service >/dev/null || systemctl enable mongodb.service
+
+ echo -e "\e[34;1m==>\e[39;1m Starting mongodb.service... \e[0m"
+ systemctl is-active mongodb.service >/dev/null || systemctl start mongodb.service
+
+ echo -e "\e[34;1m==>\e[39;1m Enabling redis.service... \e[0m"
+ systemctl is-enabled redis.service >/dev/null || systemctl enable redis.service
+
+ echo -e "\e[34;1m==>\e[39;1m Starting redis.service... \e[0m"
+ systemctl is-active redis.service >/dev/null || systemctl start redis.service
+
+ echo -e "\e[34;1m==>\e[39;1m Enabling munge.service... \e[0m"
+ systemctl is-enabled munge.service >/dev/null || systemctl enable munge.service
+
+ echo -e "\e[34;1m==>\e[39;1m Starting munge.service... \e[0m"
+ systemctl is-active munge.service >/dev/null || systemctl start munge.service
+
+ echo -e "\e[34;1m==>\e[39;1m Loading kernel modules... \e[0m"
+ [ -z "$(lsmod | grep ^squashfs)" ] && modprobe squashfs
+ [ -n "$(lsmod | grep ^squashfs)" ] && echo -e " \e[34;1m==>\e[39;1m squashfs... loaded\e[0m"
+}
diff --git a/shifter-slurm-runtime.install b/shifter-slurm-runtime.install
new file mode 100644
index 00000000000..c7530331fc8
--- /dev/null
+++ b/shifter-slurm-runtime.install
@@ -0,0 +1,38 @@
+post_install() {
+ post_upgrade
+
+ echo -e "\e[39;1m- Please make sure that the Shifter runtimes and the image gateway has the same munge key.\e[0m"
+ echo -e "\e[39;1m- Check the configuration file (/etc/shifter/udiRoot.conf) before pulling images or starting containers.\e[0m"
+}
+
+post_upgrade() {
+ systemctl daemon-reload
+
+ echo -e "\e[34;1m==>\e[39;1m Enabling munge.service... \e[0m"
+ systemctl is-enabled munge.service >/dev/null || systemctl enable munge.service
+
+ echo -e "\e[34;1m==>\e[39;1m Starting munge.service... \e[0m"
+ systemctl is-active munge.service >/dev/null || systemctl start munge.service
+
+ echo -e "\e[34;1m==>\e[39;1m Copying /etc/passwd... \e[0m"
+ cp -f /etc/passwd /etc/shifter/shifter_etc_files/passwd
+
+ echo -e "\e[34;1m==>\e[39;1m Copying /etc/group... \e[0m"
+ cp -f /etc/group /etc/shifter/shifter_etc_files/group
+
+ echo -e "\e[34;1m==>\e[39;1m Loading kernel modules... \e[0m"
+ [ -z "$(lsmod | grep ^ext4)" ] && modprobe ext4
+ [ -n "$(lsmod | grep ^ext4)" ] && echo -e " \e[34;1m==>\e[39;1m ext4... loaded\e[0m"
+ [ -z "$(lsmod | grep ^loop)" ] && modprobe loop
+ [ -n "$(lsmod | grep ^loop)" ] && echo -e " \e[34;1m==>\e[39;1m loop... loaded\e[0m"
+ [ -z "$(lsmod | grep ^squashfs)" ] && modprobe squashfs
+ [ -n "$(lsmod | grep ^squashfs)" ] && echo -e " \e[34;1m==>\e[39;1m squashfs... loaded\e[0m"
+}
+
+pre_remove() {
+ echo -e "\e[34;1m==>\e[39;1m Deleting /etc/shifter/shifter_etc_files/passwd... \e[0m"
+ rm -f /etc/shifter/shifter_etc_files/passwd
+
+ echo -e "\e[34;1m==>\e[39;1m Deleting /etc/shifter/shifter_etc_files/group... \e[0m"
+ rm -f /etc/shifter/shifter_etc_files/group
+}
diff --git a/start-imagegw.sh b/start-imagegw.sh
new file mode 100755
index 00000000000..491129ba0c2
--- /dev/null
+++ b/start-imagegw.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+if [ ! -f /etc/shifter/imagemanager.json ]; then
+ echo 'error: /etc/shifter/imagemanager.json does not exist'
+ exit 1
+fi
+SYSTEMS=($(jq '.Platforms | keys[]' /etc/shifter/imagemanager.json | sed -e 's/"//g'))
+ROOT_TREE=/opt/shifter/imagegw
+
+for QA in ${SYSTEMS}; do
+ echo "Starting Celery Queue $QA"
+ celery2 -A shifter_imagegw.imageworker worker -Q $QA --loglevel=INFO -n worker.queue.$QA &
+done
+
+echo "Starting imagegw API"
+python2 $ROOT_TREE/imagegwapi.py &