diff options
author | kyechou | 2017-07-27 17:00:45 +0800 |
---|---|---|
committer | kyechou | 2017-07-31 10:20:09 +0800 |
commit | f688754e0fd01f7540571df235cd478887bcdda6 (patch) | |
tree | cbaf0c0a24256241e4793a04e68cf7f92d8675c5 | |
download | aur-f688754e0fd01f7540571df235cd478887bcdda6.tar.gz |
shifter aur files
-rw-r--r-- | .SRCINFO | 61 | ||||
-rw-r--r-- | PKGBUILD | 92 | ||||
-rw-r--r-- | build_ssh.patch | 75 | ||||
-rw-r--r-- | shifter-imagegw.conf | 1 | ||||
-rw-r--r-- | shifter-imagegw.install | 60 | ||||
-rw-r--r-- | shifter-imagegw.service | 11 | ||||
-rw-r--r-- | shifter-runtime.conf | 3 | ||||
-rw-r--r-- | shifter-runtime.install | 52 | ||||
-rwxr-xr-x | start-imagegw.sh | 16 | ||||
-rw-r--r-- | transfer.patch | 13 |
10 files changed, 384 insertions, 0 deletions
diff --git a/.SRCINFO b/.SRCINFO new file mode 100644 index 000000000000..4b700f102988 --- /dev/null +++ b/.SRCINFO @@ -0,0 +1,61 @@ +pkgbase = shifter + pkgdesc = NERSC Shifter: Linux Containers for HPC + pkgver = 16.08.3 + pkgrel = 2 + url = https://github.com/NERSC/shifter + arch = x86_64 + license = BSD + makedepends = gcc + makedepends = glibc + makedepends = make + makedepends = autoconf + makedepends = automake + makedepends = libtool + makedepends = libcap + makedepends = munge + makedepends = json-c + source = https://github.com/NERSC/shifter/archive/16.08.3_2.tar.gz + source = build_ssh.patch + source = transfer.patch + source = shifter-runtime.conf + source = shifter-imagegw.conf + source = shifter-imagegw.service + source = start-imagegw.sh + sha512sums = c5de6953909b3c33693a5957c797d3fe505f0a18507cf25e801c0ac8b02f87e8a17388e0768782a659c2c5ee7c879d98e8193ddd8ec0331b965884d2f8eee656 + sha512sums = bcb8da45ba803f40a76902961b86783d2399732272d68554dd75e09115958c9d1cf5c8d9d6d1f0db406d36a16e404500c47fa40c70d4f17ab0c6b7d94cd28f06 + sha512sums = b6006b3983fbfef08d262c979437fdaca6806e518887095276e33aadba9faeca45efa86f0f48f8dc5b2a26314c8792e6f17343d3560e4946404144dc7be23cc6 + sha512sums = 8df5449ea93988364e8c350a2652e661545d883fada2f589b944cc7a2bd0d34b494f31e0f63bb0aea9c82e2805b4c9393b2290575dd14497fcd873b286d51b36 + sha512sums = c56ad3b92901abdb41cffaeceeff9a6c8dae3882aacf9a4b654a8898f1e26eeadda3ac3ac799b127b176cc31d397652f27a07bcdfbf7f4ede3c6826d08f7bd7d + sha512sums = daf8163025a5d4099dc64cab8bab57d0b1a4101f04c563b5b03521b386d87581565d0e03c2fb3a02d791daff16cfb139244cc6bd7a886661e3caa0d107a7cb0a + sha512sums = afd15fb203ccf76ebb2a3542cbbde216f9f988752808479f25292ef4d9949a793070eead996072b564fe19dec6c33da72be21c1732a0e7d5817b30e7306d65e3 + +pkgname = shifter-runtime + install = shifter-runtime.install + depends = munge + depends = curl + depends = pam + depends = squashfs-tools + depends = json-c + depends = xfsprogs + depends = wget + provides = shifter-runtime + conflicts = shifter-runtime-slurm + +pkgname = shifter-imagegw + install = shifter-imagegw.install + depends = munge + depends = curl + depends = pam + depends = squashfs-tools + depends = mongodb + depends = redis + depends = jq + depends = python2-celery + depends = python2-pymongo + depends = python2-flask + depends = python2-redis + depends = gunicorn-python2 + depends = python2-pylint + provides = shifter-imagegw + conflicts = shifter-imagegw-slurm + diff --git a/PKGBUILD b/PKGBUILD new file mode 100644 index 000000000000..b18ac7ae7e30 --- /dev/null +++ b/PKGBUILD @@ -0,0 +1,92 @@ +# Maintainer: Kuan-Yen Chou <forendef2846 at gmail dot com> + +pkgbase=shifter +pkgname=('shifter-runtime' 'shifter-imagegw') +pkgver=16.08.3 +pkgrel=2 +pkgdesc='NERSC Shifter: Linux Containers for HPC' +makedepends=('gcc' 'glibc' 'make' 'autoconf' 'automake' 'libtool' 'libcap' 'munge' 'json-c') +arch=('x86_64') +url="https://github.com/NERSC/shifter" +license=('BSD') +source=("https://github.com/NERSC/shifter/archive/${pkgver}_${pkgrel}.tar.gz" + 'build_ssh.patch' + 'transfer.patch' + 'shifter-runtime.conf' + 'shifter-imagegw.conf' + 'shifter-imagegw.service' + 'start-imagegw.sh') +sha512sums=('c5de6953909b3c33693a5957c797d3fe505f0a18507cf25e801c0ac8b02f87e8a17388e0768782a659c2c5ee7c879d98e8193ddd8ec0331b965884d2f8eee656' + 'bcb8da45ba803f40a76902961b86783d2399732272d68554dd75e09115958c9d1cf5c8d9d6d1f0db406d36a16e404500c47fa40c70d4f17ab0c6b7d94cd28f06' + 'b6006b3983fbfef08d262c979437fdaca6806e518887095276e33aadba9faeca45efa86f0f48f8dc5b2a26314c8792e6f17343d3560e4946404144dc7be23cc6' + '8df5449ea93988364e8c350a2652e661545d883fada2f589b944cc7a2bd0d34b494f31e0f63bb0aea9c82e2805b4c9393b2290575dd14497fcd873b286d51b36' + 'c56ad3b92901abdb41cffaeceeff9a6c8dae3882aacf9a4b654a8898f1e26eeadda3ac3ac799b127b176cc31d397652f27a07bcdfbf7f4ede3c6826d08f7bd7d' + 'daf8163025a5d4099dc64cab8bab57d0b1a4101f04c563b5b03521b386d87581565d0e03c2fb3a02d791daff16cfb139244cc6bd7a886661e3caa0d107a7cb0a' + 'afd15fb203ccf76ebb2a3542cbbde216f9f988752808479f25292ef4d9949a793070eead996072b564fe19dec6c33da72be21c1732a0e7d5817b30e7306d65e3') + +prepare() { + cd "${srcdir}/shifter-${pkgver}_${pkgrel}" + patch -p0 < ../build_ssh.patch + patch -p0 < ../transfer.patch + sed -i -e 's/\/api\/expire\/<system>\/<imgtype>\/<tag>\//\/api\/expire\/<system>\/<imgtype>\/<path:tag>\//' imagegw/shifter_imagegw/api.py +} + +build() { + cd "${srcdir}/shifter-${pkgver}_${pkgrel}" + export PYTHON=/usr/bin/python2 + ./autogen.sh + ./configure \ + --prefix=/usr \ + --sbindir=/usr/bin \ + --sysconfdir=/etc/shifter \ + --with-json-c \ + --with-libcurl \ + --with-munge + make +} + +package_shifter-runtime() { + depends=('munge' 'curl' 'pam' 'squashfs-tools' + 'json-c' 'xfsprogs' 'wget') + install="${pkgname}.install" + provides=('shifter-runtime') + conflicts=('shifter-runtime-slurm') + cd "${srcdir}/shifter-${pkgver}_${pkgrel}" + make DESTDIR="${pkgdir}" install + install -Dm 644 LICENSE "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE" + install -Dm 644 ../shifter-runtime.conf -t "${pkgdir}/etc/modules-load.d" + rm -rf "${pkgdir}/etc/shifter/shifter_etc_files/group" + rm -rf "${pkgdir}/etc/shifter/shifter_etc_files/passwd" + rm -rf "${pkgdir}/etc/shifter/imagemanager.json.example" + rm -rf "${pkgdir}/usr/lib" + rm -rf "${pkgdir}/usr/libexec/shifter/imagecli.py" + rm -rf "${pkgdir}/usr/libexec/shifter/imagegwapi.py" + rm -rf "${pkgdir}/usr/libexec/shifter/sitecustomize.py" + rm -rf "${pkgdir}/usr/share/shifter" +} + +package_shifter-imagegw() { + depends=('munge' 'curl' 'pam' 'squashfs-tools' + 'mongodb' 'redis' 'jq' + 'python2-celery' 'python2-pymongo' 'python2-flask' + 'python2-redis' 'gunicorn-python2' 'python2-pylint') + install="${pkgname}.install" + provides=('shifter-imagegw') + conflicts=('shifter-imagegw-slurm') + cd "${srcdir}/shifter-${pkgver}_${pkgrel}" + make DESTDIR="${pkgdir}" install + install -Dm 644 LICENSE "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE" + install -Dm 644 imagegw/*{.py,.sh,.yml,.json} -t "${pkgdir}/opt/shifter/imagegw/" + install -Dm 755 ../start-imagegw.sh -t "${pkgdir}/opt/shifter/imagegw" + install -Dm 644 ../shifter-imagegw.service -t "${pkgdir}/usr/lib/systemd/system" + install -Dm 644 ../shifter-imagegw.conf -t "${pkgdir}/etc/modules-load.d" + rm -rf "${pkgdir}/etc/shifter/udiRoot.conf.example" + rm -rf "${pkgdir}/etc/shifter/shifter_etc_files" + rm -rf "${pkgdir}/usr/bin" + rm -rf "${pkgdir}/usr/libexec/shifter/mount" + rm -rf "${pkgdir}/usr/libexec/shifter/opt" + rm -rf "${pkgdir}/usr/libexec/shifter/shifter_slurm_dws_support" + rm -rf "${pkgdir}/usr/share/shifter" +} + +# vim: set ts=4 sw=4 et : diff --git a/build_ssh.patch b/build_ssh.patch new file mode 100644 index 000000000000..23606bbb8d80 --- /dev/null +++ b/build_ssh.patch @@ -0,0 +1,75 @@ +--- dep/build_ssh.sh 2017-05-08 23:53:14.000000000 +0800 ++++ dep/build_ssh.sh 2017-07-27 15:51:29.847977423 +0800 +@@ -8,7 +8,7 @@ + INST_PREFIX=${INST_PREFIX:-/opt/udiImage} + SPRT_PREFIX=$( mktemp -d ) + PREFIX=$( mktemp -d ) +-MUSL_VERSION=1.1.8 ++MUSL_VERSION=latest + LIBRESSL_VERSION=2.1.6 + ZLIB_VERSION=1.2.8 + OPENSSH_VERSION=6.8p1 +@@ -23,12 +23,6 @@ + exit 1 + fi + +-if [[ ! -e "musl-${MUSL_VERSION}.tar.gz" && -n "$DEPTAR_DIR" && -e "$DEPTAR_DIR/musl-${MUSL_VERSION}.tar.gz" ]]; then +- cp "$DEPTAR_DIR/musl-${MUSL_VERSION}.tar.gz" . +-fi +-if [[ ! -e "musl-${MUSL_VERSION}.tar.gz" ]]; then +- curl -o "musl-${MUSL_VERSION}.tar.gz" "http://www.musl-libc.org/releases/musl-${MUSL_VERSION}.tar.gz" +-fi + if [[ ! -e "libressl-${LIBRESSL_VERSION}.tar.gz" && -n "$DEPTAR_DIR" && -e "$DEPTAR_DIR/libressl-${LIBRESSL_VERSION}.tar.gz" ]]; then + cp "$DEPTAR_DIR/libressl-${LIBRESSL_VERSION}.tar.gz" . + fi +@@ -48,31 +42,11 @@ + curl -o "openssh-${OPENSSH_VERSION}.tar.gz" "http://mirrors.sonic.net/pub/OpenBSD/OpenSSH/portable/openssh-${OPENSSH_VERSION}.tar.gz" + fi + +-mkdir -p musl +-tar xf "musl-${MUSL_VERSION}.tar.gz" -C musl --strip-components=1 +-cd musl +-./configure "--prefix=${SPRT_PREFIX}" --enable-static --disable-shared +-make +-make install +-cd "${builddir}" +- +-dirs="linux asm asm-generic x86_64-linux-gnu/asm" +-for dir in $dirs; do +- if [[ -e "/usr/include/$dir" ]]; then +- if [[ -L "/usr/include/$dir" ]]; then +- # SLES has symlinks for asm +- realpath=$(readlink -f "/usr/include/$dir") +- cp -rp "$realpath" "${SPRT_PREFIX}/include/" +- fi +- cp -rp "/usr/include/$dir" "${SPRT_PREFIX}/include/" +- fi +-done +- + cd "${builddir}" + mkdir -p libressl + tar xf "libressl-${LIBRESSL_VERSION}.tar.gz" -C libressl --strip-components=1 + cd libressl +-CC="${SPRT_PREFIX}/bin/musl-gcc" ./configure "--prefix=${SPRT_PREFIX}" --enable-static --disable-shared ++./configure "--prefix=${SPRT_PREFIX}" --enable-static --disable-shared + make + make install + +@@ -80,7 +54,7 @@ + mkdir -p zlib + tar xf "zlib-${ZLIB_VERSION}.tar.gz" -C zlib --strip-components=1 + cd zlib +-CC="${SPRT_PREFIX}/bin/musl-gcc" ./configure "--prefix=${SPRT_PREFIX}" ++./configure "--prefix=${SPRT_PREFIX}" + make + make install + +@@ -93,7 +67,7 @@ + ## the image is not infected with all kinds of silly paths (sshd sets PATH to + ## very nearly the path it was built with) + export PATH="/usr/bin:/bin" +-LDFLAGS="-L${SPRT_PREFIX}/lib -L${SPRT_PREFIX}/lib64" CC="${SPRT_PREFIX}/bin/musl-gcc" ./configure --without-pam "--with-ssl-dir=${SPRT_PREFIX}" --without-ssh1 --enable-static --disable-shared "--with-zlib=${SPRT_PREFIX}" "--prefix=${INST_PREFIX}" ++./configure --without-pam "--with-ssl-dir=${SPRT_PREFIX}" --without-ssh1 --enable-static --disable-shared "--with-zlib=${SPRT_PREFIX}" "--prefix=${INST_PREFIX}" + make + make install "DESTDIR=${PREFIX}" + cd "${builddir}" diff --git a/shifter-imagegw.conf b/shifter-imagegw.conf new file mode 100644 index 000000000000..7c1157fa31e7 --- /dev/null +++ b/shifter-imagegw.conf @@ -0,0 +1 @@ +squashfs diff --git a/shifter-imagegw.install b/shifter-imagegw.install new file mode 100644 index 000000000000..7f50cfe3fcdc --- /dev/null +++ b/shifter-imagegw.install @@ -0,0 +1,60 @@ +post_install() { + systemctl daemon-reload + + echo -e "\e[34;1m==>\e[39;1m Enabling mongodb.service... \e[0m" + MONGODB_ENABLED="$(systemctl list-unit-files | grep enabled | grep mongodb.service)" + [ -z "$MONGODB_ENABLED" ] && systemctl enable mongodb.service + + echo -e "\e[34;1m==>\e[39;1m Starting mongodb.service... \e[0m" + MONGODB_STARTED="$(systemctl | grep running | grep mongodb.service)" + [ -z "$MONGODB_STARTED" ] && systemctl start mongodb.service + + echo -e "\e[34;1m==>\e[39;1m Enabling redis.service... \e[0m" + REDIS_ENABLED="$(systemctl list-unit-files | grep enabled | grep redis.service)" + [ -z "$REDIS_ENABLED" ] && systemctl enable redis.service + + echo -e "\e[34;1m==>\e[39;1m Starting redis.service... \e[0m" + REDIS_STARTED="$(systemctl | grep running | grep redis.service)" + [ -z "$REDIS_STARTED" ] && systemctl start redis.service + + echo -e "\e[34;1m==>\e[39;1m Enabling munge.service... \e[0m" + MUNGE_ENABLED="$(systemctl list-unit-files | grep enabled | grep munge.service)" + [ -z "$MUNGE_ENABLED" ] && systemctl enable munge.service + + echo -e "\e[34;1m==>\e[39;1m Starting munge.service... \e[0m" + MUNGE_STARTED="$(systemctl | grep running | grep munge.service)" + [ -z "$MUNGE_STARTED" ] && systemctl start munge.service + + echo -e "\e[34;1m==>\e[39;1m Loading kernel modules... \e[0m" + [ -z "$(lsmod | grep ^squashfs)" ] && modprobe squashfs + [ -n "$(lsmod | grep ^squashfs)" ] && echo -e " \e[34;1m==>\e[39;1m squashfs... loaded\e[0m" + + echo -e "\e[39;1m- Please make sure that the Shifter runtimes and the image gateway has the same munge key.\e[0m" + echo -e "\e[39;1m- Check the configuration file (/etc/shifter/imagemanager.json) before starting the shifter-imagegw.service.\e[0m" +} + +post_upgrade() { + echo -e "\e[34;1m==>\e[39;1m Enabling mongodb.service... \e[0m" + MONGODB_ENABLED="$(systemctl list-unit-files | grep enabled | grep mongodb.service)" + [ -z "$MONGODB_ENABLED" ] && systemctl enable mongodb.service + + echo -e "\e[34;1m==>\e[39;1m Starting mongodb.service... \e[0m" + MONGODB_STARTED="$(systemctl | grep running | grep mongodb.service)" + [ -z "$MONGODB_STARTED" ] && systemctl start mongodb.service + + echo -e "\e[34;1m==>\e[39;1m Enabling redis.service... \e[0m" + REDIS_ENABLED="$(systemctl list-unit-files | grep enabled | grep redis.service)" + [ -z "$REDIS_ENABLED" ] && systemctl enable redis.service + + echo -e "\e[34;1m==>\e[39;1m Starting redis.service... \e[0m" + REDIS_STARTED="$(systemctl | grep running | grep redis.service)" + [ -z "$REDIS_STARTED" ] && systemctl start redis.service + + echo -e "\e[34;1m==>\e[39;1m Enabling munge.service... \e[0m" + MUNGE_ENABLED="$(systemctl list-unit-files | grep enabled | grep munge.service)" + [ -z "$MUNGE_ENABLED" ] && systemctl enable munge.service + + echo -e "\e[34;1m==>\e[39;1m Starting munge.service... \e[0m" + MUNGE_STARTED="$(systemctl | grep running | grep munge.service)" + [ -z "$MUNGE_STARTED" ] && systemctl start munge.service +} diff --git a/shifter-imagegw.service b/shifter-imagegw.service new file mode 100644 index 000000000000..14052dc2e3aa --- /dev/null +++ b/shifter-imagegw.service @@ -0,0 +1,11 @@ +[Unit] +Description=Shifter Image Gateway +Wants=network.target + +[Service] +Type=forking +ExecStart=/opt/shifter/imagegw/start-imagegw.sh +KillMode=control-group + +[Install] +WantedBy=multi-user.target diff --git a/shifter-runtime.conf b/shifter-runtime.conf new file mode 100644 index 000000000000..2aaf72f8aac4 --- /dev/null +++ b/shifter-runtime.conf @@ -0,0 +1,3 @@ +ext4 +loop +squashfs diff --git a/shifter-runtime.install b/shifter-runtime.install new file mode 100644 index 000000000000..4dd57f268c4f --- /dev/null +++ b/shifter-runtime.install @@ -0,0 +1,52 @@ +post_install() { + systemctl daemon-reload + + echo -e "\e[34;1m==>\e[39;1m Enabling munge.service... \e[0m" + MUNGE_ENABLED="$(systemctl list-unit-files | grep enabled | grep munge.service)" + [ -z "$MUNGE_ENABLED" ] && systemctl enable munge.service + + echo -e "\e[34;1m==>\e[39;1m Starting munge.service... \e[0m" + MUNGE_STARTED="$(systemctl | grep running | grep munge.service)" + [ -z "$MUNGE_STARTED" ] && systemctl start munge.service + + echo -e "\e[34;1m==>\e[39;1m Copying /etc/passwd... \e[0m" + cp -f /etc/passwd /etc/shifter/shifter_etc_files/passwd + + echo -e "\e[34;1m==>\e[39;1m Copying /etc/group... \e[0m" + cp -f /etc/group /etc/shifter/shifter_etc_files/group + + echo -e "\e[34;1m==>\e[39;1m Loading kernel modules... \e[0m" + [ -z "$(lsmod | grep ^ext4)" ] && modprobe ext4 + [ -n "$(lsmod | grep ^ext4)" ] && echo -e " \e[34;1m==>\e[39;1m ext4... loaded\e[0m" + [ -z "$(lsmod | grep ^loop)" ] && modprobe loop + [ -n "$(lsmod | grep ^loop)" ] && echo -e " \e[34;1m==>\e[39;1m loop... loaded\e[0m" + [ -z "$(lsmod | grep ^squashfs)" ] && modprobe squashfs + [ -n "$(lsmod | grep ^squashfs)" ] && echo -e " \e[34;1m==>\e[39;1m squashfs... loaded\e[0m" + + echo -e "\e[39;1m- Please make sure that the Shifter runtimes and the image gateway has the same munge key.\e[0m" + echo -e "\e[39;1m- Check the configuration file (/etc/shifter/udiRoot.conf) before pulling images or starting containers.\e[0m" +} + +post_upgrade() { + echo -e "\e[34;1m==>\e[39;1m Enabling munge.service... \e[0m" + MUNGE_ENABLED="$(systemctl list-unit-files | grep enabled | grep munge.service)" + [ -z "$MUNGE_ENABLED" ] && systemctl enable munge.service + + echo -e "\e[34;1m==>\e[39;1m Starting munge.service... \e[0m" + MUNGE_STARTED="$(systemctl | grep running | grep munge.service)" + [ -z "$MUNGE_STARTED" ] && systemctl start munge.service + + echo -e "\e[34;1m==>\e[39;1m Copying /etc/passwd... \e[0m" + cp -f /etc/passwd /etc/shifter/shifter_etc_files/passwd + + echo -e "\e[34;1m==>\e[39;1m Copying /etc/group... \e[0m" + cp -f /etc/group /etc/shifter/shifter_etc_files/group +} + +pre_remove() { + echo -e "\e[34;1m==>\e[39;1m Deleting /etc/shifter/shifter_etc_files/passwd... \e[0m" + rm -f /etc/shifter/shifter_etc_files/passwd + + echo -e "\e[34;1m==>\e[39;1m Deleting /etc/shifter/shifter_etc_files/group... \e[0m" + rm -f /etc/shifter/shifter_etc_files/group +} diff --git a/start-imagegw.sh b/start-imagegw.sh new file mode 100755 index 000000000000..491129ba0c2b --- /dev/null +++ b/start-imagegw.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +if [ ! -f /etc/shifter/imagemanager.json ]; then + echo 'error: /etc/shifter/imagemanager.json does not exist' + exit 1 +fi +SYSTEMS=($(jq '.Platforms | keys[]' /etc/shifter/imagemanager.json | sed -e 's/"//g')) +ROOT_TREE=/opt/shifter/imagegw + +for QA in ${SYSTEMS}; do + echo "Starting Celery Queue $QA" + celery2 -A shifter_imagegw.imageworker worker -Q $QA --loglevel=INFO -n worker.queue.$QA & +done + +echo "Starting imagegw API" +python2 $ROOT_TREE/imagegwapi.py & diff --git a/transfer.patch b/transfer.patch new file mode 100644 index 000000000000..360bc0f22ec4 --- /dev/null +++ b/transfer.patch @@ -0,0 +1,13 @@ +--- imagegw/shifter_imagegw/transfer.py 2017-05-08 23:53:14.000000000 +0800 ++++ imagegw/shifter_imagegw/transfer.py 2017-07-31 00:16:39.167788950 +0800 +@@ -170,6 +170,10 @@ + try: + mv_cmd = sh_cmd(system, 'mv', temp_fn, target_fn) + ret = _exec_and_log(mv_cmd, logger) ++ if ret == 0: ++ ## fix permissions on the image file ++ chmod_cmd = sh_cmd(system, 'chmod', 'a+r', target_fn) ++ ret = _exec_and_log(chmod_cmd, logger) + return ret == 0 + except: + ### TODO we might also need to remove target_fn in this case |