refactor

Signed-off-by: Yurii Kolesnykov <root@yurikoles.com>

6 files changed, 242 insertions, 71 deletions

diff --git a/.SRCINFO b/.SRCINFO
index e71b6d4ddd78..668cc8536439 100644
--- a/.SRCINFO
+++ b/.SRCINFO
@@ -1,24 +1,31 @@
 pkgbase = zfs-utils-git
 	pkgdesc = Userspace utilities for the Zettabyte File System.
-	pkgver = 2.0.0rc1.r38.gcd80273909
+	pkgver = 2.1.99.r1271.g1c0c729ab4
 	pkgrel = 1
 	epoch = 2
 	url = https://zfsonlinux.org/
-	arch = i686
 	arch = x86_64
+	groups = zfs-git
 	license = CDDL
+	makedepends = python
+	makedepends = python-setuptools
+	makedepends = python-cffi
 	makedepends = git
-	optdepends = python: for arcstat/arc_summary/dbufstat
-	provides = zfs-utils=2.0.0rc1.r38.gcd80273909
+	optdepends = python: pyzfs and extra utilities,
+	optdepends = python-cffi: pyzfs
+	provides = zfs-utils=2.1.99.r1271.g1c0c729ab4
 	conflicts = zfs-utils
 	source = git+https://github.com/zfsonlinux/zfs.git
 	source = zfs.initcpio.install
 	source = zfs.initcpio.hook
+	source = zfs.initcpio.zfsencryptssh.install
 	sha256sums = SKIP
-	sha256sums = da1cdc045d144d2109ec7b5d97c53a69823759d8ecff410e47c3a66b69e6518d
-	sha256sums = f95ad1a5421ccbb8b01f448373f46cfd1f718361a82c2687a597325cf9827e3e
+	sha256sums = 600f49d610906476f492d53ee1830154e4ebedf999284485e07d9cb2b3084766
+	sha256sums = 8b8c9b6ebfddfb51f2ab70fb943f53f08f6140140561efcb106120941edbc36e
+	sha256sums = 93e6ac4e16f6b38b2fa397a63327bcf7001111e3a58eb5fb97c888098c932a51
 	b2sums = SKIP
-	b2sums = 570e995bba07ea0fb424dff191180b8017b6469501964dc0b70fd51e338a4dad260f87cc313489866cbfd1583e4aac2522cf7309c067cc5314eb83c37fe14ff3
-	b2sums = 491a7f20b0c6b4ce4fcedab617b2d7a4f2a01f1bc8f1ae57efde2fb22c2ab09a1107a8f4877b95c27576fe4216d01f181936787f51ce532bb13c5806badf7519
+	b2sums = 5147f165bc53cb792aaf11724cef253601fe853cd7bc43aeff0ef5cd3a23dbde57e38710c941803d6b5f5838bde99271804608221f1c6b33f30b734edbd85913
+	b2sums = 32352e2e188073da4f61278899b6d343313ee3494d69dd10a38ab87bce2f2003767f0f49bc1c3c4d785dd1fc67eab4a27a6fdd5ffc5e63cf94d25f3dbffae4c1
+	b2sums = 04e2af875e194df393d6cff983efc3fdf02a03a745d1b0b1e4a745f873d910b4dd0a45db956c1b5b2d97e9d5bf724ef12e23f7a2be3d5c12be027eaccf42349a
 
 pkgname = zfs-utils-git
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 000000000000..c3e43b1da9f3
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,5 @@
+zfs
+src
+pkg
+*.pkg.*
+*.log
diff --git a/PKGBUILD b/PKGBUILD
index 481e8a1382da..a18cf467a49f 100644
--- a/PKGBUILD
+++ b/PKGBUILD
@@ -1,66 +1,67 @@
-# Maintainer: Eli Schwartz <eschwartz@archlinux.org>
+# Maintainer: Yurii Kolesnykov <root@yurikoles.com>
+# Contributor: Eli Schwartz <eschwartz@archlinux.org>
 # Contributor: Iacopo Isimbaldi <isiachi@rhye.it>
 
-# All my PKGBUILDs are managed at https://github.com/eli-schwartz/pkgbuilds
+# PRs are welcome: https://github.com/yurikoles-aur/zfs-utils-git
 
 pkgname=zfs-utils-git
-pkgver=2.0.0rc1.r38.gcd80273909
+pkgver=2.1.99.r1271.g1c0c729ab4
 pkgrel=1
 epoch=2
 pkgdesc="Userspace utilities for the Zettabyte File System."
-arch=("i686" "x86_64")
-url="https://zfsonlinux.org/"
+arch=('x86_64')
+url='https://zfsonlinux.org/'
 license=('CDDL')
-optdepends=('python: for arcstat/arc_summary/dbufstat')
-makedepends=('git')
-conflicts=("${pkgname%-git}")
+groups=('zfs-git')
+makedepends=('python' 'python-setuptools' 'python-cffi' 'git')
+optdepends=('python: pyzfs and extra utilities', 'python-cffi: pyzfs')
 provides=("${pkgname%-git}=${pkgver}")
-source=("git+https://github.com/zfsonlinux/zfs.git"
-        "zfs.initcpio.install"
-        "zfs.initcpio.hook")
+conflicts=("${pkgname%-git}")
+source=('git+https://github.com/zfsonlinux/zfs.git'
+        'zfs.initcpio.install'
+        'zfs.initcpio.hook'
+        'zfs.initcpio.zfsencryptssh.install')
 sha256sums=('SKIP'
-            'da1cdc045d144d2109ec7b5d97c53a69823759d8ecff410e47c3a66b69e6518d'
-            'f95ad1a5421ccbb8b01f448373f46cfd1f718361a82c2687a597325cf9827e3e')
+            '600f49d610906476f492d53ee1830154e4ebedf999284485e07d9cb2b3084766'
+            '8b8c9b6ebfddfb51f2ab70fb943f53f08f6140140561efcb106120941edbc36e'
+            '93e6ac4e16f6b38b2fa397a63327bcf7001111e3a58eb5fb97c888098c932a51')
 b2sums=('SKIP'
-        '570e995bba07ea0fb424dff191180b8017b6469501964dc0b70fd51e338a4dad260f87cc313489866cbfd1583e4aac2522cf7309c067cc5314eb83c37fe14ff3'
-        '491a7f20b0c6b4ce4fcedab617b2d7a4f2a01f1bc8f1ae57efde2fb22c2ab09a1107a8f4877b95c27576fe4216d01f181936787f51ce532bb13c5806badf7519')
+        '5147f165bc53cb792aaf11724cef253601fe853cd7bc43aeff0ef5cd3a23dbde57e38710c941803d6b5f5838bde99271804608221f1c6b33f30b734edbd85913'
+        '32352e2e188073da4f61278899b6d343313ee3494d69dd10a38ab87bce2f2003767f0f49bc1c3c4d785dd1fc67eab4a27a6fdd5ffc5e63cf94d25f3dbffae4c1'
+        '04e2af875e194df393d6cff983efc3fdf02a03a745d1b0b1e4a745f873d910b4dd0a45db956c1b5b2d97e9d5bf724ef12e23f7a2be3d5c12be027eaccf42349a')
 
 pkgver() {
-    cd "${srcdir}"/zfs
+    cd zfs
 
     git describe --long | sed 's/^zfs-//;s/-rc/rc/;s/\([^-]*-g\)/r\1/;s/-/./g'
 }
 
 prepare() {
-    cd "${srcdir}"/zfs
-
-    # pyzfs is not built, but build system tries to check for python anyway
-    ln -sf /bin/true python3-fake
+    cd zfs
 
     autoreconf -fi
 }
 
 build() {
-    cd "${srcdir}"/zfs
+    cd zfs
 
-    ./configure --prefix=/usr \
-                --sysconfdir=/etc \
-                --sbindir=/usr/bin \
-                --with-mounthelperdir=/usr/bin \
-                --with-udevdir=/usr/lib/udev \
-                --libexecdir=/usr/lib/zfs \
-                --with-python="$PWD/python3-fake" \
-                --enable-pyzfs=no \
-                --enable-systemd \
-                --with-config=user
+    ./configure \
+        --prefix=/usr \
+        --sysconfdir=/etc \
+        --sbindir=/usr/bin \
+        --with-mounthelperdir=/usr/bin \
+        --with-udevdir=/usr/lib/udev \
+        --libexecdir=/usr/lib/zfs \
+        --enable-pyzfs \
+        --enable-systemd \
+        --with-config=user \
+        --with-zfsexecdir=/usr/lib/zfs
     make
 }
 
 package() {
-    cd "${srcdir}"/zfs
-
+    cd zfs
     make DESTDIR="${pkgdir}" install
-    install -D -m644 contrib/bash_completion.d/zfs "${pkgdir}"/usr/share/bash-completion/completions/zfs
 
     # Remove uneeded files
     rm -r "${pkgdir}"/etc/init.d
@@ -69,8 +70,10 @@ package() {
     #rm -r "${pkgdir}"/usr/lib/dracut
     rm -r "${pkgdir}"/usr/lib/modules-load.d
     rm -r "${pkgdir}"/usr/share/initramfs-tools
-    rm -r "${pkgdir}"/usr/share/zfs
 
+    # Install the support files
     install -D -m644 "${srcdir}"/zfs.initcpio.hook "${pkgdir}"/usr/lib/initcpio/hooks/zfs
     install -D -m644 "${srcdir}"/zfs.initcpio.install "${pkgdir}"/usr/lib/initcpio/install/zfs
+    install -D -m644 "${srcdir}"/zfs.initcpio.zfsencryptssh.install "${pkgdir}"/usr/lib/initcpio/install/zfsencryptssh
+    install -D -m644 contrib/bash_completion.d/zfs "${pkgdir}"/usr/share/bash-completion/completions/zfs
 }
diff --git a/zfs.initcpio.hook b/zfs.initcpio.hook
index 7288ab8018b3..7fecb13030c8 100644
--- a/zfs.initcpio.hook
+++ b/zfs.initcpio.hook
@@ -6,6 +6,7 @@
 #
 ZPOOL_FORCE=""
 ZPOOL_IMPORT_FLAGS=""
+ZFS_BOOT_ONLY=""
 
 zfs_get_bootfs () {
     for zfs_dataset in $(zpool list -H -o bootfs); do
@@ -25,13 +26,86 @@ zfs_get_bootfs () {
     return 1
 }
 
+zfs_decrypt_fs() {
+    dataset=$1
+
+    # Make sure dataset is encrypted; get fails if ZFS does not support encryption
+    encryption="$(zfs get -H -o value encryption "${dataset}" 2>/dev/null)" || return 0
+    [ "${encryption}" != "off" ] || return 0
+
+    # Make sure the dataset is locked
+    keystatus="$(zfs get -H -o value keystatus "${dataset}")" || return 0
+    [ "${keystatus}" != "available" ] || return 0
+
+    # Make sure the encryptionroot is sensible
+    encryptionroot="$(zfs get -H -o value encryptionroot "${dataset}")" || return 0
+    [ "${encryptionroot}" != "-" ] || return 0
+
+    # Export encryption root to be used by other hooks (SSH)
+    echo "${encryptionroot}" > /.encryptionroot
+
+    prompt_override=""
+    if keylocation="$(zfs get -H -o value keylocation "${encryptionroot}")"; then
+        # If key location is a file, determine if it can by overridden by prompt
+        if [ "${keylocation}" != "prompt" ]; then
+            if keyformat="$(zfs get -H -o value keyformat "${encryptionroot}")"; then
+                [ "${keyformat}" = "passphrase" ] && prompt_override="yes"
+            fi
+        fi
+
+        # If key location is a local file, check if file exists
+        if [ "${keylocation%%://*}" = "file" ]; then
+            keyfile="${keylocation#file://}"
+
+            # If file not yet exist, wait for udev to create device nodes
+            if [ ! -r "${keyfile}" ]; then
+                udevadm settle
+
+                # Wait for udev up to 10 seconds
+                if [ ! -r "${keyfile}" ]; then 
+                    echo "Waiting for key ${keyfile} for ${encryptionroot}..."
+                    for _ in $(seq 1 20); do
+                        sleep 0.5s
+                        [ -r "${keyfile}" ] && break
+                    done
+                fi
+
+                if [ ! -r "${keyfile}" ]; then
+                    echo "Key ${keyfile} for ${encryptionroot} hasn't appeared. Trying anyway."
+                fi
+            fi
+        fi
+    fi
+
+    # Loop until key is loaded here or by another vector (SSH, for instance)
+    while [ "$(zfs get -H -o value keystatus "${encryptionroot}")" != "available" ]; do
+        # Try the default loading mechanism
+        zfs load-key "${encryptionroot}" && break
+
+        # Load failed, try a prompt if the failure was not a prompt
+        if [ -n "${prompt_override}" ]; then
+            echo "Unable to load key ${keylocation}; please type the passphrase"
+            echo "To retry the file, interrupt now or repeatedly input a wrong passphrase"
+            zfs load-key -L prompt "${encryptionroot}" && break
+        fi
+
+        # Throttle retry attempts
+        sleep 2
+    done
+
+    if [ -f /.encryptionroot ]; then
+        rm /.encryptionroot
+    fi
+}
+
 zfs_mount_handler () {
     if [ "${ZFS_DATASET}" = "bootfs" ] ; then
         if ! zfs_get_bootfs ; then
             # Lets import everything and try again
             zpool import ${ZPOOL_IMPORT_FLAGS} -N -a ${ZPOOL_FORCE}
             if ! zfs_get_bootfs ; then
-                die "ZFS: Cannot find bootfs."
+                err "ZFS: Cannot find bootfs."
+                exit 1
             fi
         fi
     fi
@@ -39,7 +113,7 @@ zfs_mount_handler () {
     local pool="${ZFS_DATASET%%/*}"
     local rwopt_exp="${rwopt:-ro}"
 
-    if ! zpool list -H "${pool}" 2>&1 > /dev/null ; then
+    if ! zpool list -H "${pool}" > /dev/null 2>&1; then
         if [ ! "${rwopt_exp}" = "rw" ]; then
             msg "ZFS: Importing pool ${pool} readonly."
             ZPOOL_IMPORT_FLAGS="${ZPOOL_IMPORT_FLAGS} -o readonly=on"
@@ -48,48 +122,67 @@ zfs_mount_handler () {
         fi
 
         if ! zpool import ${ZPOOL_IMPORT_FLAGS} -N "${pool}" ${ZPOOL_FORCE} ; then
-            die "ZFS: Unable to import pool ${pool}."
+            err "ZFS: Unable to import pool ${pool}."
+            exit 1
         fi
     fi
 
     local node="$1"
+    local rootmnt=$(zfs get -H -o value mountpoint "${ZFS_DATASET}")
     local tab_file="${node}/etc/fstab"
     local zfs_datasets="$(zfs list -H -o name -t filesystem -r ${ZFS_DATASET})"
 
     # Mount the root, and any child datasets
     for dataset in ${zfs_datasets}; do
         mountpoint=$(zfs get -H -o value mountpoint "${dataset}")
-        case ${mountpoint} in
-            "none")
-                # skip this line/dataset.
-                ;;
-            "legacy")
-                if [ -f "${tab_file}" ]; then
-                    if findmnt -snero source -F "${tab_file}" -S "${dataset}" > /dev/null 2>&1; then
-                        opt=$(findmnt -snero options -F "${tab_file}" -S "${dataset}")
-                        mnt=$(findmnt -snero target -F "${tab_file}" -S "${dataset}")
-                        mount -t zfs -o "${opt}" "${dataset}" "${node}${mnt}"
-                    fi
+        canmount=$(zfs get -H -o value canmount "${dataset}")
+        # skip dataset
+        [ ${dataset} != "${ZFS_DATASET}" -a \( ${canmount} = "off" -o ${canmount} = "noauto" -o ${mountpoint} = "none" \) ] && continue
+        if [ ${mountpoint} = "legacy" ]; then
+            if [ -f "${tab_file}" ]; then
+                if findmnt -snero source -F "${tab_file}" -S "${dataset}" > /dev/null 2>&1; then
+                    opt=$(findmnt -snero options -F "${tab_file}" -S "${dataset}")
+                    mnt=$(findmnt -snero target -F "${tab_file}" -S "${dataset}")
+                    zfs_decrypt_fs "${dataset}"
+                    mount -t zfs -o "${opt}" "${dataset}" "${node}${mnt}"
                 fi
-                ;;
-            *)
-                mount -t zfs -o "zfsutil,${rwopt_exp}" "${dataset}" "${node}${mountpoint}"
-                ;;
-        esac
+            fi
+        else
+            zfs_decrypt_fs "${dataset}"
+            mount -t zfs -o "zfsutil,${rwopt_exp}" "${dataset}" "${node}/${mountpoint##${rootmnt}}"
+        fi
     done
 }
 
-run_hook() {
+set_flags() {
     # Force import the pools, useful if the pool has not properly been exported using 'zpool export <pool>'
     [ ! "${zfs_force}" = "" ] && ZPOOL_FORCE="-f"
 
+    # Disable late hook, useful if we want to use zfs-import-cache.service instead
+    [ ! "${zfs_boot_only}" = "" ] && ZFS_BOOT_ONLY="1"
+
     # Add import directory to import command flags
     [ ! "${zfs_import_dir}" = "" ] && ZPOOL_IMPORT_FLAGS="${ZPOOL_IMPORT_FLAGS} -d ${zfs_import_dir}"
+    [ "${zfs_import_dir}" = "" ] && [ -f /etc/zfs/zpool.cache.org ] && ZPOOL_IMPORT_FLAGS="${ZPOOL_IMPORT_FLAGS} -c /etc/zfs/zpool.cache.org"
+}
+
+run_hook() {
+    set_flags
 
     # Wait 15 seconds for ZFS devices to show up
     [ "${zfs_wait}" = "" ] && ZFS_WAIT="15" || ZFS_WAIT="${zfs_wait}"
 
-    [ "${root}" = "zfs" ] && mount_handler="zfs_mount_handler"
+    case ${root} in
+        # root=zfs
+        "zfs")
+            mount_handler="zfs_mount_handler"
+            ;;
+        # root=ZFS=... syntax (grub)
+        "ZFS="*)
+            mount_handler="zfs_mount_handler"
+            ZFS_DATASET="${root#*[=]}"
+            ;;
+    esac
 
     case ${zfs} in
         "")
@@ -98,22 +191,46 @@ run_hook() {
         auto|bootfs)
             ZFS_DATASET="bootfs"
             mount_handler="zfs_mount_handler"
+            local pool="[a-zA-Z][^ ]*"
             ;;
         *)
             ZFS_DATASET="${zfs}"
             mount_handler="zfs_mount_handler"
+            local pool="${ZFS_DATASET%%/*}"
             ;;
     esac
 
-    # Allow up to n seconds for zfs device to show up
-    for i in $(seq 1 ${ZFS_WAIT}); do
-        [ -c "/dev/zfs" ] && break
+    # Allow at least n seconds for zfs device to show up.  Especially
+    # when using zfs_import_dir instead of zpool.cache, the listing of
+    # available pools can be slow, so this loop must be top-tested to
+    # ensure we do one 'zpool import' pass after the timer has expired.
+    sleep ${ZFS_WAIT} & pid=$!
+    local break_after=0
+    while :; do
+        kill -0 $pid > /dev/null 2>&1 || break_after=1
+        if [ -c "/dev/zfs" ]; then
+            zpool import ${ZPOOL_IMPORT_FLAGS} | awk "
+                BEGIN     { pool_found=0; online=0; unavail=0 }
+                /^	${pool} .*/ { pool_found=1 }
+                /^\$/      { pool_found=0 }
+                /UNAVAIL/ { if (pool_found == 1) { unavail=1 } }
+                /ONLINE/  { if (pool_found == 1) { online=1 } }
+                END       { if (online == 1 && unavail != 1)
+                              { exit 0 }
+                            else
+                              { exit 1 }
+                          }" && break
+        fi
+        [ $break_after == 1 ] && break
         sleep 1
     done
+    kill $pid > /dev/null 2>&1
 }
 
 run_latehook () {
-    zpool import -N -a ${ZPOOL_FORCE}
+    set_flags
+    # only run zpool import, if flags were set (cache file found / zfs_import_dir specified) and zfs_boot_only is not set
+    [ ! "${ZPOOL_IMPORT_FLAGS}" = "" ] && [ "${ZFS_BOOT_ONLY}" = "" ] && zpool import ${ZPOOL_IMPORT_FLAGS} -N -a ${ZPOOL_FORCE}
 }
 
 # vim:set ts=4 sw=4 ft=sh et:
diff --git a/zfs.initcpio.install b/zfs.initcpio.install
index 589b46bee544..d51183b19e89 100644
--- a/zfs.initcpio.install
+++ b/zfs.initcpio.install
@@ -22,7 +22,8 @@ build() {
         zstreamdump \
         /lib/udev/vdev_id \
         /lib/udev/zvol_id \
-        findmnt
+        findmnt \
+        udevadm
 
     map add_file \
         /lib/udev/rules.d/60-zvol.rules \
@@ -38,9 +39,8 @@ build() {
     # allow mount(8) to "autodetect" ZFS
     echo 'zfs' >>"${BUILDROOT}/etc/filesystems"
 
-    [[ -f /etc/zfs/zpool.cache ]] && add_file "/etc/zfs/zpool.cache"
+    [[ -f /etc/zfs/zpool.cache ]] && cp "/etc/zfs/zpool.cache" "${BUILDROOT}/etc/zfs/zpool.cache.org"
     [[ -f /etc/modprobe.d/zfs.conf ]] && add_file "/etc/modprobe.d/zfs.conf"
-    [[ -f /etc/hostid ]] && add_file "/etc/hostid"
 }
 
 help() {
diff --git a/zfs.initcpio.zfsencryptssh.install b/zfs.initcpio.zfsencryptssh.install
new file mode 100644
index 000000000000..f724eeec5663
--- /dev/null
+++ b/zfs.initcpio.zfsencryptssh.install
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+make_etc_passwd() {
+    echo 'root:x:0:0:root:/root:/bin/zfsdecrypt_shell' >> "${BUILDROOT}"/etc/passwd
+    echo '/bin/zfsdecrypt_shell' > "${BUILDROOT}"/etc/shells
+}
+
+make_zfsdecrypt_shell() {
+    decrypt_shell='#!/bin/sh
+if [ -f "/.encryptionroot" ]; then
+    # source zfs hook functions
+    . /hooks/zfs
+    # decrypt bootfs
+    zfs_decrypt_fs "$(cat /.encryptionroot)"
+    # kill pending decryption attempt to allow the boot process to continue
+    killall zfs
+else
+    echo "ZFS is not ready yet. Please wait!"
+fi'
+    printf '%s' "$decrypt_shell" > "${BUILDROOT}"/bin/zfsdecrypt_shell
+    chmod a+x "${BUILDROOT}"/bin/zfsdecrypt_shell
+}
+
+build ()
+{
+    make_etc_passwd
+    make_zfsdecrypt_shell
+}
+
+help ()
+{
+    cat<<HELPEOF
+This hook is meant to be used in conjunction with mkinitcpio-dropbear, 
+mkinitcpio-netconf and/ormkinitcpio-ppp. This will provide a way to unlock 
+your encrypted ZFS root filesystem remotely.
+HELPEOF
+}
+
+# vim: set ts=4 sw=4 ft=sh et: