diff options
| -rw-r--r-- | .SRCINFO | 34 | ||||
| -rw-r--r-- | LICENSE | 13 | ||||
| -rw-r--r-- | PKGBUILD | 47 | ||||
| -rw-r--r-- | README | 83 | ||||
| -rwxr-xr-x | arch.build | 93 | ||||
| -rwxr-xr-x | arch.install | 79 | ||||
| -rw-r--r-- | cirrus.conf | 2 | ||||
| -rw-r--r-- | fstab | 5 | ||||
| -rw-r--r-- | pacman.conf | 102 | ||||
| -rwxr-xr-x | proxy.cgi | 32 | ||||
| -rw-r--r-- | sudoers.d | 1 | ||||
| -rw-r--r-- | syslinux.cfg | 5 |
12 files changed, 496 insertions, 0 deletions
diff --git a/.SRCINFO b/.SRCINFO new file mode 100644 index 000000000000..90c439e429b7 --- /dev/null +++ b/.SRCINFO @@ -0,0 +1,34 @@ +pkgbase = buildstuff + pkgdesc = Stuff to build packages in virtual machines + pkgver = 0.8 + pkgrel = 1 + url = http://arch.vesath.org/ + arch = any + license = ISC + depends = qemu + depends = openssh + depends = e2fsprogs + depends = jfsutils + source = arch.build + source = arch.install + source = cirrus.conf + source = fstab + source = pacman.conf + source = proxy.cgi + source = sudoers.d + source = syslinux.cfg + source = LICENSE + source = README + sha1sums = fb8d7856d925bd92f1e9ba84d1a75ea43e107844 + sha1sums = 0a0a12473122c73c2a536c9a77d46cfd9e5a1424 + sha1sums = 315fc0847eaf46e74a92b4c62c0cd3ef73d1e89a + sha1sums = e7e2949822a35d2e70f826e3e942f40c5c6f41bb + sha1sums = 2b709786e16586baf28278292472e58dea10da5f + sha1sums = ef7077f9e61da1eb53a3ba5ea096daf8d733cba7 + sha1sums = 7d19d155ab1887ac1f84e682cbec54078c63758d + sha1sums = c3ea9e9250bba2fbfa591fc85065e0dabe4b8e46 + sha1sums = 352a7d6d53c2945f8fbbfff47d4f1dccc8d33829 + sha1sums = b0071da8f3dd319be0b8c637349c949ae5e71291 + +pkgname = buildstuff + diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000000..c5fa00e2ba81 --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +Copyright (C) 2012, Gaetan Bisson <bisson@archlinux.org>. + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. diff --git a/PKGBUILD b/PKGBUILD new file mode 100644 index 000000000000..d107e73e584b --- /dev/null +++ b/PKGBUILD @@ -0,0 +1,47 @@ +# Maintainer: Gaetan Bisson <bisson@archlinux.org> + +pkgname=buildstuff +pkgver=0.8 +pkgrel=1 +pkgdesc='Stuff to build packages in virtual machines' +url='http://arch.vesath.org/' +arch=('any') +license=('ISC') +depends=('qemu' 'openssh' 'e2fsprogs' 'jfsutils') + +source=('arch.build' + 'arch.install' + 'cirrus.conf' + 'fstab' + 'pacman.conf' + 'proxy.cgi' + 'sudoers.d' + 'syslinux.cfg' + 'LICENSE' + 'README') + +package() { + cd "${srcdir}" + install -d "${pkgdir}/var/lib/${pkgname}" + install -Dm755 arch.build "${pkgdir}/usr/bin/arch.build" + install -Dm755 arch.install "${pkgdir}/usr/bin/arch.install" + install -Dm755 proxy.cgi "${pkgdir}/usr/share/${pkgname}/proxy.cgi" + install -Dm644 pacman.conf "${pkgdir}/usr/share/${pkgname}/pacman.conf" + install -Dm644 fstab "${pkgdir}/usr/share/${pkgname}/guest/etc/fstab" + install -Dm644 sudoers.d "${pkgdir}/usr/share/${pkgname}/guest/etc/sudoers.d/user" + install -Dm644 cirrus.conf "${pkgdir}/usr/share/${pkgname}/guest/etc/modprobe.d/cirrus.conf" + install -Dm644 syslinux.cfg "${pkgdir}/usr/share/${pkgname}/guest/boot/syslinux/syslinux.cfg" + install -Dm644 LICENSE "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE" + install -Dm644 README "${pkgdir}/usr/share/doc/${pkgname}/README" +} + +sha1sums=('fb8d7856d925bd92f1e9ba84d1a75ea43e107844' + '0a0a12473122c73c2a536c9a77d46cfd9e5a1424' + '315fc0847eaf46e74a92b4c62c0cd3ef73d1e89a' + 'e7e2949822a35d2e70f826e3e942f40c5c6f41bb' + '2b709786e16586baf28278292472e58dea10da5f' + 'ef7077f9e61da1eb53a3ba5ea096daf8d733cba7' + '7d19d155ab1887ac1f84e682cbec54078c63758d' + 'c3ea9e9250bba2fbfa591fc85065e0dabe4b8e46' + '352a7d6d53c2945f8fbbfff47d4f1dccc8d33829' + 'b0071da8f3dd319be0b8c637349c949ae5e71291') diff --git a/README b/README new file mode 100644 index 000000000000..4435055b45bc --- /dev/null +++ b/README @@ -0,0 +1,83 @@ +IF YOU ARE IN A HURRY +===================== + +Go to any directory that contains a PKGBUILD and run + + arch.build i686 + +This will build your package for the given architecture. You may also append a +list of repositories to use on top of [core], [extra], and [community], for +instance: + + arch.build x86_64 testing multilib-testing multilib + + +FOR BETTER PERFORMANCES +======================= + +Make sure that hardware virtualization is enabled. + +Make sure that your /tmp is a tmpfs; this is where the copy-on-write drive is +stored and there is no point that it ever be synced to disk. + +Make your host and guest share their pacman package cache through the use of +the /usr/share/buildstuff/proxy.cgi script; proceed as follows: +- install any HTTP server on your host computer +- make this sript run as CGI when called as http://localhost/proxy.cgi +- make /var/cache/pacman/pkg writeable by the user running this CGI instance + + +HOW THIS ALL WORKS +================== + +The arch.install script creates a base Arch Linux disk image, and configures it +for unattended use in qemu. You may simply call it as: + + arch.install i686 ~/vm.img + +It creates a sparse disk image and partitions it as 8 GB swap, 64 MB ext2 boot, +8 GB JFS root, and 16 GB JFS home (where packages are built), and installs it +with [base] and [base-devel] using syslinux as boot loader and systemd as init. +All this is to reduce VM latency as much as possible. Eventually the disk image +amounts roughly to 700 MB of actual disk space. + +Any other Arch installer would do too, provided that: +- the guest system has a "user" user able to sudo at will +- the guest system runs an SSH server at bootup +- the host user is able to access the guest "user" user account + +The latter is achieved in arch.install by copying the host user's SSH public +keys into the guest user's authorized_keys file. + +Once created, these disk images will be used as read-only most of the time: the +virtual machines run in arch.build do copy-on-write on top of those base disk +images, storing the changes into a temporary file discarded after each use. The +base images are then only updated if they have not been in a day. + +Let us go through what happens step by step in arch.build: + +1. A (hopefully unique to this instance) random number gets picked: vport; it +is used as the dmsetup device name, port of the host that gets forwarded to the +guest SSH's, and name for the temporary copy-on-write drive. + +2. If the base disk image does not exist yet, create it by calling arch.install + +3. If the base disk image has not been updated for a day, run it within qemu +and update it. Note that this image never runs any repository beyond [core], +[extra], and [community]. + +4. Create the temporary drive and set up the dmsetup copy-on-write instance. +Run this within qemu. + +5. Add to this qemu instance the repositories that the user provided as +arguments; for instance, above, that was [testing], [multilib-testing], and +[multilib] so that the former takes precedence over the latter. + +6. Update the qemu instance. + +7. Run the build job. + +8. If everything went well, run cleanup instructions located in cleanup.$vport; +if anything went wrong, arch.build aborts and it is up to you to kill the qemu +instance and then run cleanup.$vport after you have investigated the build +problem. diff --git a/arch.build b/arch.build new file mode 100755 index 000000000000..c0cdefb7b4cc --- /dev/null +++ b/arch.build @@ -0,0 +1,93 @@ +#!/bin/bash + +set -e + +arch=$1 +shift +repo=$@ + +case $arch in + x86_64) cpu=qemu64 ;; + i686) cpu=qemu32 ;; + *) exit 1 ;; +esac + +img=/var/lib/buildstuff/$arch.img +vport=$((32768+$RANDOM)) +vhost=" + -o ConnectTimeout=5 + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o Port=$vport user@localhost +" + +start_vm() { + local kvm + pacman -Qoq qemu-system-x86_64 | grep -q kvm || kvm='-machine accel=kvm:tcg' + qemu-system-x86_64 \ + -smp 2 -m 768 -cpu $cpu $kvm \ + -device virtio-net,netdev=vlan \ + -netdev user,id=vlan,hostfwd=::$vport-:22 \ + -drive file=$1,cache=none,aio=native \ + -display none & + while sleep 3; do ssh $vhost true && break; done +} + +stop_vm() { + ssh $vhost 'sudo poweroff' || + pkill -f $vport || + true + wait +} + +trap_err() { + trap - ERR + trap - INT + cat <<EOF + +Please investigate the error that occured via \`ssh -p $vport user@localhost\`; +when you are ready, you may press enter to stop the virtual machine. +EOF + read x + stop_vm + sudo sh cleanup.$vport || + echo "Run: \`cd `pwd`; sudo sh cleanup.$vport\`" +} +trap trap_err ERR +trap trap_err INT +set -E + + +[[ -e $img ]] || { + sudo arch.install $arch $img + sudo chown "$USER" $img +} + +(( $(stat -c%Y $img) < $(date +%s) - 86400 )) && { + start_vm $img + ssh $vhost 'sudo pacman --noconfirm -Syu' + ssh $vhost 'sudo pacman --noconfirm -Su' + stop_vm +} + +dd if=/dev/zero count=0 bs=4K seek=1M of=/tmp/$vport +loopn=$(sudo losetup -f --show /tmp/$vport) +loopr=$(sudo losetup -f --show $img) +echo "dmsetup remove $vport && losetup -d $loopn $loopr && rm /tmp/$vport cleanup.$vport" > cleanup.$vport +sudo dmsetup create $vport --table "0 $(sudo blockdev --getsize $loopr) snapshot $loopr $loopn N 1" +sudo chown "$USER" /dev/mapper/$vport + +start_vm /dev/mapper/$vport +for r in $repo; do + echo "sudo sed '/\[core\]/i [$r]\nInclude = /etc/pacman.d/mirrorlist\n' -i /etc/pacman.conf" +done | ssh $vhost +ssh $vhost 'sudo pacman --noconfirm -Syu' +ssh $vhost 'sudo pacman --noconfirm -Su' +sftp $vhost <<<'mkdir build' +sftp $vhost <<<'put * build' +sftp $vhost <<<"put $HOME/.makepkg.conf" +ssh $vhost 'cd build; makepkg --log --noconfirm --syncdeps' +sftp $vhost <<<'get build/*.pkg.tar.xz' +sftp $vhost <<<'get build/*.log' +stop_vm +sudo sh cleanup.$vport diff --git a/arch.install b/arch.install new file mode 100755 index 000000000000..094db0e5f9c5 --- /dev/null +++ b/arch.install @@ -0,0 +1,79 @@ +#!/bin/bash + +set -e + +(( ! $UID )) + +arch=${1-x86_64} +img=${2-arch.img} + +(( $(</sys/module/loop/parameters/max_part) )) || { + modprobe -r loop || true + modprobe loop max_part=9 max_loop=9 +} + +[[ -e ${img} ]] || dd if=/dev/zero count=0 bs=4K seek=8M of="${img}" +sda=$(losetup -f --show "${img}") + +sfdisk -u M ${sda} <<\EOF +,8192 +,64 +,8192 +, +EOF + +mkswap ${sda}p1 +mke2fs -q ${sda}p2 +jfs_mkfs -q ${sda}p3 +jfs_mkfs -q ${sda}p4 + +install -d "${img}.mnt" +mount ${sda}p3 "${img}.mnt" +pushd "${img}.mnt" + +install -d boot home dev proc sys tmp +mount ${sda}p2 boot +mount ${sda}p4 home +mount -o bind /dev dev +mount -t proc proc proc +mount -t sysfs sys sys +mount -t tmpfs tmp tmp + +install -d var/lib/pacman +setarch "${arch}" pacman --config /usr/share/buildstuff/pacman.conf --root . \ + --noconfirm -Sy base base-devel openssh sudo syslinux + +find /usr/share/buildstuff/guest/ -type f | +while read i; do install -Dm644 "${i}" "${i##*/guest/}"; done + +install -Dm644 /etc/pacman.d/mirrorlist etc/pacman.d/mirrorlist +install -d home/user/.ssh; cat ~/.ssh/id_*.pub > home/user/.ssh/authorized_keys +sed -i etc/pacman.d/mirrorlist -e '1i Server = http://10.0.2.2/proxy.cgi/$repo/os/$arch/' + +setarch "${arch}" chroot . bash -c ' + mkinitcpio -S autodetect -k /boot/vmlinuz-linux -g /boot/initramfs-linux.img + syslinux-install_update -i -a -m + systemctl enable multi-user.target + systemctl enable {dhcpcd,sshd}.service + + useradd -p 00as1wm0AZG56 -d /home/user user + chown -R user:users /home/user + passwd -l root + + sed "s/--quiet --lsign-key/--batch --yes --quiet --lsign-key/" -i /usr/bin/pacman-key + pacman-key --init + pacman-key --populate archlinux + sed "s/--batch --yes//" -i /usr/bin/pacman-key +' + +umount dev +umount proc +umount sys +umount tmp +umount home +umount boot +popd +umount "${img}.mnt" +rmdir "${img}.mnt" + +losetup -d ${sda} diff --git a/cirrus.conf b/cirrus.conf new file mode 100644 index 000000000000..55054c01a467 --- /dev/null +++ b/cirrus.conf @@ -0,0 +1,2 @@ +# kvm breaks qemu's text-mode ncurses output +blacklist cirrus diff --git a/fstab b/fstab new file mode 100644 index 000000000000..60d75aeea3b8 --- /dev/null +++ b/fstab @@ -0,0 +1,5 @@ +tmpfs /tmp tmpfs nodev,nosuid 0 0 +/dev/sda1 swap swap defaults 0 0 +/dev/sda2 /boot ext2 defaults 0 1 +/dev/sda3 / jfs defaults 0 1 +/dev/sda4 /home jfs defaults 0 2 diff --git a/pacman.conf b/pacman.conf new file mode 100644 index 000000000000..df5aec796f9b --- /dev/null +++ b/pacman.conf @@ -0,0 +1,102 @@ +# +# /etc/pacman.conf +# +# See the pacman.conf(5) manpage for option and repository directives + +# +# GENERAL OPTIONS +# +[options] +# The following paths are commented out with their default values listed. +# If you wish to use different paths, uncomment and update the paths. +#RootDir = / +#DBPath = /var/lib/pacman/ +#CacheDir = /var/cache/pacman/pkg/ +#LogFile = /var/log/pacman.log +#GPGDir = /etc/pacman.d/gnupg/ +HoldPkg = pacman glibc +#XferCommand = /usr/bin/curl -C - -f %u > %o +#XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u +#CleanMethod = KeepInstalled +#UseDelta = 0.7 +Architecture = auto + +# Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup +#IgnorePkg = +#IgnoreGroup = + +#NoUpgrade = +#NoExtract = + +# Misc options +#UseSyslog +Color +TotalDownload +CheckSpace +VerbosePkgLists + +# By default, pacman accepts packages signed by keys that its local keyring +# trusts (see pacman-key and its man page), as well as unsigned packages. +SigLevel = Required DatabaseOptional +LocalFileSigLevel = Optional +#RemoteFileSigLevel = Required + +# NOTE: You must run `pacman-key --init` before first using pacman; the local +# keyring can then be populated with the keys of all official Arch Linux +# packagers with `pacman-key --populate archlinux`. + +# +# REPOSITORIES +# - can be defined here or included from another file +# - pacman will search repositories in the order defined here +# - local/custom mirrors can be added here or in separate files +# - repositories listed first will take precedence when packages +# have identical names, regardless of version number +# - URLs will have $repo replaced by the name of the current repo +# - URLs will have $arch replaced by the name of the architecture +# +# Repository entries are of the format: +# [repo-name] +# Server = ServerName +# Include = IncludePath +# +# The header [repo-name] is crucial - it must be present and +# uncommented to enable the repo. +# + +# The testing repositories are disabled by default. To enable, uncomment the +# repo name header and Include lines. You can add preferred servers immediately +# after the header, and they will be used before the default mirrors. + +#[testing] +#Include = /etc/pacman.d/mirrorlist + +[core] +Include = /etc/pacman.d/mirrorlist + +[extra] +Include = /etc/pacman.d/mirrorlist + +#[community-testing] +#Include = /etc/pacman.d/mirrorlist + +[community] +Include = /etc/pacman.d/mirrorlist + +# If you want to run 32 bit applications on your x86_64 system, +# enable the multilib repositories as required here. + +#[multilib-testing] +#Include = /etc/pacman.d/mirrorlist + +#[multilib] +#Include = /etc/pacman.d/mirrorlist + +# An example of a custom package repository. See the pacman manpage for +# tips on creating your own repositories. +#[custom] +#SigLevel = Optional TrustAll +#Server = file:///home/custompkgs + +[vesath] +Server = http://arch.vesath.org/$arch/ diff --git a/proxy.cgi b/proxy.cgi new file mode 100755 index 000000000000..6ee2e8079b5a --- /dev/null +++ b/proxy.cgi @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +# This CGI script is a caching Arch Linux mirror proxy: it forwards new package +# requests to an actual mirror while caching content to /var/cache/pacman/pkg. +# Use in pacman.conf with: +# +# Server = http://server/path/to/proxy.cgi/$repo/os/$arch/ + +[[ $REMOTE_ADDR = 127.0.0.1 ]] +cd /var/cache/pacman/pkg +touch used-by-proxy + +read MIR < /etc/pacman.d/mirrorlist +MIR=${MIR%%\$*} +MIR=${MIR##*= } + +[[ $PATH_INFO = /vesath/* ]] && MIR='http://arch.vesath.org' + +PKG=${PATH_INFO##*/} +[[ $PKG = *.db* ]] && exec curl --silent --include "$MIR$PATH_INFO" + +cat <<EOF +Content-Type: application/octet-stream +Connection: close + +EOF + +[[ -e $PKG ]] && exec cat "$PKG" +curl --silent "$MIR$PATH_INFO" | tee "$PKG.part" +mv "$PKG.part" "$PKG" diff --git a/sudoers.d b/sudoers.d new file mode 100644 index 000000000000..5841129998fe --- /dev/null +++ b/sudoers.d @@ -0,0 +1 @@ +user ALL=(ALL) NOPASSWD: ALL diff --git a/syslinux.cfg b/syslinux.cfg new file mode 100644 index 000000000000..95a066cdc053 --- /dev/null +++ b/syslinux.cfg @@ -0,0 +1,5 @@ +DEFAULT linux +LABEL linux +LINUX ../vmlinuz-linux +APPEND root=/dev/sda3 ro +INITRD ../initramfs-linux.img |