syncrepo


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

#!/bin/bash
set -euo pipefail

# based on https://git.archlinux.org/infrastructure.git/tree/roles/syncrepo/files/syncrepo-template.sh
# This is a simple mirroring script. To save bandwidth it first checks a
# timestamp via HTTP and only runs rsync when the timestamp differs from the
# local copy. As of 2016, a single rsync run without changes transfers roughly
# 6MiB of data which adds up to roughly 250GiB of traffic per month when rsync
# is run every minute. Performing a simple check via HTTP first can thus save a
# lot of traffic.

if tty -s; then
	. /usr/lib/syncrepo/syncrepo.conf
	if [ -f /etc/conf.d/syncrepo ]; then
		. /etc/conf.d/syncrepo
	fi
fi

SOURCE_URL="${SOURCE_URL:?Missing source url, please edit /etc/conf.d/syncrepo}"
LASTUPDATE_URL="${LASTUPDATE_URL:-"${SOURCE_URL/rsync/http}/lastupdate"}"
VERBOSE="${VERBOSE:-""}"

FORCE_SYNC="${1:-""}"
if [[ "$FORCE_SYNC" != "--force" ]]; then
	FORCE_SYNC=""
fi


[ -d "${TARGET}" ] || mkdir -p "${TARGET}"
[ -d "${TMP}" ] || mkdir -p "${TMP}"

exec 9>"${LOCK}"
flock -n 9 || exit

rsync_cmd() {
	local -a cmd=(rsync -rtlH --safe-links --delete-after ${VERBOSE} "--timeout=600" "--contimeout=60" -p \
		--delay-updates --no-motd "--temp-dir=${TMP}")

	if stty &>/dev/null; then
		cmd+=(-h -v --progress)
	else
		cmd+=(--quiet)
	fi

	if ((BWLIMIT>0)); then
		cmd+=("--bwlimit=$BWLIMIT")
	fi

	if ((EXCLUDE_ISO>0)); then
		cmd+=("--exclude=/iso")
	fi

	"${cmd[@]}" "$@"
}


# first check if there was a change by comparing the lastupdate files
if [[ -z "$FORCE_SYNC" ]] && [[ -f "$TARGET/lastupdate" ]] && (($(curl -Ls "$LASTUPDATE_URL")<=$(<"$TARGET/lastupdate"))); then
	echo "lastupdate file has not changed, not syncing." 
	if tty -s; then
		echo "To force sync use: $0 --force"
	fi
	# keep lastsync file in sync for statistics generated by the Arch Linux website
	rsync_cmd "$SOURCE_URL/lastsync" "$TARGET/lastsync"
	exit 0
fi

rsync_cmd \
	--exclude='*.links.tar.gz*' \
	--exclude='/other' \
	--exclude='/sources' \
	"${SOURCE_URL}" \
	"${TARGET}"

#echo "Last sync was $(date -d @$(cat ${target}/lastsync))"