diff options
author | Eric Anderson | 2015-06-20 09:13:47 -0700 |
---|---|---|
committer | Eric Anderson | 2015-06-20 09:13:47 -0700 |
commit | 41136425acd8699e386e91e7fa25341bbf965c92 (patch) | |
tree | 597dafb69eef63986d630d8988a026ff86dabded | |
download | aur-41136425acd8699e386e91e7fa25341bbf965c92.tar.gz |
Initial import
-rw-r--r-- | .AURINFO | 23 | ||||
-rw-r--r-- | .SRCINFO | 23 | ||||
-rw-r--r-- | PKGBUILD | 31 | ||||
-rwxr-xr-x | pkgdistcache-client | 175 | ||||
-rwxr-xr-x | pkgdistcache-daemon | 139 | ||||
-rw-r--r-- | pkgdistcache.conf | 16 | ||||
-rw-r--r-- | pkgdistcache.install | 18 | ||||
-rw-r--r-- | pkgdistcached.service | 8 |
8 files changed, 433 insertions, 0 deletions
diff --git a/.AURINFO b/.AURINFO new file mode 100644 index 000000000000..4de8cf26e1de --- /dev/null +++ b/.AURINFO @@ -0,0 +1,23 @@ +pkgbase = pkgdistcache + pkgdesc = A distributed local-network cache for pacman packages + pkgver = 0.3.1 + pkgrel = 4 + url = http://venator.ath.cx/dw/doku.php?id=linux:pkgdistcache + install = pkgdistcache.install + arch = any + license = GPL + depends = avahi + depends = python2-dbus + depends = python2-gobject2 + depends = curl + source = pkgdistcache-client + source = pkgdistcache-daemon + source = pkgdistcache.conf + source = pkgdistcached.service + sha256sums = d53ba7417b6d6db3c36876f7ef92933553ce27597b94ce727deeef8c6edac1f8 + sha256sums = 812075ee7b50f576d9706e0f9aab48ccfb2ee1fd8c06d91476c90c0fe7d576d8 + sha256sums = d77ac418aa651bc622cd91204d6907554c6cdb4bb989e484cc54da32342faa51 + sha256sums = 5eb96f9e4bcec466d097ac46d72fd9626fb36bd61a3d3ceb1ca69706036f27c2 + +pkgname = pkgdistcache + diff --git a/.SRCINFO b/.SRCINFO new file mode 100644 index 000000000000..4de8cf26e1de --- /dev/null +++ b/.SRCINFO @@ -0,0 +1,23 @@ +pkgbase = pkgdistcache + pkgdesc = A distributed local-network cache for pacman packages + pkgver = 0.3.1 + pkgrel = 4 + url = http://venator.ath.cx/dw/doku.php?id=linux:pkgdistcache + install = pkgdistcache.install + arch = any + license = GPL + depends = avahi + depends = python2-dbus + depends = python2-gobject2 + depends = curl + source = pkgdistcache-client + source = pkgdistcache-daemon + source = pkgdistcache.conf + source = pkgdistcached.service + sha256sums = d53ba7417b6d6db3c36876f7ef92933553ce27597b94ce727deeef8c6edac1f8 + sha256sums = 812075ee7b50f576d9706e0f9aab48ccfb2ee1fd8c06d91476c90c0fe7d576d8 + sha256sums = d77ac418aa651bc622cd91204d6907554c6cdb4bb989e484cc54da32342faa51 + sha256sums = 5eb96f9e4bcec466d097ac46d72fd9626fb36bd61a3d3ceb1ca69706036f27c2 + +pkgname = pkgdistcache + diff --git a/PKGBUILD b/PKGBUILD new file mode 100644 index 000000000000..bc0378ccd44f --- /dev/null +++ b/PKGBUILD @@ -0,0 +1,31 @@ +# Contributor: Alessio Bianchi <venator85 at gmail dot com> +# Maintainer: Eric Anderson <ejona86@gmail.com> + +pkgname=pkgdistcache +pkgver=0.3.1 +pkgrel=4 +pkgdesc='A distributed local-network cache for pacman packages' +arch=('any') +url='http://venator.ath.cx/dw/doku.php?id=linux:pkgdistcache' +license=('GPL') +depends=('avahi' 'python2-dbus' 'python2-gobject2' 'curl') +install="${pkgname}.install" +source=('pkgdistcache-client' + 'pkgdistcache-daemon' + 'pkgdistcache.conf' + 'pkgdistcached.service') +sha256sums=('d53ba7417b6d6db3c36876f7ef92933553ce27597b94ce727deeef8c6edac1f8' + '812075ee7b50f576d9706e0f9aab48ccfb2ee1fd8c06d91476c90c0fe7d576d8' + 'd77ac418aa651bc622cd91204d6907554c6cdb4bb989e484cc54da32342faa51' + '5eb96f9e4bcec466d097ac46d72fd9626fb36bd61a3d3ceb1ca69706036f27c2') + +package() { + install -d "${pkgdir}/usr/bin/" + install -m755 "${srcdir}/pkgdistcache-client" "${pkgdir}/usr/bin/" + install -m755 "${srcdir}/pkgdistcache-daemon" "${pkgdir}/usr/bin/" + install -d "${pkgdir}/etc/" + install -m644 "${srcdir}/pkgdistcache.conf" "${pkgdir}/etc/" + install -d "${pkgdir}/usr/lib/systemd/system/" + install -m644 "${srcdir}/pkgdistcached.service" \ + "${pkgdir}/usr/lib/systemd/system/" +} diff --git a/pkgdistcache-client b/pkgdistcache-client new file mode 100755 index 000000000000..e051d26d18a2 --- /dev/null +++ b/pkgdistcache-client @@ -0,0 +1,175 @@ +#!/usr/bin/python2 +# coding: utf-8 +# +# pkgdistcache client v0.3.1 +# by Alessio Bianchi <venator85@gmail.com> +# + +import sys +import os +import os.path +import subprocess +import string +import avahi +import dbus +import gobject +import dbus.glib +import pickle + +colors = {'none': '\033[0m', + 'black': '\033[0;30m', 'bold_black': '\033[1;30m', + 'red': '\033[0;31m', 'bold_red': '\033[1;31m', + 'green': '\033[0;32m', 'bold_green': '\033[1;32m', + 'yellow': '\033[0;33m', 'bold_yellow': '\033[1;33m', + 'blue': '\033[0;34m', 'bold_blue': '\033[1;34m', + 'magenta': '\033[0;35m', 'bold_magenta': '\033[1;35m', + 'cyan': '\033[0;36m', 'bold_cyan': '\033[1;36m', + 'white': '\033[0;37m', 'bold_white': '\033[1;37m'} + +def printmsg(msg): + print "%s>> %s%s" % (colors['bold_blue'], msg, colors['none']) + +def printerr(msg): + print "%s!! %s%s" % (colors['bold_red'], msg, colors['none']) + +def printwarn(msg): + print "%s!! %s%s" % (colors['bold_yellow'], msg, colors['none']) + +# Run a command synchronously, redirecting stdout and stderr to strings +def runcmd(cmd, cwd=None): + pipe = subprocess.Popen(cmd, shell=True, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdout, stderr) = pipe.communicate() # wait for process to terminate and return stdout and stderr + return {'stdout': stdout.strip(), 'stderr': stderr.strip(), 'retcode': pipe.returncode} + +# Run a command synchronously, sending stdout and stderr to shell +def runcmd2(cmd, cwd=None): + pipe = subprocess.Popen(cmd, shell=True, cwd=cwd, stdout=None, stderr=None) + pipe.communicate() # wait for process to terminate + return pipe.returncode + +class Service(object): + def __init__(self, service, host, ip, port): + self.service = service + self.host = host + self.ip = ip + self.port = port + + def __str__(self): + return "(%s, %s, %s, %d)" % (self.service, self.host, self.ip, self.port) + + def __repr__(self): + # il tostring sulle liste รจ repr + return self.__str__() + + def __hash__(self): + return hash(self.__str__()) + + def __eq__(self, other): + return hash(self) == hash(other) + +class AvahiBrowser(object): + def __init__(self): + # Connect to the system bus... + self.bus = dbus.SystemBus() + # Get a proxy to the object we want to talk to. + avahi_proxy = self.bus.get_object(avahi.DBUS_NAME, avahi.DBUS_PATH_SERVER) + # Set the interface we want to use; server in this case. + self.server = dbus.Interface(avahi_proxy, avahi.DBUS_INTERFACE_SERVER) + self.version_string = self.server.GetVersionString() + self.domain = "local" + self.loop = gobject.MainLoop() + self.services = [] + + def browse(self, stype): + # Ask the server for a path to the browser object for the service we're interested in... + browser_path = self.server.ServiceBrowserNew(avahi.IF_UNSPEC, avahi.PROTO_UNSPEC, stype, self.domain, dbus.UInt32(0)) + # Get it's proxy object... + browser_proxy = self.bus.get_object(avahi.DBUS_NAME, browser_path) + # And set the interface we want to use + browser = dbus.Interface(browser_proxy, avahi.DBUS_INTERFACE_SERVICE_BROWSER) + + # Now connect the call backs to the relevant signals. + browser.connect_to_signal('ItemNew', self.new_service) + browser.connect_to_signal('AllForNow', self.all_for_now) + self.loop.run() + return self.services + + def new_service(self, interface, protocol, name, stype, domain, flags): + try: + s = self.server.ResolveService(interface, protocol, name, stype, domain, avahi.PROTO_UNSPEC, dbus.UInt32(0)) + service = Service(str(s[3]), str(s[2]), str(s[7]), int(s[8])) # service name, host, ip, port + self.services.append(service) + except TimeoutError: + printwarn("Timeout error during service resolution!") + + def all_for_now(self): + self.loop.quit() + +def main(argv): + # load configuration file + conf_file = '/etc/pkgdistcache.conf' + if os.path.isfile(conf_file): + config = eval(open(conf_file).read()) + else: + printerr("Config file " + conf_file + " not found") + return 2 + + download_cmd_template = string.Template(config['download_cmd']) + + pkg = os.path.basename(argv[1]) # argv[1] = %u passed by pacman + + must_download = True + if not pkg.endswith('.db.tar.gz'): + # find first /tmp/pkgdistcache.* file modified less than CACHE_FILE_LIFE minutes ago + cmd = 'find /tmp -name "pkgdistcache.*" -mmin -' + str(config['cache_file_life']) + cache_file = runcmd(cmd)['stdout'] + if len(cache_file) > 0: + # recent cache file found, use it + with open(cache_file, 'rb') as f: + pkgdistcache_clients = pickle.load(f) + else: + # recent cache file not found, discover other pkgdistcache capable hosts via avahi and save result to a new cache file + runcmd("rm -f /tmp/pkgdistcache.*") # remove any old cache file + cache_file = runcmd('mktemp /tmp/pkgdistcache.XXXXX')['stdout'] # create new cache file + + hostname = runcmd('hostname')['stdout'] + browser = AvahiBrowser() + clients = browser.browse("_pkgdistcache._tcp") + clients = set(clients) # remove duplicates (eg services offered on more than a network card etc.) + pkgdistcache_clients = [] + for client in clients: + if client.host != hostname: # exclude current machine from results + pkgdistcache_clients.append(client) + + with open(cache_file, 'wb') as f: + pickle.dump(pkgdistcache_clients, f, -1) + + for client in pkgdistcache_clients: + url = "http://" + client.ip + ":" + str(client.port) + "/" + pkg + dst = argv[2] + printmsg("Downloading " + pkg + " from host '" + client.host + "'") + download_cmd = download_cmd_template.substitute({'u': url, 'o': dst}) + try: + ret = runcmd2(download_cmd) + if ret == 0: + must_download = False + break + else: + printwarn("Host '%s' doesn't have %s in cache" % (client.host, pkg)) + except KeyboardInterrupt: + printerr("Aborted") + return 1 + + # download package file from mirror if necessary + if must_download == True: + try: + download_cmd = download_cmd_template.substitute({'u': argv[1], 'o': argv[2]}) + return runcmd2(download_cmd) + except KeyboardInterrupt: + printerr("Aborted") + return 1 + else: + return 0 + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/pkgdistcache-daemon b/pkgdistcache-daemon new file mode 100755 index 000000000000..0819f41f6b41 --- /dev/null +++ b/pkgdistcache-daemon @@ -0,0 +1,139 @@ +#!/usr/bin/python2 +# coding: utf-8 +# +# pkgdistcache daemon v0.3.1 +# by Alessio Bianchi <venator85@gmail.com> +# +# Daemon code by Chad J. Schroeder +# http://code.activestate.com/recipes/278731/ +# + +import sys +import os +import os.path +import subprocess +import string +import avahi +import dbus +import gobject +import dbus.glib +import SimpleHTTPServer +import BaseHTTPServer +import signal + +avahi_service = None + +def terminate(signum, frame): + avahi_service.unpublish() + sys.exit(0) + +# Run a command synchronously, redirecting stdout and stderr to strings +def runcmd(cmd, cwd=None): + pipe = subprocess.Popen(cmd, shell=True, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdout, stderr) = pipe.communicate() # wait for process to terminate and return stdout and stderr + return {'stdout': stdout.strip(), 'stderr': stderr.strip(), 'retcode': pipe.returncode} + +# Detach a process from the controlling terminal and run it in the background as a daemon +def daemonize(): + try: + pid = os.fork() + except OSError, e: + raise Exception, "%s [%d]" % (e.strerror, e.errno) + + if (pid == 0): # The first child. + os.setsid() + try: + pid = os.fork() # Fork a second child. + except OSError, e: + raise Exception, "%s [%d]" % (e.strerror, e.errno) + + if (pid != 0): # The second child. + os._exit(0) # Exit parent (the first child) of the second child. + else: + os._exit(0) # Exit parent of the first child. + + import resource # Resource usage information. + maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1] + if (maxfd == resource.RLIM_INFINITY): + maxfd = 1024 + + # Iterate through and close all file descriptors. + for fd in range(0, maxfd): + try: + os.close(fd) + except OSError: # ERROR, fd wasn't open to begin with (ignored) + pass + + # The standard I/O file descriptors are redirected to /dev/null by default. + os.open("/dev/null", os.O_RDWR) # standard input (0) + os.dup2(0, 1) # standard output (1) + os.dup2(0, 2) # standard error (2) + return(0) + +class AvahiPublisher: + #Based on http://avahi.org/wiki/PythonPublishExample + def __init__(self, name, stype, host, port): + self.name = name + self.stype = stype + self.domain = 'local' + self.host = host + self.port = port + self.systemBus = dbus.SystemBus() + self.server = dbus.Interface(self.systemBus.get_object(avahi.DBUS_NAME, avahi.DBUS_PATH_SERVER), avahi.DBUS_INTERFACE_SERVER) + + def publish(self): + self.group = dbus.Interface( + self.systemBus.get_object(avahi.DBUS_NAME, self.server.EntryGroupNew()), + avahi.DBUS_INTERFACE_ENTRY_GROUP) + + self.group.AddService( + avahi.IF_UNSPEC, #interface + avahi.PROTO_UNSPEC, #protocol + dbus.UInt32(0), #flags + self.name, self.stype, + self.domain, self.host, + dbus.UInt16(self.port), + avahi.string_array_to_txt_array([])) + self.group.Commit() + + def unpublish(self): + self.group.Reset() + +def main(args): + import optparse + parser = optparse.OptionParser() + parser.add_option("-F", "--foreground", action="store_true", dest="no_daemon", default=False, + help="run pkgdistcache-daemon in foreground") + (options, args) = parser.parse_args() + if options.no_daemon == False: + # fork daemon in background + daemonize() + + # load configuration file + conf_file = '/etc/pkgdistcache.conf' + if os.path.isfile(conf_file): + config = eval(open(conf_file).read()) + else: + printerr("Config file " + conf_file + " not found") + return 2 + + port = config['port'] + hostname = runcmd('hostname')['stdout'] + global avahi_service + avahi_service = AvahiPublisher(hostname, '_pkgdistcache._tcp', '', port) + avahi_service.publish() + + os.chdir('/var/cache/pacman/pkg') + handler = SimpleHTTPServer.SimpleHTTPRequestHandler + httpd = BaseHTTPServer.HTTPServer(('', port), handler) + + try: + httpd.serve_forever() + except KeyboardInterrupt: + avahi_service.unpublish() + + return 0 + +if __name__ == '__main__': + signal.signal(signal.SIGTERM, terminate) + main(sys.argv) diff --git a/pkgdistcache.conf b/pkgdistcache.conf new file mode 100644 index 000000000000..1271087c0464 --- /dev/null +++ b/pkgdistcache.conf @@ -0,0 +1,16 @@ +# +# pkgdistcache configuration file + +{ + # The maximum life for the cache file in minutes. After this + # amount of time, a new Avahi query will be done to discover + # pkgdistcache-capable hosts on local net. + 'cache_file_life': 10, + + # The TCP port for the daemon to listen on + 'port': 12500, + + # Command to use to download files + 'download_cmd': "/usr/bin/curl -C - -f -# -o $o $u" +} + diff --git a/pkgdistcache.install b/pkgdistcache.install new file mode 100644 index 000000000000..3e6c98264c62 --- /dev/null +++ b/pkgdistcache.install @@ -0,0 +1,18 @@ +post_install() { + cat <<EOF + +==> Quick start: + +*** SERVER MODE *** +Start and enable service: + $ sudo systemctl start pkgdistcached + $ sudo systemctl enable pkgdistcached + +*** CLIENT MODE *** +1) Make sure avahi-daemon is enabled: + $ sudo systemctl enable avahi-daemon +2) Edit /etc/pacman.conf, set the option: + XferCommand = /usr/bin/pkgdistcache-client %u %o + +EOF +} diff --git a/pkgdistcached.service b/pkgdistcached.service new file mode 100644 index 000000000000..2dfa1ec4043b --- /dev/null +++ b/pkgdistcached.service @@ -0,0 +1,8 @@ +[Unit] +Description=Distributed pacman package cache + +[Service] +ExecStart=/usr/bin/pkgdistcache-daemon -F + +[Install] +WantedBy=multi-user.target |