diff options
author | Eric Anderson | 2017-12-28 13:24:35 -0600 |
---|---|---|
committer | Eric Anderson | 2017-12-28 13:45:08 -0600 |
commit | f784a3a45dc26f443074e188208288e77cddfa78 (patch) | |
tree | 3ba4bbc8015bd7fa478e6a1fd2fb8be44876fffa | |
parent | 1faebce10e86ca3b6865de71ea6cdbdd625a5130 (diff) | |
download | aur-f784a3a45dc26f443074e188208288e77cddfa78.tar.gz |
Update style of source to better match PEP 8
-rw-r--r-- | .SRCINFO | 8 | ||||
-rw-r--r-- | PKGBUILD | 6 | ||||
-rwxr-xr-x | pkgdistcache-client | 365 | ||||
-rwxr-xr-x | pkgdistcache-daemon | 241 |
4 files changed, 337 insertions, 283 deletions
@@ -1,9 +1,9 @@ # Generated by mksrcinfo v8 -# Thu Dec 28 18:17:16 UTC 2017 +# Thu Dec 28 19:44:58 UTC 2017 pkgbase = pkgdistcache pkgdesc = A distributed local-network cache for pacman packages pkgver = 0.4.6 - pkgrel = 1 + pkgrel = 2 url = http://venator.ath.cx/dw/doku.php?id=linux:pkgdistcache install = pkgdistcache.install arch = any @@ -19,8 +19,8 @@ pkgbase = pkgdistcache source = pkgdistcache-daemon source = pkgdistcache.conf source = pkgdistcached.service - sha256sums = c1d9df5a1e37dde556ba47f1262d19a61fff57fbd7e1a0e370b9e671802e343c - sha256sums = 8f298c9f1548b56373038fe69f8568dc77e17a700476594155359df20eb275a4 + sha256sums = 4141d8d07a7c67ebd0dba24ca8aa2a97bcf93a2915b7afbebbd85728bbbc356c + sha256sums = c03074bdcc7456ae1dbe4cf32a6fb8854a5fbb6445bfd16208e3fb85c9b91d5e sha256sums = d77ac418aa651bc622cd91204d6907554c6cdb4bb989e484cc54da32342faa51 sha256sums = 756c0bd1139e296da88937a89ab19e0b5d6c0d5d0f719034d4029b1fe1ea09e9 @@ -3,7 +3,7 @@ pkgname=pkgdistcache pkgver=0.4.6 -pkgrel=1 +pkgrel=2 pkgdesc='A distributed local-network cache for pacman packages' arch=('any') url='http://venator.ath.cx/dw/doku.php?id=linux:pkgdistcache' @@ -14,8 +14,8 @@ source=('pkgdistcache-client' 'pkgdistcache-daemon' 'pkgdistcache.conf' 'pkgdistcached.service') -sha256sums=('c1d9df5a1e37dde556ba47f1262d19a61fff57fbd7e1a0e370b9e671802e343c' - '8f298c9f1548b56373038fe69f8568dc77e17a700476594155359df20eb275a4' +sha256sums=('4141d8d07a7c67ebd0dba24ca8aa2a97bcf93a2915b7afbebbd85728bbbc356c' + 'c03074bdcc7456ae1dbe4cf32a6fb8854a5fbb6445bfd16208e3fb85c9b91d5e' 'd77ac418aa651bc622cd91204d6907554c6cdb4bb989e484cc54da32342faa51' '756c0bd1139e296da88937a89ab19e0b5d6c0d5d0f719034d4029b1fe1ea09e9') diff --git a/pkgdistcache-client b/pkgdistcache-client index f120f820e335..c153d40052d8 100755 --- a/pkgdistcache-client +++ b/pkgdistcache-client @@ -5,193 +5,228 @@ # by Alessio Bianchi <venator85@gmail.com> # -import sys import os import os.path -import subprocess +import pickle import string +import subprocess +import sys +import time + import avahi import dbus -from gi.repository import GObject as gobject import dbus.glib -import pickle import requests -import time import xdg.BaseDirectory +from gi.repository import GObject + +colors = { + 'none': '\033[0m', + 'black': '\033[0;30m', 'bold_black': '\033[1;30m', + 'red': '\033[0;31m', 'bold_red': '\033[1;31m', + 'green': '\033[0;32m', 'bold_green': '\033[1;32m', + 'yellow': '\033[0;33m', 'bold_yellow': '\033[1;33m', + 'blue': '\033[0;34m', 'bold_blue': '\033[1;34m', + 'magenta': '\033[0;35m', 'bold_magenta': '\033[1;35m', + 'cyan': '\033[0;36m', 'bold_cyan': '\033[1;36m', + 'white': '\033[0;37m', 'bold_white': '\033[1;37m'} -colors = {'none': '\033[0m', - 'black': '\033[0;30m', 'bold_black': '\033[1;30m', - 'red': '\033[0;31m', 'bold_red': '\033[1;31m', - 'green': '\033[0;32m', 'bold_green': '\033[1;32m', - 'yellow': '\033[0;33m', 'bold_yellow': '\033[1;33m', - 'blue': '\033[0;34m', 'bold_blue': '\033[1;34m', - 'magenta': '\033[0;35m', 'bold_magenta': '\033[1;35m', - 'cyan': '\033[0;36m', 'bold_cyan': '\033[1;36m', - 'white': '\033[0;37m', 'bold_white': '\033[1;37m'} def printmsg(msg): - print("%s>> %s%s" % (colors['bold_blue'], msg, colors['none'])) + print("%s>> %s%s" % (colors['bold_blue'], msg, colors['none'])) + def printerr(msg): - print("%s!! %s%s" % (colors['bold_red'], msg, colors['none'])) + print("%s!! %s%s" % (colors['bold_red'], msg, colors['none'])) + def printwarn(msg): - print("%s!! %s%s" % (colors['bold_yellow'], msg, colors['none'])) + print("%s!! %s%s" % (colors['bold_yellow'], msg, colors['none'])) + # Run a command synchronously, sending stdout and stderr to shell def runcmd2(cmd, cwd=None): - pipe = subprocess.Popen(cmd, shell=True, cwd=cwd, stdout=None, stderr=None) - pipe.communicate() # wait for process to terminate - return pipe.returncode + pipe = subprocess.Popen(cmd, shell=True, cwd=cwd, stdout=None, stderr=None) + pipe.communicate() # wait for process to terminate + return pipe.returncode + class Service(object): - def __init__(self, service, host, ip, port): - self.service = service - self.host = host - self.ip = ip - self.port = port - - def __str__(self): - return "(%s, %s, %s, %d)" % (self.service, self.host, self.ip, self.port) - - def __repr__(self): - # il tostring sulle liste è repr - return self.__str__() - - def __hash__(self): - return hash(self.__str__()) - - def __eq__(self, other): - return hash(self) == hash(other) + def __init__(self, service, host, ip, port): + self.service = service + self.host = host + self.ip = ip + self.port = port + + def __str__(self): + return ("(%s, %s, %s, %d)" + % (self.service, self.host, self.ip, self.port)) + + def __repr__(self): + # il tostring sulle liste è repr + return self.__str__() + + def __hash__(self): + return hash(self.__str__()) + + def __eq__(self, other): + return hash(self) == hash(other) + class AvahiBrowser(object): - def __init__(self): - # Connect to the system bus... - self.bus = dbus.SystemBus() - # Get a proxy to the object we want to talk to. - avahi_proxy = self.bus.get_object(avahi.DBUS_NAME, avahi.DBUS_PATH_SERVER) - # Set the interface we want to use; server in this case. - self.server = dbus.Interface(avahi_proxy, avahi.DBUS_INTERFACE_SERVER) - self.version_string = self.server.GetVersionString() - self.domain = "local" - self.loop = gobject.MainLoop() - self.services = [] - - def browse(self, stype): - # Ask the server for a path to the browser object for the service we're interested in... - browser_path = self.server.ServiceBrowserNew(avahi.IF_UNSPEC, avahi.PROTO_UNSPEC, stype, self.domain, dbus.UInt32(0)) - # Get it's proxy object... - browser_proxy = self.bus.get_object(avahi.DBUS_NAME, browser_path) - # And set the interface we want to use - browser = dbus.Interface(browser_proxy, avahi.DBUS_INTERFACE_SERVICE_BROWSER) - - # Now connect the call backs to the relevant signals. - browser.connect_to_signal('ItemNew', self.new_service) - browser.connect_to_signal('AllForNow', self.all_for_now) - self.loop.run() - return self.services - - def new_service(self, interface, protocol, name, stype, domain, flags): - if flags & avahi.LOOKUP_RESULT_LOCAL: - # The service is on this machine; ignore - return - try: - s = self.server.ResolveService(interface, protocol, name, stype, domain, avahi.PROTO_UNSPEC, dbus.UInt32(0)) - service = Service(str(s[3]), str(s[2]), str(s[7]), int(s[8])) # service name, host, ip, port - self.services.append(service) - except dbus.DBusException as ex: - # Mainly expect to see org.freedesktop.Avahi.TimeoutError: Timeout reached - printwarn(ex) - - def all_for_now(self): - self.loop.quit() + def __init__(self): + # Connect to the system bus... + self.bus = dbus.SystemBus() + # Get a proxy to the object we want to talk to. + avahi_proxy = self.bus.get_object( + avahi.DBUS_NAME, avahi.DBUS_PATH_SERVER) + # Set the interface we want to use; server in this case. + self.server = dbus.Interface(avahi_proxy, avahi.DBUS_INTERFACE_SERVER) + self.version_string = self.server.GetVersionString() + self.domain = "local" + self.loop = GObject.MainLoop() + self.services = [] + + def browse(self, stype): + # Ask the server for a path to the browser object for the service we're + # interested in... + browser_path = self.server.ServiceBrowserNew( + avahi.IF_UNSPEC, + avahi.PROTO_UNSPEC, + stype, + self.domain, + dbus.UInt32(0)) + # Get it's proxy object... + browser_proxy = self.bus.get_object(avahi.DBUS_NAME, browser_path) + # And set the interface we want to use + browser = dbus.Interface( + browser_proxy, avahi.DBUS_INTERFACE_SERVICE_BROWSER) + + # Now connect the call backs to the relevant signals. + browser.connect_to_signal('ItemNew', self.new_service) + browser.connect_to_signal('AllForNow', self.all_for_now) + self.loop.run() + return self.services + + def new_service(self, interface, protocol, name, stype, domain, flags): + if flags & avahi.LOOKUP_RESULT_LOCAL: + # The service is on this machine; ignore + return + try: + s = self.server.ResolveService( + interface, + protocol, + name, + stype, + domain, + avahi.PROTO_UNSPEC, + dbus.UInt32(0)) + # service name, host, ip, port + service = Service(str(s[3]), str(s[2]), str(s[7]), int(s[8])) + self.services.append(service) + except dbus.DBusException as ex: + # Mainly expect to see: + # org.freedesktop.Avahi.TimeoutError: Timeout reached + printwarn(ex) + + def all_for_now(self): + self.loop.quit() + def main(argv): - # load configuration file - conf_file = '/etc/pkgdistcache.conf' - if os.path.isfile(conf_file): - config = eval(open(conf_file).read()) - else: - printerr("Config file " + conf_file + " not found") - return 2 - - download_cmd_template = string.Template(config['download_cmd']) - - pkg = os.path.basename(argv[1]) # argv[1] = %u passed by pacman - - must_download = True - if not (pkg.endswith('.db') or pkg.endswith('.db.sig')): - runtime_dir = xdg.BaseDirectory.get_runtime_dir(strict=False) - cache_file = os.path.join(runtime_dir, 'pkgdistcache') - try: - # No exception means file exists - stat = os.stat(cache_file) - # Check age of file - cache_life_secs = int(config['cache_file_life']) * 60 - cache_valid = time.time() < stat.st_mtime + cache_life_secs - except FileNotFoundError: - cache_valid = False - if cache_valid: - with open(cache_file, 'rb') as f: - pkgdistcache_clients = pickle.load(f) - else: - # recent cache file not found, discover other pkgdistcache capable - # hosts via avahi and save result to a new cache file - browser = AvahiBrowser() - clients = browser.browse("_pkgdistcache._tcp") - clients = set(clients) # remove duplicates (eg services offered on more than a network card etc.) - pkgdistcache_clients = list(clients) - - try: - os.unlink(cache_file) # remove any old cache file - except FileNotFoundError: - pass - with open(cache_file, 'wb') as f: - pickle.dump(pkgdistcache_clients, f, -1) - - if pkgdistcache_clients: - print("-- Discovered hosts: %s" % ", ".join(set([c.host for c in pkgdistcache_clients]))) - else: - print("-- No hosts discovered") - - for client in pkgdistcache_clients: - clientip = client.ip - if ":" in clientip: - clientip = "[" + clientip + "]" - url = "http://" + clientip + ":" + str(client.port) + "/" + pkg - try: - r = requests.head(url, timeout=1) - except Exception as e: - printwarn("Failed checking host '%s' with ip '%s': %s" % (client.host, clientip, repr(e))) - continue - if r.status_code != 200: - continue - - dst = argv[2] - printmsg("Downloading %s from host '%s'" % (pkg, client.host)) - download_cmd = download_cmd_template.substitute({'u': url, 'o': dst}) - try: - ret = runcmd2(download_cmd) - if ret == 0: - must_download = False - break - else: - printwarn("Host '%s' doesn't have %s in cache" % (client.host, pkg)) - except KeyboardInterrupt: - printerr("Aborted") - return 1 - - # download package file from mirror if necessary - if must_download == True: - print(">> Downloading %s from mirror" % pkg) - try: - download_cmd = download_cmd_template.substitute({'u': argv[1], 'o': argv[2]}) - return runcmd2(download_cmd) - except KeyboardInterrupt: - printerr("Aborted") - return 1 - else: - return 0 - + # load configuration file + conf_file = '/etc/pkgdistcache.conf' + if os.path.isfile(conf_file): + config = eval(open(conf_file).read()) + else: + printerr("Config file " + conf_file + " not found") + return 2 + + download_cmd_template = string.Template(config['download_cmd']) + + pkg = os.path.basename(argv[1]) # argv[1] = %u passed by pacman + + must_download = True + if not (pkg.endswith('.db') or pkg.endswith('.db.sig')): + runtime_dir = xdg.BaseDirectory.get_runtime_dir(strict=False) + cache_file = os.path.join(runtime_dir, 'pkgdistcache') + try: + # No exception means file exists + stat = os.stat(cache_file) + # Check age of file + cache_life_secs = int(config['cache_file_life']) * 60 + cache_valid = time.time() < stat.st_mtime + cache_life_secs + except FileNotFoundError: + cache_valid = False + if cache_valid: + with open(cache_file, 'rb') as f: + pkgdistcache_clients = pickle.load(f) + else: + # recent cache file not found, discover other pkgdistcache capable + # hosts via avahi and save result to a new cache file + browser = AvahiBrowser() + clients = browser.browse("_pkgdistcache._tcp") + # remove duplicates (eg services offered on more than a network + # card etc.) + clients = set(clients) + pkgdistcache_clients = list(clients) + + try: + os.unlink(cache_file) # remove any old cache file + except FileNotFoundError: + pass + with open(cache_file, 'wb') as f: + pickle.dump(pkgdistcache_clients, f, -1) + + if pkgdistcache_clients: + print("-- Discovered hosts: %s" + % ", ".join(set([c.host for c in pkgdistcache_clients]))) + else: + print("-- No hosts discovered") + + for client in pkgdistcache_clients: + clientip = client.ip + if ":" in clientip: + clientip = "[" + clientip + "]" + url = "http://" + clientip + ":" + str(client.port) + "/" + pkg + try: + r = requests.head(url, timeout=1) + except Exception as e: + printwarn("Failed checking host '%s' with ip '%s': %s" + % (client.host, clientip, repr(e))) + continue + if r.status_code != 200: + continue + + dst = argv[2] + printmsg("Downloading %s from host '%s'" % (pkg, client.host)) + download_cmd = download_cmd_template.substitute( + {'u': url, 'o': dst}) + try: + ret = runcmd2(download_cmd) + if ret == 0: + must_download = False + break + else: + printwarn("Host '%s' doesn't have %s in cache" + % (client.host, pkg)) + except KeyboardInterrupt: + printerr("Aborted") + return 1 + + # download package file from mirror if necessary + if must_download: + print(">> Downloading %s from mirror" % pkg) + try: + download_cmd = download_cmd_template.substitute( + {'u': argv[1], 'o': argv[2]}) + return runcmd2(download_cmd) + except KeyboardInterrupt: + printerr("Aborted") + return 1 + else: + return 0 + + if __name__ == '__main__': - sys.exit(main(sys.argv)) + sys.exit(main(sys.argv)) diff --git a/pkgdistcache-daemon b/pkgdistcache-daemon index bdc883d1931c..c1cccd5394e6 100755 --- a/pkgdistcache-daemon +++ b/pkgdistcache-daemon @@ -8,136 +8,155 @@ # http://code.activestate.com/recipes/278731/ # -import sys +import http.server import os import os.path -import subprocess +import signal +import socket import string +import subprocess +import sys + import avahi import dbus -from gi.repository import GObject as gobject import dbus.glib -import http.server -import signal -import socket +from gi.repository import GObject as gobject avahi_service = None + def terminate(signum, frame): - avahi_service.unpublish() - sys.exit(0) + avahi_service.unpublish() + sys.exit(0) + # Run a command synchronously, redirecting stdout and stderr to strings def runcmd(cmd, cwd=None): - pipe = subprocess.Popen(cmd, shell=True, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdout, stderr) = pipe.communicate() # wait for process to terminate and return stdout and stderr - return {'stdout': stdout.strip(), 'stderr': stderr.strip(), 'retcode': pipe.returncode} - -# Detach a process from the controlling terminal and run it in the background as a daemon + pipe = subprocess.Popen(cmd, shell=True, cwd=cwd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # wait for process to terminate and return stdout and stderr + (stdout, stderr) = pipe.communicate() + return { + 'stdout': stdout.strip(), + 'stderr': stderr.strip(), + 'retcode': pipe.returncode} + + +# Detach a process from the controlling terminal and run it in the background +# as a daemon def daemonize(): - try: - pid = os.fork() - except OSError as e: - raise Exception("%s [%d]" % (e.strerror, e.errno)) - - if (pid == 0): # The first child. - os.setsid() - try: - pid = os.fork() # Fork a second child. - except OSError as e: - raise Exception("%s [%d]" % (e.strerror, e.errno)) - - if (pid != 0): # The second child. - os._exit(0) # Exit parent (the first child) of the second child. - else: - os._exit(0) # Exit parent of the first child. - - import resource # Resource usage information. - maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1] - if (maxfd == resource.RLIM_INFINITY): - maxfd = 1024 - - # Iterate through and close all file descriptors. - for fd in range(0, maxfd): - try: - os.close(fd) - except OSError: # ERROR, fd wasn't open to begin with (ignored) - pass - - # The standard I/O file descriptors are redirected to /dev/null by default. - os.open("/dev/null", os.O_RDWR) # standard input (0) - os.dup2(0, 1) # standard output (1) - os.dup2(0, 2) # standard error (2) - return(0) + try: + pid = os.fork() + except OSError as e: + raise Exception("%s [%d]" % (e.strerror, e.errno)) + + if (pid == 0): # The first child. + os.setsid() + try: + pid = os.fork() # Fork a second child. + except OSError as e: + raise Exception("%s [%d]" % (e.strerror, e.errno)) + + if (pid != 0): # The second child. + os._exit(0) # Exit parent (the first child) of the second child. + else: + os._exit(0) # Exit parent of the first child. + + import resource # Resource usage information. + maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1] + if (maxfd == resource.RLIM_INFINITY): + maxfd = 1024 + + # Iterate through and close all file descriptors. + for fd in range(0, maxfd): + try: + os.close(fd) + except OSError: # ERROR, fd wasn't open to begin with (ignored) + pass + + # The standard I/O file descriptors are redirected to /dev/null by default. + os.open("/dev/null", os.O_RDWR) # standard input (0) + os.dup2(0, 1) # standard output (1) + os.dup2(0, 2) # standard error (2) + return(0) + class AvahiPublisher: - #Based on http://avahi.org/wiki/PythonPublishExample - def __init__(self, name, stype, host, port): - self.name = name - self.stype = stype - self.domain = 'local' - self.host = host - self.port = port - self.systemBus = dbus.SystemBus() - self.server = dbus.Interface(self.systemBus.get_object(avahi.DBUS_NAME, avahi.DBUS_PATH_SERVER), avahi.DBUS_INTERFACE_SERVER) - - def publish(self): - self.group = dbus.Interface( - self.systemBus.get_object(avahi.DBUS_NAME, self.server.EntryGroupNew()), - avahi.DBUS_INTERFACE_ENTRY_GROUP) - - self.group.AddService( - avahi.IF_UNSPEC, #interface - avahi.PROTO_UNSPEC, #protocol - dbus.UInt32(0), #flags - self.name, self.stype, - self.domain, self.host, - dbus.UInt16(self.port), - avahi.string_array_to_txt_array([])) - self.group.Commit() - - def unpublish(self): - self.group.Reset() + # Based on http://avahi.org/wiki/PythonPublishExample + def __init__(self, name, stype, host, port): + self.name = name + self.stype = stype + self.domain = 'local' + self.host = host + self.port = port + self.systemBus = dbus.SystemBus() + self.server = dbus.Interface( + self.systemBus.get_object( + avahi.DBUS_NAME, avahi.DBUS_PATH_SERVER), + avahi.DBUS_INTERFACE_SERVER) + + def publish(self): + self.group = dbus.Interface( + self.systemBus.get_object( + avahi.DBUS_NAME, self.server.EntryGroupNew()), + avahi.DBUS_INTERFACE_ENTRY_GROUP) + + self.group.AddService( + avahi.IF_UNSPEC, # interface + avahi.PROTO_UNSPEC, # protocol + dbus.UInt32(0), # flags + self.name, self.stype, + self.domain, self.host, + dbus.UInt16(self.port), + avahi.string_array_to_txt_array([])) + self.group.Commit() + + def unpublish(self): + self.group.Reset() + class HTTPServerV6(http.server.HTTPServer): - address_family = socket.AF_INET6 + address_family = socket.AF_INET6 + def main(args): - import optparse - parser = optparse.OptionParser() - parser.add_option("-F", "--foreground", action="store_true", dest="no_daemon", default=False, - help="run pkgdistcache-daemon in foreground") - (options, args) = parser.parse_args() - if options.no_daemon == False: - # fork daemon in background - daemonize() - - # load configuration file - conf_file = '/etc/pkgdistcache.conf' - if os.path.isfile(conf_file): - config = eval(open(conf_file).read()) - else: - printerr("Config file " + conf_file + " not found") - return 2 - - port = config['port'] - hostname = runcmd('hostname')['stdout'] - global avahi_service - avahi_service = AvahiPublisher(hostname, '_pkgdistcache._tcp', '', port) - avahi_service.publish() - - os.chdir('/var/cache/pacman/pkg') - handler = http.server.SimpleHTTPRequestHandler - httpd = HTTPServerV6(('', port), handler) - - try: - # Disable useless polling since we never call shutdown - httpd.serve_forever(poll_interval=None) - except KeyboardInterrupt: - avahi_service.unpublish() - - return 0 + import optparse + parser = optparse.OptionParser() + parser.add_option("-F", "--foreground", action="store_true", + dest="no_daemon", default=False, + help="run pkgdistcache-daemon in foreground") + (options, args) = parser.parse_args() + if not options.no_daemon: + # fork daemon in background + daemonize() + + # load configuration file + conf_file = '/etc/pkgdistcache.conf' + if os.path.isfile(conf_file): + config = eval(open(conf_file).read()) + else: + printerr("Config file " + conf_file + " not found") + return 2 + + port = config['port'] + hostname = runcmd('hostname')['stdout'] + global avahi_service + avahi_service = AvahiPublisher(hostname, '_pkgdistcache._tcp', '', port) + avahi_service.publish() + + os.chdir('/var/cache/pacman/pkg') + handler = http.server.SimpleHTTPRequestHandler + httpd = HTTPServerV6(('', port), handler) + + try: + # Disable useless polling since we never call shutdown + httpd.serve_forever(poll_interval=None) + except KeyboardInterrupt: + avahi_service.unpublish() + + return 0 + if __name__ == '__main__': - signal.signal(signal.SIGTERM, terminate) - main(sys.argv) + signal.signal(signal.SIGTERM, terminate) + main(sys.argv) |