diff options
-rw-r--r-- | .SRCINFO | 10 | ||||
-rw-r--r-- | PKGBUILD | 10 | ||||
-rw-r--r-- | hostsblock-common.sh | 49 | ||||
-rwxr-xr-x | hostsblock-urlcheck.sh | 8 | ||||
-rw-r--r-- | hostsblock.changelog | 11 | ||||
-rw-r--r-- | hostsblock.conf | 12 | ||||
-rwxr-xr-x | hostsblock.sh | 15 |
7 files changed, 61 insertions, 54 deletions
@@ -1,6 +1,6 @@ pkgbase = hostsblock pkgdesc = A script and cronjob that downloads, sorts, and installs multiple ad- and malware-blocking hosts files. - pkgver = 0.999.2 + pkgver = 0.999.3 pkgrel = 1 url = http://gaenserich.github.com/hostsblock/ install = hostsblock.install @@ -31,10 +31,10 @@ pkgbase = hostsblock source = hosts.head source = hostsblock.service source = hostsblock.timer - md5sums = ccc9ec72746b7302aae9d970f0a752e7 - md5sums = aadad00fec8fe842af5a748d39ef5484 - md5sums = 8d01f7141845ecd481098300ecde652e - md5sums = c198b1dd1e69982e00000d2e3065a400 + md5sums = c37b64ca1e12e349f33e851bd1c0d0cc + md5sums = befb6ba1997af9b60c22ffc1ac1b51bc + md5sums = 5dd054e334343338b7f4e550b39fc6dc + md5sums = ae9e36a9c77780306fa48d10684f71fd md5sums = 216d5af213e0eb3690ea3c27d4cc6258 md5sums = 3a6ea9f5b0eef002b6ca1dd57388d78a md5sums = 949af91b7a40582de127eb43a96f001e @@ -1,6 +1,6 @@ # Maintainer: Jake VanderKolk <jakevanderkolk@gmail.com> pkgname=hostsblock -pkgver=0.999.2 +pkgver=0.999.3 pkgrel=1 pkgdesc="A script and cronjob that downloads, sorts, and installs multiple ad- and malware-blocking hosts files." arch=(any) @@ -17,10 +17,10 @@ backup=('etc/hostsblock/hostsblock.conf' 'etc/hostsblock/black.list' 'etc/hostsb changelog=$pkgname.changelog install=$pkgname.install source=('hostsblock.sh' 'hostsblock-urlcheck.sh' 'hostsblock-common.sh' 'hostsblock.conf' 'black.list' 'white.list' 'hosts.head' 'hostsblock.service' 'hostsblock.timer') -md5sums=('ccc9ec72746b7302aae9d970f0a752e7' - 'aadad00fec8fe842af5a748d39ef5484' - '8d01f7141845ecd481098300ecde652e' - 'c198b1dd1e69982e00000d2e3065a400' +md5sums=('c37b64ca1e12e349f33e851bd1c0d0cc' + 'befb6ba1997af9b60c22ffc1ac1b51bc' + '5dd054e334343338b7f4e550b39fc6dc' + 'ae9e36a9c77780306fa48d10684f71fd' '216d5af213e0eb3690ea3c27d4cc6258' '3a6ea9f5b0eef002b6ca1dd57388d78a' '949af91b7a40582de127eb43a96f001e' diff --git a/hostsblock-common.sh b/hostsblock-common.sh index 2cb2e15c25e2..db634211537d 100644 --- a/hostsblock-common.sh +++ b/hostsblock-common.sh @@ -10,25 +10,14 @@ ### Level 5: Level 4 + stdout/stderr from sub-processes like curl, zip, 7za, etc. _notify() { - if [ -z "$PS1" ] && [[ $- =~ i ]]; then - case $1 in - 0) _level="[\e[1;31mFATAL\e[0m]" ;; - 1) _level="[\e[1;31mWARN\e[0m]" ;; - 2) _level="[\e[0;33mNOTE\e[0m]" ;; - 3) _level="[\e[0;33mINFO\e[0m]" ;; - 4) _level="[\e[0;32mDETAIL\e[0m]" ;; - 5) _level="[\e[0;32mDEBUG\e[0m]" ;; - esac - else - case $1 in - 0) _level="[FATAL]" ;; - 1) _level="[WARN]" ;; - 2) _level="[NOTE]" ;; - 3) _level="[INFO]" ;; - 4) _level="[DETAIL]" ;; - 5) _level="[DEBUG]" ;; - esac - fi + case $1 in + 0) _level="[FATAL]" ;; + 1) _level="[WARN]" ;; + 2) _level="[NOTE]" ;; + 3) _level="[INFO]" ;; + 4) _level="[DETAIL]" ;; + 5) _level="[DEBUG]" ;; + esac if [ $verbosity -ge $1 ]; then echo -e "${_level} $2" else @@ -101,7 +90,7 @@ _extract_entries() { _notify 4 "Extracting obvious entries from $_basename_cachefile..." _cachefile_url=$(head -n1 "$cachedir"/"$_basename_cachefile".url) if grep -rah -- "^[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}" ./* | sed -e 's/[[:space:]][[:space:]]*/ /g' -e \ - "s/\#.*//g" -e "s/[[:space:]]$//g" -e "s/$_notredirect/$redirecturl/g" | sort -u | grep -vf "$whilelist" | \ + "s/\#.*//g" -e "s/[[:space:]]$//g" -e "s/$_notredirect/$redirecturl/g" | sort -u | grep -vf "$whitelist" | \ sed "s|$| \! $_cachefile_url|g" > "$_target_hostsfile"; then _notify 4 "Extracted obvious entries from $_basename_cachefile." if [ $verbosity -ge 4 ]; then @@ -113,7 +102,7 @@ _extract_entries() { _notify 4 "Extracting less-obvious entries from $_basename_cachefile" if grep -rahv "^[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}" ./* | grep -v "^\." | grep -v "\.$" | grep -v "\*" |\ grep -v "\"" | grep -v "\$" | grep "[a-z]" | grep "\." | sed "s/^/$redirecturl /g" | sed -e 's/[[:space:]][[:space:]]*/ /g' \ - -e "s/\#.*//g" -e "s/[[:space:]]$//g" | sort -u | grep -vf "$whilelist" | sed "s|$| \! $_cachefile_url|g" |\ + -e "s/\#.*//g" -e "s/[[:space:]]$//g" | sort -u | grep -vf "$whitelist" | sed "s|$| \! $_cachefile_url|g" |\ >> "$_target_hostsfile"; then _notify 4 "Extracted less-obvious entries from $_basename_cachefile." if [ $verbosity -ge 4 ]; then @@ -224,8 +213,8 @@ _strip_entries() { case "$2" in *.gz) which pigz &>/dev/null && \ - pigz -dc "$2" | grep -v "$1" | pigz -zc - > "$2".tmp || \ - gzip -dc "$2" | grep -v "$1" | gzip -zc - > "$2".tmp + pigz -dc "$2" | grep -v "$1" | pigz -c - > "$2".tmp || \ + gzip -dc "$2" | grep -v "$1" | gzip -c - > "$2".tmp ;; *) grep -v "$1" "$2" > "$2".tmp && \ @@ -258,8 +247,8 @@ _check_url(){ read -p "1-3 (default: 3): " b if [[ $b == 1 || "$b" == "1" ]]; then echo "Unblocking just $@" - echo " $@ " >> "$whitelist" - _strip_entries " $@$" "$annotate" + echo " $@" >> "$whitelist" + _strip_entries " $@ \!" "$annotate" _strip_entries " $@$" "$blacklist" _strip_entries " $@$" "$hostsfile" _changed=1 @@ -282,11 +271,11 @@ _check_url(){ if which pigz &>/dev/null; then pigz -dc "$annotate" > "$annotate".tmp echo "$redirecturl $@ \! $blacklist" >> "$annotate".tmp - sort -u "$annotate".tmp | pigz -zc - > "$annotate" + sort -u "$annotate".tmp | pigz -c - > "$annotate" else gzip -dc "$annotate" > "$annotate".tmp echo "$redirecturl $@ \! $blacklist" >> "$annotate".tmp - sort -u "$annotate".tmp | gzip -zc - > "$annotate" + sort -u "$annotate".tmp | gzip -c - > "$annotate" fi rm -f "$_v" -- "$annotate".tmp ;; @@ -306,11 +295,11 @@ _check_url(){ if which pigz &>/dev/null; then pigz -dc "$annotate" > "$annotate".tmp echo "$redirecturl $@ \! $blacklist" >> "$annotate".tmp - sort -u "$annotate".tmp | pigz -zc - > "$annotate" + sort -u "$annotate".tmp | pigz -c - > "$annotate" else gzip -dc "$annotate" > "$annotate".tmp echo "$redirecturl $@ \! $blacklist" >> "$annotate".tmp - sort -u "$annotate".tmp | gzip -zc - > "$annotate" + sort -u "$annotate".tmp | gzip -c - > "$annotate" fi rm -f "$_v" -- "$annotate".tmp ;; @@ -335,7 +324,7 @@ postprocess() { } export blocklists=("http://support.it-mate.co.uk/downloads/HOSTS.txt") export blacklist="/etc/hostsblock/black.list" -export whilelist="/etc/hostsblock/white.list" +export whitelist="/etc/hostsblock/white.list" export hostshead="0" export cachedir="/var/cache/hostsblock" export redirects="0" diff --git a/hostsblock-urlcheck.sh b/hostsblock-urlcheck.sh index 3ef2bfe8503f..ff123483ab84 100755 --- a/hostsblock-urlcheck.sh +++ b/hostsblock-urlcheck.sh @@ -47,7 +47,13 @@ _set_subprocess_verbosity _changed=0 echo "Checking to see if url is blocked or unblocked..." _check_url $(echo "$@" | sed -e "s/.*https*:\/\///g" -e "s/[\/?'\" :<>\(\)].*//g") -[ $_changed == 1 ] && postprocess &>/dev/null +if [ $_changed == 1 ]; then + if [ $verbosity -ge 5 ]; then + postprocess + else + postprocess &>/dev/null + fi +fi read -p "Page domain verified. Scan the whole page for other domains for (un)blocking? [y/N] " a if [[ $a == "y" || $a == "Y" ]]; then for LINE in `curl -L --location-trusted -s "$@" | tr ' ' '\n' | grep "https*:\/\/" | sed -e "s/.*https*:\/\/\(.*\)$/\1/g" -e "s/\//\n/g" | grep "\." |\ diff --git a/hostsblock.changelog b/hostsblock.changelog index 1d075834420f..a633a54d73f9 100644 --- a/hostsblock.changelog +++ b/hostsblock.changelog @@ -1,3 +1,14 @@ +See https://github.com/gaenserich/hostsblock/commits/master for a complete list of updates + +Version 0.999.3 (Alpha 3) (30.10.2015) +*Added more optional source files to hostsblock.conf. Added user agent info to curl to make work with more sites. Sanitized hostsblock.conf file a little bit. +*Added function to force target file update even if no changes are found to source files +*Fixed incorrect invocation of _strip_entries subroutine +*Fixed remaining appearances of gzip -z. (Thanks to Julián Landerreche) +*Fixed misspelled whitelist/whilelist variable +*Fixed incorrect use of -z with gzip/pigz +*Fixed line 261 in hostsblock-common.sh (white list write function) + (07.25.2014) PRE-ALPHA RELEASE *new main config file: /etc/hostsblock/hostsblock.conf diff --git a/hostsblock.conf b/hostsblock.conf index 2d1db6de7574..c8b1153f281b 100644 --- a/hostsblock.conf +++ b/hostsblock.conf @@ -1,7 +1,3 @@ -# LOGGING FILE. DEPRECIATED. All error and status messages go to stdout now. -# Redirect to a logfile on the command line, e.g.: -## /usr/bin/hostsblock &>/var/log/hostsblock.log - # CACHE DIRECTORY. Directory where blocklists will be downloaded and stored. # They will only be updated if the server has a newer version. cachedir="/var/cache/hostsblock" # DEFAULT @@ -46,8 +42,8 @@ redirects="0" # DEFAULT # /etc/hosts so as to preserve your loopback devices. Give hostshead="0" to # disable this feature. For those targeting /etc/hosts, it is advisable to copy # their old /etc/hosts file to this file so as to preserve existing entries. -hostshead="0" # DEFAULT -#hostshead="/etc/hostsblock/hosts.head" # If not using dns caching. +#hostshead="0" +hostshead="/etc/hostsblock/hosts.head" # DEFAULT # BLACKLISTED SUBDOMAINS. File containing specific subdomains to blacklist which # may not be in the downloaded blacklists. Be sure to provide not just the @@ -123,6 +119,10 @@ blocklists=( 'http://sysctl.org/cameleon/hosts' # anti-ad (sysctl.org/cameleon/) ## OPTIONAL LISTS #'http://hosts-file.net/download/yahoo_servers.zip' # hpHosts for yahoo adservers (hosts-file.net/) +#'http://adblock.gjtech.net/?format=unix-hosts' # another general-use list to block adds +#'http://adblock.mahakala.is/hosts' # more anti-ad +#'http://optimate.dl.sourceforge.net/project/adzhosts/HOSTS.txt' # anti-ad specifically for smartphones +#'https://hosts.neocities.org/' # anti-ad metalist #'http://abp.mozilla-hispano.org/nauscopio/hosts.zip' # supplemented version of someonewhocares.org list (https://nauscopio.wordpress.com/category/filtrado/) #'http://rlwpx.free.fr/WPFF/htrc.7z' # anti-trackers (very large) (rlwpx.free.fr/WPFF) #'http://rlwpx.free.fr/WPFF/hpub.7z' # anti-advertising (very large) (rlwpx.free.fr/WPFF) diff --git a/hostsblock.sh b/hostsblock.sh index 4596ef2d5411..63c049ff6703 100755 --- a/hostsblock.sh +++ b/hostsblock.sh @@ -1,10 +1,11 @@ #!/bin/bash - +_changed=0 # GET OPTIONS -while getopts "v:f:h" _option; do +while getopts "v:f:hu" _option; do case "$_option" in f) [ "$OPTARG" != "" ] && _configfile="$OPTARG";; v) [ "$OPTARG" != "" ] && _verbosity_override=$OPTARG;; + u) _changed=1;; *) cat << EOF Usage: @@ -16,6 +17,7 @@ Help Options: Application Options: -f CONFIGFILE Specify an alternative configuration file (instead of /etc/hostsblock/hostsblock.conf) -v VERBOSITY Specify how much information hostsblock provides (0=only fatal errors to 5=the kitchen sink) + -u Force hostsblock to update its target file, even if no changes to source files are found EOF exit 1 ;; @@ -59,7 +61,6 @@ else fi # DOWNLOAD BLOCKLISTS -_changed=0 _notify 3 "Checking blocklists for updates..." for _url in ${blocklists[*]}; do _outfile=$(echo $_url | sed "s|http:\/\/||g" | tr '/%&+?=' '.') @@ -76,7 +77,7 @@ for _url in ${blocklists[*]}; do _notify 4 "Cache file $cachedir/$_outfile for blocklist $_url not found. It will be downloaded." fi _notify 4 "Checking and, if needed, downloading blocklist $_url to $cachedir/$_outfile" - if curl $_v_curl --compressed -L --connect-timeout $connect_timeout --retry $retry -z "$cachedir"/"$_outfile" "$_url" -o "$cachedir"/"$_outfile"; then + if curl -A 'Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' -e http://forum.xda-developers.com/ $_v_curl --compressed -L --connect-timeout $connect_timeout --retry $retry -z "$cachedir"/"$_outfile" "$_url" -o "$cachedir"/"$_outfile"; then _notify 3 "Refreshed blocklist $_url." _new_ls=$(ls -l "$cachedir"/"$_outfile") if [ "$_old_ls" != "$_new_ls" ]; then @@ -185,7 +186,7 @@ if [ $_changed != 0 ]; then _notify 3 "Compiling redirect entries into $hostsfile..." if grep -ahEv -- "^$redirecturl" "$tmpdir"/hostsblock/hosts.block.d/* |\ grep -ah -- "^[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}" | tee -a "$annotate".tmp |\ - sed "s/ \!.*$//g" | sort -u | grep -vf "$whilelist" >> "$hostsfile"; then + sed "s/ \!.*$//g" | sort -u | grep -vf "$whitelist" >> "$hostsfile"; then _notify 3 "Compiled redirect entries into $hostsfile." else _notify 1 "FAILED to compile redirect entries into $hostsfile." @@ -206,8 +207,8 @@ if [ $_changed != 0 ]; then case "$annotate" in *.gz) which pigz &>/dev/null && \ - sort -u "$annotate".tmp | pigz -zc - > "$annotate" || \ - sort -u "$annotate".tmp | gzip -zc - > "$annotate" + sort -u "$annotate".tmp | pigz -c - > "$annotate" || \ + sort -u "$annotate".tmp | gzip -c - > "$annotate" ;; *) sort -u "$annotate".tmp > "$annotate" |