summarylogtreecommitdiffstats
path: root/hostsblock.sh
blob: 23bf183211a9406218df8a8f03dba28b100440ac (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
#!/bin/bash

# SUBROUTINES

_notify() {
    [ $_verbosity -ge $1 ] && echo "$2"
}

_count_hosts() {
    grep -ah -- "^[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}" "$@" | cut -d" " -f1 | sort -u | while read _addr; do
        _number=$(grep -Fc -- "$_addr " "$@")
        echo "$@: $_number urls redirected to $_addr."
    done
}

_strip_entries() {
    which pigz &>/dev/null
    if [ $? -eq 0 ]; then
        pigz -dc "$2" | grep -v "$1" | pigz $pigz_opt -c - > "$2".tmp
    else
        gzip -dc "$2" | grep -v "$1" | gzip $gzip_opt -c - > "$2".tmp
    fi
}

_extract_entries() {
    if [ ! -d "$_cachefile_dir" ]; then
        mkdir $_v -p -- "$_cachefile_dir"
        if [ $? -ne 0 ]; then
            _notify 0 "FAILED TO CREATE DIRECTORY $_cachefile_dir. EXITING..."
            exit 9
        fi
    fi
    cd "$_cachefile_dir"
    case "$_decompresser" in
        none)
            _compress_exit=0
            cp $_v -- "$_cachefile" "$_cachefile_dir"/ || _notify 1 "FAILED to move ${_cachefile##*/} to $_cachefile_dir."
        ;;
        unzip)
            unzip -B -o -j $_v_unzip -- "$_cachefile"
            _compress_exit=$?
            [ $_compress_exit -ne 0 ] && _notify 1 "FAILED to unzip ${_cachefile##*/}."
        ;;
        7z*)
            if [ $_verbosity -le 1 ]; then
                eval $_7zip_available e "$_cachefile" &>/dev/null
                _compress_exit=$?
            else
                eval $_7zip_available e "$_cachefile"
                _compress_exit=$?
            fi
            [ $_compress_exit -ne 0 ] && _notify 1 "FAILED to un7zip ${_cachefile##*/}."
        ;;
    esac
    if [ $_compress_exit -eq 0 ]; then
        _target_hostsfile="$tmpdir/hostsblock/hosts.block.d/${_cachefile##*/}.hosts"
        _cachefile_url=$(head -n1 "$cachedir"/"${_cachefile##*/}".url)
        grep -rah -- "^[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}" ./* | tr '\t' ' ' | tr -s ' ' | \
          sed -e "s/\#.*//g" -e "s/[[:space:]]$//g" -e "s/$_notredirect/$redirecturl/g" | sort -u | grep -Fvf "$whitelist" | \
          sed "s|$| \! $_cachefile_url|g" > "$_target_hostsfile"
        if [ $? -eq 0 ]; then
            [ $_verbosity -ge 2 ] && _count_hosts "$_target_hostsfile"
        else
            _notify 1 "FAILED to extract any obvious entries from ${_cachefile##*/}."
        fi
        grep -rahv "^[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}" ./* | grep -v -e "^\." -e "\.$" -e "\*" -e "\"" -e "\$" |\
          grep -P '^(?=.*[[:alpha:]])(?=.*\.)' | sed "s/^/$redirecturl /g" | tr '\t' ' ' | tr -s ' ' | \
          sed -e "s/\#.*//g" -e "s/[[:space:]]$//g" | sort -u | grep -Fvf "$whitelist" | sed "s|$| \! $_cachefile_url|g" \
          >> "$_target_hostsfile"
        if [ $? -eq 0 ]; then
            [ $_verbosity -ge 2 ] && _count_hosts "$_target_hostsfile"
        else
            _notify 1 "FAILED to extract any less-obvious entries from ${_cachefile##*/}."
        fi
        cd "$tmpdir"/hostsblock && rm $_v -r -- "$_cachefile_dir" || _notify 1 "FAILED to delete $_cachefile_dir."
    fi
}

_check_url() {
    which pigz &>/dev/null
    if [ $? -eq 0 ]; then
        _matches=$(pigz -dc "$annotate" | grep -F " $@ ")
    else
        _matches=$(gzip -dc "$annotate" | grep -F " $@ ")
    fi
    _block_matches=$(echo "$_matches" | grep -- "^$redirecturl" | sed "s/.* \!\(.*\)$/\1/g" | tr '\n' ',' | sed "s/,$//g")
    _redirect_matches=$(echo "$_matches" | grep -v "^$redirecturl" | \
      sed "s/^\([0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\) .* \! \(.*\)$/to \1 by blocklist \2/g" | \
      tr '\n' ',' | sed "s/,$//g")
    _block_matches_count=$(echo "$_block_matches" | wc -w)
    _redirect_matches_count=$(echo "$_redirect_matches" | wc -w)
    if [ $_block_matches_count -gt 0 ] || [ $_redirect_matches_count -gt 0 ]; then
        [ $_block_matches_count -gt 0 ] && echo -e "\n'$@' \e[1;31mBLOCKED \e[0mby blocklist(s)${_block_matches}"
        [ $_redirect_matches_count -gt 0 ] && echo -e "\n'$@' \e[1;33mREDIRECTED \e[0m$_redirect_matches"
        echo -e "\t1) Unblock/unredirect just $@\n\t2) Unblock/unredirect all sites containing url $@\n\t3) Keep blocked/redirected"
        read -p "1-3 (default: 3): " b
        if [[ $b == 1 || "$b" == "1" ]]; then
            echo "Unblocking just $@"
            echo " $@" >> "$whitelist"
            _strip_entries " $@ \!" "$annotate"
            sed -i "/ $@$/d" "$blacklist"
            sed -i "/ $@$/d" "$hostsfile"
            [ ! -d "$tmpdir"/hostsblock ] && mkdir $_v -p "$tmpdir"/hostsblock
            touch "$tmpdir"/hostsblock/changed
        elif [[ $b == 2 || "$b" == "2" ]]; then
            echo "Unblocking all sites containing url $@"
            echo "$@" >> "$whitelist"
            _strip_entries "$@" "$annotate"
            sed -i "/$@/d" "$blacklist"
            sed -i "/$@/d" "$hostsfile"
            [ ! -d "$tmpdir"/hostsblock ] && mkdir $_v -p "$tmpdir"/hostsblock
            touch "$tmpdir"/hostsblock/changed
        fi
    else
        echo -e "\n'$@' \e[0;32mNOT BLOCKED/REDIRECTED\e[0m\n\t1) Block $@\n\t2) Block $@ and delete all whitelist url entries containing $@\n\t3) Keep unblocked (default)"
        read -p "1-3 (default: 3): " c
        if [[ $c == 1 || "$c" == "1" ]]; then
            echo "Blocking $@"
            echo "$@" >> "$blacklist"
            (
              if which pigz &>/dev/null; then
                  pigz -dc "$annotate" > "$tmpdir"/hostsblock/"${annotate##*/}".tmp
                  echo "$redirecturl $@ \! $blacklist" >> "$tmpdir"/hostsblock/"${annotate##*/}".tmp
                  sort -u "$tmpdir"/hostsblock/"${annotate##*/}".tmp | pigz $pigz_opt -c - > "$annotate"
              else
                  gzip -dc "$annotate" > "$tmpdir"/hostsblock/"${annotate##*/}".tmp
                  echo "$redirecturl $@ \! $blacklist" >> "$tmpdir"/hostsblock/"${annotate##*/}".tmp
                  sort -u "$tmpdir"/hostsblock/"${annotate##*/}".tmp | gzip $gzip_opt -c - > "$annotate"
              fi
              rm -f "$_v" -- "$tmpdir"/hostsblock/"${annotate##*/}".tmp
            ) &
            sed -i "/^$@$/d" "$whitelist" &
            echo "$redirecturl $@" >> "$hostsfile" &
            [ ! -d "$tmpdir"/hostsblock ] && mkdir $_v -p "$tmpdir"/hostsblock
            touch "$tmpdir"/hostsblock/changed
            wait
        elif [[ $c == 2 || "$c" == "2" ]]; then
            echo "Blocking $@ and deleting all whitelist url entries containing $@"
            echo "$@" >> "$blacklist" &
            (
              if which pigz &>/dev/null; then
                  pigz -dc "$annotate" > "$tmpdir"/hostsblock/"${annotate##*/}".tmp
                  echo "$redirecturl $@ \! $blacklist" >> "$tmpdir"/hostsblock/"${annotate##*/}".tmp
                  sort -u "$tmpdir"/hostsblock/"${annotate##*/}".tmp | pigz $pigz_opt -c - > "$annotate"
              else
                  gzip -dc "$annotate" > "$tmpdir"/hostsblock/"${annotate##*/}".tmp
                  echo "$redirecturl $@ \! $blacklist" >> "$tmpdir"/hostsblock/"${annotate##*/}".tmp
                  sort -u "$tmpdir"/hostsblock/"${annotate##*/}".tmp | gzip $gzip_opt -c - > "$annotate"
              fi
              rm -f "$_v" -- "$tmpdir"/hostsblock/"${annotate##*/}".tmp
            ) &
            sed -i "/$@/d" "$whitelist" &
            echo "$redirecturl $@" >> "$hostsfile" &
            [ ! -d "$tmpdir"/hostsblock ] && mkdir $_v -p "$tmpdir"/hostsblock
            touch "$tmpdir"/hostsblock/changed
        fi
    fi
}

# VARIABLE DEFAULTS
tmpdir="/tmp/hostsblock"
hostsfile="$HOME/hosts.block"
redirecturl="127.0.0.1"
postprocess() {
     /bin/true
}
blocklists=("http://support.it-mate.co.uk/downloads/HOSTS.txt")
blacklist="$HOME/black.list"
whitelist="$HOME/white.list"
hostshead="0"
cachedir="$HOME/cache"
pigz_opt="-9"
gzip_opt="-11"
redirects="0"
connect_timeout=60
retry=0
backup_old=0
recycle_old=0
annotate="$HOME/hostsblock.db.gz"
_verbosity=1
[ -f "$tmpdir"/hostsblock/changed ] && rm -f "$tmpdir"/hostsblock/changed
_check=0
max_simultaneous_downloads=4

# GET OPTIONS
while getopts "qvf:huc:" _option; do
    case "$_option" in
        f)  [ "$OPTARG" != "" ] && _configfile="$OPTARG";;
        v)  _verbosity=2;;
        q)  _verbosity=0;;
        u)
            [ ! -d "$tmpdir"/hostsblock ] && mkdir $_v -p "$tmpdir"/hostsblock
            touch "$tmpdir"/hostsblock/changed
        ;;
        c)
            _check=1
            [ "$OPTARG" != "" ] && _URL="$OPTARG"
        ;;
        *)
            cat << EOF
Usage:
  $0 [ OPTIONS ] - generate a HOSTS file with block and redirection lists
  
  $0 [ OPTIONS ] -c URL - Check if URL and other urls contained therein are blocked

Help Options:
  -h                            Show help options

Application Options:
  -f CONFIGFILE                 Specify an alternative configuration file (instead of /var/lib/hostsblock/hostsblock.conf)
  -q                            Only show fatal errors
  -v                            Be verbose.
  -u                            Force hostsblock to update its target file, even if no changes to source files are found
                                (Ignored with '-c' option)
EOF
            exit 1
        ;;
    esac
done

# SOURCE CONFIG FILE
if [ $_configfile ]; then
    if [ -f "$_configfile" ]; then
    . "$_configfile"
    elif [ $(whoami) != "root" ] && [ -f ${HOME}/.config/hostsblock/hostsblock.conf ]; then
        _notify 1 "Config file $_configfile missing. Using ${HOME}/.config/hostsblock/hostsblock.conf"
        . ${HOME}/.config/hostsblock/hostsblock.conf
    elif [ $(whoami) != "root" ] && [ -f ${HOME}/hostsblock.conf ]; then
        _notify 1 "Config file $_configfile missing. Using ${HOME}/hostsblock.conf"
        . ${HOME}/hostsblock.conf
    elif [ -f /etc/hostsblock/hostsblock.conf ]; then
        _notify 1 "Config file $_configfile missing. Using /etc/hostsblock/hostsblock.conf"
        . /etc/hostsblock/hostsblock.conf
    else
        _notify 1 "No config files found. Using defaults."
    fi
elif [ $(whoami) != "root" ] && [ -f ${HOME}/.config/hostsblock/hostsblock.conf ]; then
    . ${HOME}/.config/hostsblock/hostsblock.conf
elif [ $(whoami) != "root" ] && [ -f ${HOME}/hostsblock.conf ]; then
    . ${HOME}/hostsblock.conf
elif [ -f /etc/hostsblock/hostsblock.conf ]; then
    . /etc/hostsblock/hostsblock.conf
fi

# SET VERBOSITY FOR SCRIPT AND ITS SUBPROCESSES
if [ $_verbosity -eq 0 ]; then
    _v=""
    _v_curl="-s"
    _v_unzip="-qq"
    set +x
elif [ $_verbosity -eq 1 ]; then
    _v=""
    _v_curl="-s"
    _v_unzip="-qq"
    set +x
else
   _v="-v"
   _v_curl="-v"
   _v_unzip="-v"
   set -x
fi

# CHECK FOR CORRECT PRIVILEDGES AND DEPENDENCIES
if [ $(whoami) != "hostsblock" ]; then
    echo -e "WRONG PERMISSIONS. RUN AS USER hostsblock, EITHER DIRECTLY OR VIA SUDO, E.G. sudo -u hostsblock $0 $@\n\nYou may have to add the following line to the end of sudoers after typing 'sudo visudo':\n $(whoami)	ALL	=	(hostblock)	NOPASSWD:	$0\n\nExiting..."
    exit 3
fi

# MAKE SURE NECESSARY DEPENDENCIES ARE PRESENT
for _depends in mv cp rm sha1sum curl grep sed tr cut mkdir file; do
    which "$_depends" &>/dev/null
    if [ $? -ne 0 ]; then
        _notify 0 "MISSING REQUIRED DEPENDENCY $_depends. PLEASE INSTALL. EXITING..."
        exit 5
    fi
done

# RUN AS URLCHECK IF $_check = 1 or if run from a symlink named "hostsblock-urlcheck"
if [ $_check -eq 1 ] || [ "${0##*/}" == "hostsblock-urlcheck" ]; then
    # URLCHECK
    [ -f "$tmpdir"/hostsblock/changed ] && rm -f "$tmpdir"/hostsblock/changed
    echo "Checking to see if url is blocked or not..."
    # If run from symlink "hostsblock-urlcheck" _URL will not be set by getopts
    [ "$_URL" == "" ] && _URL="$1"
    _check_url $(echo "$_URL" | sed -e "s/.*https*:\/\///g" -e "s/[\/?'\" :<>\(\)].*//g")
    if [ -f "$tmpdir"/hostsblock/changed ]; then
        if [ $_verbosity -ge 1 ]; then
            postprocess
        else
            postprocess &>/dev/null
        fi
    fi
    read -p "Page domain verified. Scan the whole page for other domains for (un)blocking? [y/N] " a
    if [[ $a == "y" || $a == "Y" ]]; then
        for LINE in $(curl -L --location-trusted -s "$_URL" | tr ' ' '\n' | grep "https*:\/\/" | sed -e "s/.*https*:\/\/\(.*\)$/\1/g" \
          -e "s/\//\n/g" | grep "\." | grep -vFe '"' -e ")" -e "(" -e "&" -e "?" -e "<" -e ">" -e "'" -e "_" | \
          grep -Fv -e "\.php$" -e "\.html*$" | grep "[a-z]$" | sort -u | tr "\n" " "); do
            _check_url "$LINE"
        done
        _notify 1 "Whole-page scan completed."
    fi

    if [ -f "$tmpdir"/hostsblock/changed ]; then
        if [ $_verbosity -ge 1 ]; then
            postprocess
        else
            postprocess &>/dev/null
        fi
    fi
else
    # NORMAL PROCESS
    # CHECK FOR OPTIONAL DECOMPRESSION DEPENDENCIES
    if which unzip &>/dev/null; then
        _unzip_available=1
    else
        _notify 1 "Dearchiver for zip NOT FOUND. Optional functions which use this format will be skipped."
        _unzip_available=0
    fi

    if which 7za &>/dev/null; then
        _7zip_available="7za"
    elif which 7z &>/dev/null; then
        _7zip_available="7z"
    else
        _notify 1 "Dearchiver for 7za NOT FOUND. Optional functions which use this format will be skipped."
        _7zip_available=0
    fi

    # IDENTIFY WHAT WILL not BE OUR REDIRECTION URL
    if [ "$redirecturl" == "127.0.0.1" ]; then
        _notredirect="0.0.0.0"
    else
        _notredirect="127.0.0.1"
    fi

    # CREATE CACHE DIRECTORY IF NOT ALREADY EXISTENT
    if [ ! -d "$cachedir" ]; then
        mkdir $_v -p -- "$cachedir"
        if [ $? -ne 0 ]; then
            _notify 0 "CACHE DIRECTORY $cachedir COULD NOT BE CREATED. EXITING..."
            exit 6
        fi
    fi

    # DOWNLOAD BLOCKLISTS
    _notify 1 "Checking blocklists for updates..."
    for _url in ${blocklists[*]}; do
        _outfile=$(echo $_url | sed -e "s|http:\/\/||g" -e "s|https:\/\/||g" | tr '/%&+?=' '.')
        [ -f "$cachedir"/"$_outfile".url ] || echo "$_url" > "$cachedir"/"$_outfile".url
        [ -f "$cachedir"/"$_outfile" ] && _old_sha1sum=$(sha1sum < "$cachedir"/"$_outfile")

        # Make process wait until the number of curl processes are less than $max_simultaneous_downloads
        until [ $(pidof curl | wc -w) -lt $max_simultaneous_downloads ]; do
            sleep $(pidof sleep | wc -w)
        done

        # Add a User-Agent and referer string when needed
        if [ "$_url" == "http://adblock.mahakala.is/hosts" ]; then
            curl -A "Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0" -e "http://forum.xda-developers.com/" $_v_curl --compressed -L --connect-timeout $connect_timeout --retry $retry -z "$cachedir"/"$_outfile" "$_url" -o "$cachedir"/"$_outfile"
            _curl_exit=$?
        else
            curl $_v_curl --compressed -L --connect-timeout $connect_timeout --retry $retry -z "$cachedir"/"$_outfile" "$_url" -o "$cachedir"/"$_outfile"
            _curl_exit=$?
        fi
        if [ $_curl_exit -eq 0 ]; then
            _new_sha1sum=$(sha1sum < "$cachedir"/"$_outfile")
            if [ "$_old_sha1sum" != "$_new_sha1sum" ]; then
                _notify 1 "Changes found to $_url"
                [ ! -d "$tmpdir"/hostsblock ] && mkdir $_v -p "$tmpdir"/hostsblock
                touch "$tmpdir"/hostsblock/changed
            fi
        else
            _notify 1 "FAILED to refresh/download blocklist $_url"
        fi
    done &
    wait

    # IF THERE ARE CHANGES...
    if [ -f "$tmpdir"/hostsblock/changed ]; then
        _notify 1 "Changes found among blocklists. Extracting and preparing cached files to working directory..."

        # CREATE TMPDIR
        if [ ! -d "$tmpdir"/hostsblock/hosts.block.d ]; then
            mkdir $_v -p -- "$tmpdir"/hostsblock/hosts.block.d
            if [ $? -ne 0 ]; then
                _notify 0 "FAILED TO CREATED TEMPORARY DIRECTORY $tmpdir/hostsblock/hosts.block.d. EXITING..."
                exit 7
            fi
        fi

        # EXTRACT CACHED FILES TO HOSTS.BLOCK.D
        for _cachefile in "$cachedir"/*; do
            echo "$_cachefile" | grep -q "\.url$" && continue
            _cachefile_dir="$tmpdir/hostsblock/${_cachefile##*/}.d"
            case "${_cachefile##*/}" in
                *.zip)
                    if [ $_unzip_available != 0 ]; then
                        _decompresser="unzip"
                    else
                        _notify 1 "${_cachefile##*/} is a ZIP archive, but an extractor is NOT FOUND. Skipping..."
                        continue
                    fi
                ;;
                *.7z)
                    if [ $_7zip_available != 0 ]; then
                        _decompresser="7z"
                    else
                        _notify 1 "${_cachefile##*/} is a 7z archive, but an extractor is NOT FOUND. Skipping..."
                        continue
                    fi
                ;;
                *)
                    _cachefile_type=$(file -bi "$_cachefile")
                    if [[ "$_cachefile_type" = *'application/zip'* ]]; then
                        if [ $_unzip_available != 0 ]; then
                            _decompresser="unzip"
                        else
                            _notify 1 "${_cachefile##*/} is a zip archive, but an extractor is NOT FOUND. Skipping..."
                            continue
                        fi
                    elif [[ "$_cachefile_type" = *'application/x-7z-compressed'* ]]; then
                        if [ $_7zip_available != 0 ]; then
                            _decompresser="7z"
                        else
                            _notify 1 "${_cachefile##*/} is a 7z archive, but an extractor is NOT FOUND. Skipping..."
                            continue
                        fi
                    else
                        _decompresser="none"
                    fi
                ;;
            esac
            _extract_entries &
        done
        wait

        # RECYCLE OLD HOSTS FILE INTO NEW FILE
        if [ $recycle_old == 1 ] || [ "$recycle_old" == "1" ] || [ "$recycle_old" == "yes" ] || [ "$recycle_old" == "true" ]; then
            sort -u "$hostsfile" | sed "s|$| ! $hostsfile.old |g" > "$tmpdir"/hostsblock/hosts.block.d/"${hostsfile##*/}".old || \
              _notify 1 "FAILED to recycle old $hostsfile into new version."
        fi

        # BACKUP OLD HOSTS FILE
        if [ $backup_old == 0 ] || [ "$backup_old" == "0" ] || [ "$backup_old" == "no" ] || [ "$backup_old" == "false" ]; then
            true
        else
            ls "$hostsfile".old* &>/dev/null && rm $_v -- "$hostsfile".old*
            which pigz &>/dev/null
            if [ $? -eq 0 ]; then
                cat "$hostsfile" | pigz $pigz_opt -c - > "$hostsfile".old.gz
                gzip_exit=$?
            else
                cat "$hostsfile" | gzip $gzip_opt -c - > "$hostsfile".old.gz
                gzip_exit=$?
            fi
            cp $_v -f -- "$hostsfile" "$hostsfile".old && \
            [ $gzip_exit -ne 0 ] && _notify 1 "FAILED to backup and compress $hostsfile."
        fi

        # INCLUDE HOSTS.HEAD FILE AS THE BEGINNING OF THE NEW TARGET HOSTS FILE
        if [ "$hostshead" == "0" ]; then
            rm $_v -- "$hostsfile" || _notify 1 "FAILED to delete existing $hostsfile."
        else
            cp $_v -f -- "$hostshead" "$hostsfile" || _notify 1 "FAILED to replace $hostsfile with $hostshead"
        fi

        # PROCESS AND WRITE BLOCK ENTRIES TO FILE
        _notify 1 "Compiling into $hostsfile..."
        grep -ahE -- "^$redirecturl" "$tmpdir"/hostsblock/hosts.block.d/* | tee "$tmpdir"/hostsblock/"${annotate##*/}".tmp | sed "s/ \!.*$//g" |\
          sort -u | grep -Fvf "$whitelist" >> "$hostsfile"
        if [ $? -ne 0 ]; then
            _notify 0 "FAILED TO COMPILE BLOCK ENTRIES INTO $hostsfile. EXITING..."
            exit 2
        fi

        # PROCESS AND WRITE REDIRECT ENTRIES TO FILE
        if [ $redirects == 1 ] || [ "$redirects" == "1" ]; then
            grep -ahEv -- "^$redirecturl" "$tmpdir"/hostsblock/hosts.block.d/* |\
              grep -ah -- "^[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}" | tee -a "$tmpdir"/hostsblock/"${annotate##*/}".tmp |\
              sed "s/ \!.*$//g" | sort -u | grep -Fvf "$whitelist"  >> "$hostsfile" || \
                _notify 1 "FAILED to compile redirect entries into $hostsfile."
        fi

        # APPEND BLACKLIST ENTRIES
        while read _blacklistline; do
            echo "$redirecturl $_blacklistline \! $blacklist" >> "$tmpdir"/hostsblock/"${annotate##*/}".tmp
            grep -Fqx "$_blacklistline" "$hostsfile" || echo "$redirecturl $_blacklistline" >> "$hostsfile"
        done < "$blacklist" || _notify 1 "FAILED to append blacklisted entries to $hostsfile."

        # SORT AND COMPRESS ANNOTATION FILE.
        (
          which pigz &>/dev/null
          if [ $? -eq 0 ]; then
              sort -u "$tmpdir"/hostsblock/"${annotate##*/}".tmp | pigz $_pigz_opt -c - > "$annotate"
          else
              sort -u "$tmpdir"/hostsblock/"${annotate##*/}".tmp | gzip $_gzip_opt -c - > "$annotate"
          fi
          [ -f "$annotate" ] && rm -f "$_v" -- "$tmpdir"/hostsblock/"${annotate##*/}".tmp
        ) &

        # REPORT COUNT OF MODIFIED OR BLOCKED URLS
        ( [ $_verbosity -ge 1 ] && _count_hosts "$hostsfile" ) &

        # COMMANDS TO BE EXECUTED AFTER PROCESSING
        _notify 1 "Executing postprocessing..."
        postprocess || _notify 1 "Postprocessing FAILED."

        wait

        # CLEAN UP
        rm $_v -r -- "$tmpdir"/hostsblock || _notify 1 "FAILED to clean up $tmpdir/hostsblock."
    else
        _notify 1  "No new changes. DONE."
    fi
fi