1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
|
#!/bin/bash
# GET OPTIONS
while getopts "v:f:h" _option; do
case "$_option" in
f) [ "$OPTARG" != "" ] && _configfile="$OPTARG";;
v) [ "$OPTARG" != "" ] && _verbosity_override=$OPTARG;;
*)
cat << EOF
Usage:
$0 [ -f CONFIGFILE ] [ -v VERBOSITY ] - update the HOSTS file with block and redirection lists
Help Options:
-h Show help options
Application Options:
-f CONFIGFILE Specify an alternative configuration file (instead of /etc/hostsblock/hostsblock.conf)
-v VERBOSITY Specify how much information hostsblock provides (0=only fatal errors to 5=the kitchen sink)
EOF
exit 1
;;
esac
done
# SOURCE DEFAULT SETTINGS AND SUBROUTINES
if [ -f /usr/lib/hostsblock-common.sh ]; then
source /usr/lib/hostsblock-common.sh
elif [ -f /usr/local/lib/hostsblock-common.sh ]; then
source /usr/local/lib/hostsblock-common.sh
elif [ -f ./hostsblock-common.sh ]; then
source ./hostsblock-common.sh
else
echo "hostsblock-common.sh NOT FOUND. INSTALL IT TO /usr/lib/ OR /usr/local/lib/. EXITING..."
exit 1
fi
_source_configfile
_verbosity_check
_set_subprocess_verbosity
_check_root
_check_depends mv cp rm curl grep sed tr cut mkdir file
_check_unzip
_check_7z
# IDENTIFY WHAT WILL not BE OUR REDIRECTION URL
if [ "$redirecturl" == "127.0.0.1" ]; then
_notredirect="0.0.0.0"
else
_notredirect="127.0.0.1"
fi
# CREATE CACHE DIRECTORY IF NOT ALREADY EXISTENT
if [ -d "$cachedir" ]; then
_notify 4 "Cache directory $cachedir already created."
else
_notify 4 "Creating cache directory $cachedir..."
mkdir $_v -p -- "$cachedir" && _notify 3 "Created temporary cache directory $cachedir" || \
_notify 1 "FAILED to create cache directory $cachedir."
fi
# DOWNLOAD BLOCKLISTS
_changed=0
_notify 3 "Checking blocklists for updates..."
for _url in ${blocklists[*]}; do
_outfile=$(echo $_url | sed "s|http:\/\/||g" | tr '/%&+?=' '.')
if [ -f "$cachedir"/"$_outfile".url ]; then
_notify 4 "Url file for $cachedir/$_outfile present."
else
_notify 4 "Url file for $cachedir/$_outfile not present. Creating it..."
echo "$_url" > "$cachedir"/"$_outfile".url
fi
if [ -f "$cachedir"/"$_outfile" ]; then
_notify 4 "Cache file $cachedir/$_outfile for blocklist $_url exists. Noting its modification time."
_old_ls=$(ls -l "$cachedir"/"$_outfile")
else
_notify 4 "Cache file $cachedir/$_outfile for blocklist $_url not found. It will be downloaded."
fi
_notify 4 "Checking and, if needed, downloading blocklist $_url to $cachedir/$_outfile"
if curl $_v_curl --compressed -L --connect-timeout $connect_timeout --retry $retry -z "$cachedir"/"$_outfile" "$_url" -o "$cachedir"/"$_outfile"; then
_notify 3 "Refreshed blocklist $_url."
_new_ls=$(ls -l "$cachedir"/"$_outfile")
if [ "$_old_ls" != "$_new_ls" ]; then
_changed=1
_notify 2 "CHANGES FOUND for blocklist $_url."
else
_notify 4 "No changes for blocklist $_url."
fi
else
_notify 1 "FAILED to refresh/download blocklist $_url."
fi
done
# IF THERE ARE CHANGES...
if [ $_changed != 0 ]; then
_notify 3 "Changes found among blocklists. Extracting and preparing cached files to working directory..."
# CREATE TMPDIR
if [ -d "$tmpdir"/hostsblock/hosts.block.d ]; then
_notify 4 "Temporary working directory $tmpdir/hostsblock/hosts.block.d exists."
else
_notify 4 "Temporary working directory $tmpdir/hostsblock/hosts.block.d does not exist. Creating it..."
mkdir $_v -p -- "$tmpdir"/hostsblock/hosts.block.d
fi
# EXTRACT CACHED FILES TO HOSTS.BLOCK.D
_notify 4 "Extracting cached blocklist files to $tmpdir/hostsblock/hosts.block.d."
for _cachefile in "$cachedir"/*; do
echo "$_cachefile" | grep -q "\.url$" && continue
_basename_cachefile=$(basename "$_cachefile")
_cachefile_dir="$tmpdir/hostsblock/$_basename_cachefile.d"
_notify 4 "Inspecting $_basename_cachefile for extraction..."
case "$_basename_cachefile" in
*.zip)
if [ $_unzip_available != 0 ]; then
_notify 4 "$_basename_cachefile is a zip archive. Will use unzip to extract it..."
_decompresser="unzip"
else
_notify 1 "$_basename_cachefile is a zip archive, but an extractor is NOT FOUND. Skipping..."
continue
fi
;;
*.7z)
if [ $_7zip_available != 0 ]; then
_notify 4 "$_basename_cachefile is a 7z archive. Will use $_7zip_available to extract it..."
_decompresser="7z"
else
_notify 1 "$_basename_cachefile is a 7z archive, but an extractor is NOT FOUND. Skipping..."
continue
fi
;;
*)
_notify 4 "$_basename_cachefile has an ambiguous suffix. Inspecting it for its filetype..."
_cachefile_type=$(file -bi "$_cachefile")
if [[ "$_cachefile_type" = *'application/zip'* ]]; then
if [ $_unzip_available != 0 ]; then
_notify 4 "$_basename_cachefile is a zip archive. Will use unzip to extract it..."
_decompresser="unzip"
else
_notify 1 "$_basename_cachefile is a zip archive, but an extractor is NOT FOUND. Skipping..."
continue
fi
elif [[ "$_cachefile_type" = *'application/x-7z-compressed'* ]]; then
if [ $_7zip_available != 0 ]; then
_notify 4 "$_basename_cachefile is a 7z archive. Will use $_7zip_available to extract it..."
_decompresser="7z"
else
_notify 1 "$_basename_cachefile is a 7z archive, but an extractor is NOT FOUND. Skipping..."
continue
fi
else
_notify 4 "$_basename_cachefile is a plaintext file. No extractor needed."
_decompresser="none"
fi
;;
esac
_extract_entries &
done
_notify 4 "Waiting for extraction processes to finish..."
wait
_backup_old
# INCLUDE HOSTS.HEAD FILE AS THE BEGINNING OF THE NEW TARGET HOSTS FILE
if [ "$hostshead" == "0" ] || [ $hostshead == 0 ]; then
_notify 4 "Not using a hostshead file, so deleting existing $hostsfile to make way for its new version..."
rm $_v -- "$hostsfile" && _notify 4 "Deleted existing $hostsfile." || _notify 1 "FAILED to delete existing $hostsfile."
else
_notify 4 "Using a hostshead file, so overwriting $hostsfile with $hostshead..."
cp $_v -f -- "$hostshead" "$hostsfile" && _notify 3 "Replaced existing $hostsfile with $hosthead." || \
_notify 1 "FAILED to replace $hostsfile with $hostshead"
fi
# PROCESS AND WRITE BLOCK ENTRIES TO FILE
_notify 3 "Compiling block entries into $hostsfile..."
if grep -ahE -- "^$redirecturl" "$tmpdir"/hostsblock/hosts.block.d/* | tee "$annotate".tmp | sed "s/ \!.*$//g" |\
sort -u >> "$hostsfile"; then
_notify 3 "Compiled block entries into $hostsfile."
else
_notify 0 "FAILED TO COMPILE BLOCK ENTRIES INTO $hostsfile. EXITING."
exit 2
fi
# PROCESS AND WRITE REDIRECT ENTRIES TO FILE
if [ $redirects == 1 ] || [ "$redirects" == "1" ]; then
_notify 3 "Compiling redirect entries into $hostsfile..."
if grep -ahEv -- "^$redirecturl" "$tmpdir"/hostsblock/hosts.block.d/* |\
grep -ah -- "^[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}" | tee -a "$annotate".tmp |\
sed "s/ \!.*$//g" | sort -u | grep -vf "$whilelist" >> "$hostsfile"; then
_notify 3 "Compiled redirect entries into $hostsfile."
else
_notify 1 "FAILED to compile redirect entries into $hostsfile."
fi
else
_notify 4 "Skipping redirect entries..."
fi
# APPEND BLACKLIST ENTRIES
_notify 3 "Appending blacklisted entries to $hostsfile..."
while read _blacklistline; do
echo "$redirecturl $_blacklistline \! $blacklist" >> "$annotate".tmp
grep -q "$_blacklistline" "$hostsfile" || echo "$redirecturl $_blacklistline" >> "$hostsfile"
done < "$blacklist" && _notify 3 "Appended blacklisted entries to $hostsfile." || \
_notify 1 "FAILED to append blacklisted entries to $hostsfile."
# SORT AND, IF REQUESTED, COMPRESS ANNOTATION FILE.
case "$annotate" in
*.gz)
which pigz &>/dev/null && \
sort -u "$annotate".tmp | pigz -zc - > "$annotate" || \
sort -u "$annotate".tmp | gzip -zc - > "$annotate"
;;
*)
sort -u "$annotate".tmp > "$annotate"
;;
esac
[ -f "$annotate" ] && rm -f "$_v" -- "$annotate".tmp
# REPORT COUNT OF MODIFIED OR BLOCKED URLS
[ $verbosity -ge 3 ] && _count_hosts "$hostsfile"
# COMMANDS TO BE EXECUTED AFTER PROCESSING
_notify 3 "Executing postprocessing..."
if [ $verbosity -ge 5 ]; then
postprocess && _notify 3 "Postprocessing completed." || _notify 1 "Postprocessing FAILED."
else
postprocess &>/dev/null && _notify 3 "Postprocessing completed." || _notify 1 "Postprocessing FAILED."
fi
# CLEAN UP
_notify 4 "Cleaning up temporary directory $tmpdir/hostsblock..."
rm $_v -r -- "$tmpdir"/hostsblock && _notify 2 "Cleaned up $tmpdir/hostsblock." || _notify 1 "FAILED to clean up $tmpdir/hostsblock."
_notify 3 "DONE."
else
_notify 3 "No new changes. DONE."
fi
|