mirror of
https://github.com/novatiq/packages.git
synced 2026-04-30 15:38:40 +01:00
adblock: release 1.1.0
* support more router modes, as long as the firewall and the DNS server
are enabled
* new source sites can be added & changed via awk ruleset in uci config
* source domain count, last update time & overall count will be stored
in uci config
* added 3 new source sites:
ransomware tracker
rolist/easylist
winspy
* switch to minimal inline base64 encoded 1×1 GIF for pixel server
(separate png image no longer needed)
* simplified uci parser
* source download & domain sort optimization
* add whitelist parser with wildcard support
* reduced code size & various cleanups
* updated documentation
Signed-off-by: Dirk Brenken <openwrt@brenken.org>
(cherry picked from commit b9aeb82c95)
This commit is contained in:
Dirk Brenken
committed by
Hannu Nyman
Hannu Nyman
parent
9aa63712b3
commit
bb5f4f6d29
@@ -36,7 +36,7 @@ fi
|
||||
# get current directory, script- and openwrt version
|
||||
#
|
||||
adb_scriptdir="${0%/*}"
|
||||
adb_scriptver="1.0.3"
|
||||
adb_scriptver="1.1.0"
|
||||
openwrt_version="$(cat /etc/openwrt_version)"
|
||||
|
||||
# source in adblock function library
|
||||
@@ -63,103 +63,30 @@ f_envload
|
||||
#
|
||||
f_log "domain adblock processing started (${adb_scriptver}, ${openwrt_version}, $(/bin/date "+%d.%m.%Y %H:%M:%S"))"
|
||||
|
||||
# parse environment
|
||||
#
|
||||
f_envparse
|
||||
|
||||
# check environment
|
||||
#
|
||||
f_envcheck
|
||||
|
||||
# start shallalist (pre-)processing
|
||||
#
|
||||
if [ -n "${adb_arc_shalla}" ]
|
||||
then
|
||||
# start shallalist processing
|
||||
#
|
||||
shalla_archive="${adb_tmpdir}/shallalist.tar.gz"
|
||||
shalla_file="${adb_tmpdir}/shallalist.txt"
|
||||
src_name="shalla"
|
||||
adb_dnsfile="${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
|
||||
if [ -r "${adb_dnsfile}" ]
|
||||
then
|
||||
list_time="$(awk '$0 ~ /^# last modified/ {printf substr($0,18)}' "${adb_dnsfile}")"
|
||||
fi
|
||||
f_log "=> (pre-)processing adblock source '${src_name}'"
|
||||
|
||||
# only process shallalist archive with updated timestamp,
|
||||
# extract and merge only domains of selected shallalist categories
|
||||
#
|
||||
shalla_time="$(${adb_fetch} ${wget_parm} --server-response --spider "${adb_arc_shalla}" 2>&1 | awk '$0 ~ /Last-Modified/ {printf substr($0,18)}')"
|
||||
if [ -z "${shalla_time}" ]
|
||||
then
|
||||
shalla_time="$(date)"
|
||||
f_log " no online timestamp received, current date will be used"
|
||||
fi
|
||||
if [ -z "${list_time}" ] || [ "${list_time}" != "${shalla_time}" ]
|
||||
then
|
||||
${adb_fetch} ${wget_parm} --output-document="${shalla_archive}" "${adb_arc_shalla}"
|
||||
rc=${?}
|
||||
if [ $((rc)) -eq 0 ]
|
||||
then
|
||||
> "${shalla_file}"
|
||||
for category in ${adb_cat_shalla}
|
||||
do
|
||||
tar -xOzf "${shalla_archive}" BL/${category}/domains >> "${shalla_file}"
|
||||
rc=${?}
|
||||
if [ $((rc)) -ne 0 ]
|
||||
then
|
||||
f_log " archive extraction failed (${category})"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
# remove temporary files
|
||||
#
|
||||
rm -f "${shalla_archive}"
|
||||
rm -rf "${adb_tmpdir}/BL"
|
||||
if [ $((rc)) -eq 0 ]
|
||||
then
|
||||
adb_sources="${adb_sources} ${shalla_file}&ruleset=rset_shalla"
|
||||
f_log " source archive (pre-)processing finished"
|
||||
else
|
||||
rc=0
|
||||
adb_srclist="! -name ${adb_dnsprefix}.${src_name}"
|
||||
adb_errsrclist="-name ${adb_dnsprefix}.${src_name}"
|
||||
fi
|
||||
else
|
||||
rc=0
|
||||
adb_srclist="! -name ${adb_dnsprefix}.${src_name}"
|
||||
adb_errsrclist="-name ${adb_dnsprefix}.${src_name}"
|
||||
f_log " source archive download failed"
|
||||
fi
|
||||
else
|
||||
adb_srclist="! -name ${adb_dnsprefix}.${src_name}"
|
||||
f_log " source archive doesn't change, no update required"
|
||||
fi
|
||||
fi
|
||||
|
||||
# add blacklist source to active adblock domain sources
|
||||
#
|
||||
if [ -s "${adb_blacklist}" ]
|
||||
then
|
||||
adb_sources="${adb_sources} ${adb_blacklist}&ruleset=rset_blacklist"
|
||||
fi
|
||||
|
||||
# loop through active adblock domain sources,
|
||||
# download sources, prepare output and store all extracted domains in temp file
|
||||
#
|
||||
for src in ${adb_sources}
|
||||
for src_name in ${adb_sources}
|
||||
do
|
||||
url="${src/\&ruleset=*/}"
|
||||
src_name="${src/*\&ruleset=rset_/}"
|
||||
eval "url=\"\${adb_src_${src_name}}\""
|
||||
eval "src_rset=\"\${adb_src_rset_${src_name}}\""
|
||||
adb_dnsfile="${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
|
||||
if [ -r "${adb_dnsfile}" ]
|
||||
then
|
||||
list_time="$(awk '$0 ~ /^# last modified/ {printf substr($0,18)}' "${adb_dnsfile}")"
|
||||
fi
|
||||
list_time="$(uci_get "adblock" "${src_name}" "adb_src_timestamp")"
|
||||
f_log "=> processing adblock source '${src_name}'"
|
||||
|
||||
# check 'url' and 'src_rset' values
|
||||
#
|
||||
if [ -z "${url}" ] || [ -z "${src_rset}" ]
|
||||
then
|
||||
uci_set "adblock" "${src_name}" "adb_src_timestamp" "broken config"
|
||||
f_log " broken source configuration, check 'adb_src' and 'adb_src_rset' in config"
|
||||
continue
|
||||
fi
|
||||
|
||||
# prepare find statement with active adblock list sources
|
||||
#
|
||||
if [ -z "${adb_srclist}" ]
|
||||
@@ -173,30 +100,48 @@ do
|
||||
#
|
||||
if [ "${src_name}" = "blacklist" ]
|
||||
then
|
||||
url_time="$(date -r "${adb_blacklist}")"
|
||||
elif [ "${src_name}" = "shalla" ]
|
||||
then
|
||||
url_time="${shalla_time}"
|
||||
url_time="$(date -r "${url}")"
|
||||
else
|
||||
url_time="$(${adb_fetch} ${wget_parm} --server-response --spider "${url}" 2>&1 | awk '$0 ~ /Last-Modified/ {printf substr($0,18)}')"
|
||||
url_time="$(${adb_fetch} ${fetch_parm} --server-response --spider "${url}" 2>&1 | awk '$0 ~ /Last-Modified/ {printf substr($0,18)}')"
|
||||
fi
|
||||
if [ -z "${url_time}" ]
|
||||
then
|
||||
url_time="$(date)"
|
||||
f_log " no online timestamp received, current date will be used"
|
||||
fi
|
||||
if [ -z "${list_time}" ] || [ "${list_time}" != "${url_time}" ]
|
||||
if [ -z "${list_time}" ] || [ "${list_time}" != "${url_time}" ] || [ ! -r "${adb_dnsfile}" ] ||\
|
||||
([ "${backup_ok}" = "true" ] && [ ! -r "${adb_dir_backup}/${adb_dnsprefix}.${src_name}" ])
|
||||
then
|
||||
if [ "${src_name}" = "blacklist" ]
|
||||
then
|
||||
tmp_domains="$(cat "${adb_blacklist}")"
|
||||
tmp_domains="$(cat "${url}")"
|
||||
rc=${?}
|
||||
elif [ "${src_name}" = "shalla" ]
|
||||
then
|
||||
tmp_domains="$(cat "${shalla_file}")"
|
||||
shalla_archive="${adb_tmpdir}/shallalist.tar.gz"
|
||||
shalla_file="${adb_tmpdir}/shallalist.txt"
|
||||
${adb_fetch} ${fetch_parm} --output-document="${shalla_archive}" "${url}"
|
||||
rc=${?}
|
||||
if [ $((rc)) -eq 0 ]
|
||||
then
|
||||
> "${shalla_file}"
|
||||
for category in ${adb_src_cat_shalla}
|
||||
do
|
||||
tar -xOzf "${shalla_archive}" BL/${category}/domains >> "${shalla_file}"
|
||||
rc=${?}
|
||||
if [ $((rc)) -ne 0 ]
|
||||
then
|
||||
f_log " archive extraction failed (${category})"
|
||||
break
|
||||
fi
|
||||
done
|
||||
rm -f "${shalla_archive}"
|
||||
rm -rf "${adb_tmpdir}/BL"
|
||||
tmp_domains="$(cat "${shalla_file}")"
|
||||
rc=${?}
|
||||
fi
|
||||
else
|
||||
tmp_domains="$(${adb_fetch} ${wget_parm} --output-document=- "${url}")"
|
||||
tmp_domains="$(${adb_fetch} ${fetch_parm} --output-document=- "${url}")"
|
||||
rc=${?}
|
||||
fi
|
||||
else
|
||||
@@ -208,8 +153,7 @@ do
|
||||
#
|
||||
if [ $((rc)) -eq 0 ] && [ -n "${tmp_domains}" ]
|
||||
then
|
||||
eval "src_rset=\${rset_${src_name}}"
|
||||
count="$(printf "%s\n" "${tmp_domains}" | eval "${src_rset}" | tee "${adb_tmpfile}" | wc -l)"
|
||||
count="$(printf "%s\n" "${tmp_domains}" | awk "${src_rset}" | tee "${adb_tmpfile}" | wc -l)"
|
||||
f_log " source download finished (${count} entries)"
|
||||
if [ "${src_name}" = "shalla" ]
|
||||
then
|
||||
@@ -218,6 +162,7 @@ do
|
||||
unset tmp_domains
|
||||
elif [ $((rc)) -eq 0 ] && [ -z "${tmp_domains}" ]
|
||||
then
|
||||
uci_set "adblock" "${src_name}" "adb_src_timestamp" "empty download"
|
||||
f_log " empty source download finished"
|
||||
continue
|
||||
else
|
||||
@@ -228,6 +173,7 @@ do
|
||||
else
|
||||
adb_errsrclist="${adb_errsrclist} -o -name ${adb_dnsprefix}.${src_name}"
|
||||
fi
|
||||
uci_set "adblock" "${src_name}" "adb_src_timestamp" "download failed"
|
||||
f_log " source download failed"
|
||||
continue
|
||||
fi
|
||||
@@ -237,9 +183,9 @@ do
|
||||
#
|
||||
if [ $((count)) -gt 0 ] && [ -n "${adb_tmpfile}" ]
|
||||
then
|
||||
if [ -s "${adb_whitelist}" ]
|
||||
if [ -s "${adb_tmpdir}/tmp.whitelist" ]
|
||||
then
|
||||
grep -Fvxf "${adb_whitelist}" "${adb_tmpfile}" | sort -u | eval "${adb_dnsformat}" > "${adb_dnsfile}"
|
||||
grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpfile}" | sort -u | eval "${adb_dnsformat}" > "${adb_dnsfile}"
|
||||
rc=${?}
|
||||
else
|
||||
sort -u "${adb_tmpfile}" | eval "${adb_dnsformat}" > "${adb_dnsfile}"
|
||||
@@ -255,19 +201,18 @@ do
|
||||
adb_revsrclist="${adb_revsrclist} -o -name ${adb_dnsprefix}.${src_name}"
|
||||
fi
|
||||
|
||||
# write preliminary footer
|
||||
# store source timestamp in config
|
||||
#
|
||||
if [ $((rc)) -eq 0 ]
|
||||
then
|
||||
printf "%s\n" "#---------------------------------------------" >> "${adb_dnsfile}"
|
||||
printf "%s\n" "# last modified: ${url_time}" >> "${adb_dnsfile}"
|
||||
printf "%s\n" "##" >> "${adb_dnsfile}"
|
||||
uci_set "adblock" "${src_name}" "adb_src_timestamp" "${url_time}"
|
||||
f_log " domain merging finished"
|
||||
else
|
||||
f_log " domain merging failed" "${rc}"
|
||||
f_restore
|
||||
fi
|
||||
else
|
||||
uci_set "adblock" "${src_name}" "adb_src_timestamp" "empty domain input"
|
||||
f_log " empty domain input received"
|
||||
continue
|
||||
fi
|
||||
@@ -279,33 +224,23 @@ if [ -n "${adb_srclist}" ]
|
||||
then
|
||||
rm_done="$(find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" \( ${adb_srclist} \) -print -exec rm -f "{}" \;)"
|
||||
rc=${?}
|
||||
if [ "${backup_ok}" = "true" ]
|
||||
then
|
||||
find "${adb_dir_backup}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" \( ${adb_srclist} \) -exec rm -f "{}" \;
|
||||
fi
|
||||
else
|
||||
rm_done="$(find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" -print -exec rm -f "{}" \;)"
|
||||
rc=${?}
|
||||
if [ "${backup_ok}" = "true" ]
|
||||
then
|
||||
find "${adb_dir_backup}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" -exec rm -f "{}" \;
|
||||
fi
|
||||
fi
|
||||
if [ $((rc)) -eq 0 ] && [ -n "${rm_done}" ]
|
||||
then
|
||||
f_rmconfig "${rm_done}"
|
||||
f_log "disabled adblock lists removed"
|
||||
if [ "${backup_ok}" = "true" ]
|
||||
then
|
||||
if [ -n "${adb_srclist}" ]
|
||||
then
|
||||
rm_done="$(find "${adb_backupdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" \( ${adb_srclist} \) -print -exec rm -f "{}" \;)"
|
||||
rc=${?}
|
||||
else
|
||||
rm_done="$(find "${adb_backupdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" -print -exec rm -f "{}" \;)"
|
||||
rc=${?}
|
||||
fi
|
||||
if [ $((rc)) -eq 0 ] && [ -n "${rm_done}" ]
|
||||
then
|
||||
f_log "disabled adblock list backups removed"
|
||||
elif [ $((rc)) -ne 0 ]
|
||||
then
|
||||
f_log "error during removal of disabled adblock list backups" "${rc}"
|
||||
f_exit
|
||||
fi
|
||||
fi
|
||||
elif [ $((rc)) -ne 0 ]
|
||||
elif [ $((rc)) -ne 0 ] && [ -n "${rm_done}" ]
|
||||
then
|
||||
f_log "error during removal of disabled adblock lists" "${rc}"
|
||||
f_exit
|
||||
@@ -315,10 +250,11 @@ fi
|
||||
#
|
||||
if [ "${backup_ok}" = "true" ] && [ -n "${adb_errsrclist}" ]
|
||||
then
|
||||
restore_done="$(find "${adb_backupdir}" -maxdepth 1 -type f \( ${adb_errsrclist} \) -print -exec cp -pf "{}" "${adb_dnsdir}" \;)"
|
||||
restore_done="$(find "${adb_dir_backup}" -maxdepth 1 -type f \( ${adb_errsrclist} \) -print -exec cp -pf "{}" "${adb_dnsdir}" \;)"
|
||||
rc=${?}
|
||||
if [ $((rc)) -eq 0 ] && [ -n "${restore_done}" ]
|
||||
then
|
||||
f_rmconfig "${restore_done}"
|
||||
f_log "partial restore done"
|
||||
elif [ $((rc)) -ne 0 ]
|
||||
then
|
||||
@@ -335,52 +271,31 @@ then
|
||||
|
||||
# generate a unique overall block list
|
||||
#
|
||||
head -qn -3 "${adb_dnsdir}/${adb_dnsprefix}."* | sort -u > "${adb_tmpdir}/blocklist.overall"
|
||||
sort -u "${adb_dnsdir}/${adb_dnsprefix}."* > "${adb_tmpdir}/blocklist.overall"
|
||||
|
||||
# loop through all separate lists, ordered by size (ascending)
|
||||
#
|
||||
for list in $(ls -Sr "${adb_dnsdir}/${adb_dnsprefix}."*)
|
||||
for list in $(ls -ASr "${adb_dnsdir}/${adb_dnsprefix}."*)
|
||||
do
|
||||
# check overall block list vs. separate block list,
|
||||
# write only duplicate entries to a temporary separate list
|
||||
# write all duplicate entries to separate list
|
||||
#
|
||||
list="${list/*./}"
|
||||
sort "${adb_tmpdir}/blocklist.overall" "${adb_dnsdir}/${adb_dnsprefix}.${list}" | uniq -d > "${adb_tmpdir}/tmp.${list}"
|
||||
|
||||
# write only unique entries back to overall block list
|
||||
#
|
||||
sort "${adb_tmpdir}/blocklist.overall" "${adb_tmpdir}/tmp.${list}" | uniq -u > "${adb_tmpdir}/tmp.overall"
|
||||
mv -f "${adb_tmpdir}/tmp.overall" "${adb_tmpdir}/blocklist.overall"
|
||||
|
||||
# write unique result back to original separate list
|
||||
#
|
||||
tail -qn 3 "${adb_dnsdir}/${adb_dnsprefix}.${list}" >> "${adb_tmpdir}/tmp.${list}"
|
||||
mv -f "${adb_tmpdir}/tmp.${list}" "${adb_dnsdir}/${adb_dnsprefix}.${list}"
|
||||
|
||||
# write all unique entries back to overall block list
|
||||
#
|
||||
sort "${adb_tmpdir}/blocklist.overall" "${adb_dnsdir}/${adb_dnsprefix}.${list}" | uniq -u > "${adb_tmpdir}/tmp.overall"
|
||||
mv -f "${adb_tmpdir}/tmp.overall" "${adb_tmpdir}/blocklist.overall"
|
||||
done
|
||||
rm -f "${adb_tmpdir}/blocklist.overall"
|
||||
fi
|
||||
|
||||
# set separate list count & get overall count
|
||||
#
|
||||
for list in $(ls -Sr "${adb_dnsdir}/${adb_dnsprefix}."*)
|
||||
do
|
||||
list="${list/*./}"
|
||||
count="$(head -qn -3 "${adb_dnsdir}/${adb_dnsprefix}.${list}" | wc -l)"
|
||||
if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ]
|
||||
then
|
||||
count=$((count / 2))
|
||||
fi
|
||||
if [ "$(tail -qn 1 "${adb_dnsdir}/${adb_dnsprefix}.${list}")" = "##" ]
|
||||
then
|
||||
last_line="# ${0##*/} (${adb_scriptver}) - ${count} ad\/abuse domains blocked"
|
||||
sed -i "s/^##$/${last_line}/" "${adb_dnsdir}/${adb_dnsprefix}.${list}"
|
||||
fi
|
||||
adb_count=$((adb_count + count))
|
||||
done
|
||||
|
||||
# restart dnsmasq with newly generated or deleted adblock lists,
|
||||
# check dnsmasq startup afterwards
|
||||
# restart & check dnsmasq with newly generated set of adblock lists
|
||||
#
|
||||
f_cntconfig
|
||||
adb_count="$(uci_get "adblock" "global" "adb_overall_count")"
|
||||
if [ -n "${adb_revsrclist}" ] || [ -n "${rm_done}" ] || [ -n "${restore_done}" ]
|
||||
then
|
||||
/etc/init.d/dnsmasq restart
|
||||
@@ -395,19 +310,19 @@ then
|
||||
f_restore
|
||||
fi
|
||||
else
|
||||
f_log "adblock lists with overall ${adb_count} domains are still valid, no dnsmasq restart required"
|
||||
f_log "adblock lists with overall ${adb_count} domains are still valid, no update required"
|
||||
fi
|
||||
|
||||
# create adblock list backups
|
||||
#
|
||||
if [ "${backup_ok}" = "true" ] && [ -n "${adb_revsrclist}" ] && [ "$(printf "${adb_dnsdir}/${adb_dnsprefix}."*)" != "${adb_dnsdir}/${adb_dnsprefix}.*" ]
|
||||
if [ "${backup_ok}" = "true" ] && [ -n "${adb_revsrclist}" ]
|
||||
then
|
||||
backup_done="$(find "${adb_dnsdir}" -maxdepth 1 -type f \( ${adb_revsrclist} \) -print -exec cp -pf "{}" "${adb_backupdir}" \;)"
|
||||
backup_done="$(find "${adb_dnsdir}" -maxdepth 1 -type f \( ${adb_revsrclist} \) -print -exec cp -pf "{}" "${adb_dir_backup}" \;)"
|
||||
rc=${?}
|
||||
if [ $((rc)) -eq 0 ] && [ -n "${backup_done}" ]
|
||||
then
|
||||
f_log "new adblock list backups generated"
|
||||
elif [ $((rc)) -ne 0 ]
|
||||
elif [ $((rc)) -ne 0 ] && [ -n "${backup_done}" ]
|
||||
then
|
||||
f_log "error during backup of adblock lists" "${rc}"
|
||||
f_exit
|
||||
|
||||
Reference in New Issue
Block a user