mirror of
https://github.com/novatiq/packages.git
synced 2026-04-30 15:38:40 +01:00
adblock: update 1.4.4
Update for-15.05 adblock from 1.2.1 to 1.4.4 Signed-off-by: Hannu Nyman <hannu.nyman@iki.fi> Original commit messages of 1.2.5-1.4.4 commits: adblock: update 1.2.5 * restructured sources * fix logical glitches in config handling * many corner case fixes & cosmetics * show runtime errors in LuCI (in lastrun section) adbock: update 1.2.6 * small addition in case of a failed list download adblock: update 1.2.7 * provide adblock statistics as a separate function (/etc/init.d/adblock stats) adblock: update 1.2.8 * fw rule changes: force_dns now supports multiple lan devices disable needless force_dns- & forward/output-rules in 'ap mode' check return codes during adblock chain creation * simplified the test for a running firewall * documentation update adblock: release 1.3.0 * revised hotplug script * remove wget package dependency * support uclient-fetch or wget with ssl support * documentation update adblock: update 1.3.1 * fix uclient-fetch detection * cosmetics adblock: update 1.3.2 * only a few more fixes adblock: update 1.3.3 * enable automatic restore on empty source downloads adblock: release 1.4.0 * rework/speed up overall sort * simplified dns error handling adblock: update 1.4.1 * fix two possible overflows in adblock statistics adblock: update 1.4.2 * ad broad blocklist source 'hphosts' https://hosts-file.net adblock: update 1.4.3 * fix race condition in restricted mode * cosmetics adblock: update 1.4.4 * filter non-printable characters/binary data in input stream * fix IPv4 adblock statistics in CC
This commit is contained in:
Dirk Brenken
committed by
Hannu Nyman
Hannu Nyman
parent
01c323c293
commit
7ea2bf5899
+103
-139
@@ -6,112 +6,103 @@
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# set the C locale
|
||||
#
|
||||
LC_ALL=C
|
||||
|
||||
# set pid & logger
|
||||
# prepare environment
|
||||
#
|
||||
adb_pid="${$}"
|
||||
adb_pidfile="/var/run/adblock.pid"
|
||||
adb_log="$(which logger)"
|
||||
|
||||
adb_scriptver="1.4.4"
|
||||
adb_mincfgver="2.3"
|
||||
adb_scriptdir="${0%/*}"
|
||||
if [ -r "${adb_pidfile}" ]
|
||||
then
|
||||
rc=255
|
||||
"${adb_log}" -s -t "adblock[${adb_pid}] error" "adblock service already running ($(cat ${adb_pidfile}))"
|
||||
logger -s -t "adblock[${adb_pid}] error" "adblock service already running ($(cat ${adb_pidfile}))"
|
||||
exit ${rc}
|
||||
else
|
||||
printf "${adb_pid}" > "${adb_pidfile}"
|
||||
fi
|
||||
|
||||
# get current directory and set script/config version
|
||||
#
|
||||
adb_scriptdir="${0%/*}"
|
||||
adb_scriptver="1.2.1"
|
||||
adb_mincfgver="2.2"
|
||||
|
||||
# source in adblock function library
|
||||
#
|
||||
if [ -r "${adb_scriptdir}/adblock-helper.sh" ]
|
||||
then
|
||||
. "${adb_scriptdir}/adblock-helper.sh"
|
||||
else
|
||||
rc=254
|
||||
"${adb_log}" -s -t "adblock[${adb_pid}] error" "adblock function library not found"
|
||||
rm -f "${adb_pidfile}"
|
||||
exit ${rc}
|
||||
if [ -r "${adb_scriptdir}/adblock-helper.sh" ]
|
||||
then
|
||||
. "${adb_scriptdir}/adblock-helper.sh"
|
||||
f_envload
|
||||
else
|
||||
rc=254
|
||||
logger -s -t "adblock[${adb_pid}] error" "adblock function library not found"
|
||||
rm -f "${adb_pidfile}"
|
||||
exit ${rc}
|
||||
fi
|
||||
fi
|
||||
|
||||
# call trap function on error signals (HUP, INT, QUIT, BUS, SEGV, TERM)
|
||||
#
|
||||
trap "rc=250; f_log 'error signal received/trapped' '${rc}'; f_exit" 1 2 3 10 11 15
|
||||
|
||||
# load environment
|
||||
#
|
||||
f_envload
|
||||
|
||||
# start logging
|
||||
#
|
||||
f_log "domain adblock processing started (${adb_scriptver}, ${adb_sysver}, $(/bin/date "+%d.%m.%Y %H:%M:%S"))"
|
||||
trap "rc=250; f_log 'error signal received/trapped'; f_exit" 1 2 3 10 11 15
|
||||
|
||||
# check environment
|
||||
#
|
||||
f_envcheck
|
||||
|
||||
# loop through active adblock domain sources,
|
||||
# download sources, prepare output and store all extracted domains in temp file
|
||||
# main loop for all block list sources
|
||||
#
|
||||
for src_name in ${adb_sources}
|
||||
do
|
||||
# check disabled sources
|
||||
#
|
||||
eval "enabled=\"\${enabled_${src_name}}\""
|
||||
if [ "${enabled}" = "0" ]
|
||||
then
|
||||
if [ -r "${adb_dnsdir}/${adb_dnsprefix}.${src_name}" ]
|
||||
then
|
||||
rm -f "${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
|
||||
if [ "${backup_ok}" = "true" ] && [ -r "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz" ]
|
||||
then
|
||||
rm -f "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz"
|
||||
fi
|
||||
rm_done="true"
|
||||
f_log "=> disabled source '${src_name}' removed"
|
||||
fi
|
||||
"${adb_uci}" -q delete "adblock.${src_name}.adb_src_count"
|
||||
"${adb_uci}" -q delete "adblock.${src_name}.adb_src_timestamp"
|
||||
continue
|
||||
fi
|
||||
|
||||
f_log "=> processing source '${src_name}'"
|
||||
eval "url=\"\${adb_src_${src_name}}\""
|
||||
eval "src_rset=\"\${adb_src_rset_${src_name}}\""
|
||||
eval "list_time=\"\${CONFIG_${src_name}_adb_src_timestamp}\""
|
||||
adb_dnsfile="${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
|
||||
list_time="$(${adb_uci} -q get "adblock.${src_name}.adb_src_timestamp")"
|
||||
f_log "=> processing adblock source '${src_name}'"
|
||||
|
||||
# check 'url' and 'src_rset' values
|
||||
#
|
||||
if [ -z "${url}" ] || [ -z "${src_rset}" ]
|
||||
then
|
||||
"${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=broken config"
|
||||
f_log " broken source configuration, check 'adb_src' and 'adb_src_rset' in config"
|
||||
f_log " broken source configuration, skipped"
|
||||
continue
|
||||
fi
|
||||
|
||||
# prepare find statement with active adblock list sources
|
||||
#
|
||||
if [ -z "${adb_srclist}" ]
|
||||
then
|
||||
adb_srclist="! -name ${adb_dnsprefix}.${src_name}*"
|
||||
else
|
||||
adb_srclist="${adb_srclist} -a ! -name ${adb_dnsprefix}.${src_name}*"
|
||||
fi
|
||||
|
||||
# download only block list with newer/updated timestamp
|
||||
#
|
||||
if [ "${src_name}" = "blacklist" ]
|
||||
then
|
||||
url_time="$(date -r "${url}")"
|
||||
else
|
||||
url_time="$(${adb_fetch} ${fetch_parm} --server-response --spider "${url}" 2>&1 | awk '$0 ~ /Last-Modified/ {printf substr($0,18)}')"
|
||||
url_time="$(${adb_fetch} ${fetch_parm} ${response_parm} "${url}" 2>&1 | awk '$0 ~ /Last-Modified/ {printf substr($0,18)}')"
|
||||
fi
|
||||
if [ -z "${url_time}" ]
|
||||
then
|
||||
url_time="$(date)"
|
||||
f_log " no online timestamp received"
|
||||
f_log " no online timestamp"
|
||||
fi
|
||||
if [ -z "${list_time}" ] || [ "${list_time}" != "${url_time}" ] || [ ! -r "${adb_dnsfile}" ] ||\
|
||||
([ "${backup_ok}" = "true" ] && [ ! -r "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz" ])
|
||||
then
|
||||
if [ "${src_name}" = "blacklist" ]
|
||||
then
|
||||
tmp_domains="$(cat "${url}")"
|
||||
tmp_domains="$(cat "${url}" | strings -n 1)"
|
||||
elif [ "${src_name}" = "shalla" ]
|
||||
then
|
||||
shalla_archive="${adb_tmpdir}/shallalist.tar.gz"
|
||||
shalla_file="${adb_tmpdir}/shallalist.txt"
|
||||
"${adb_fetch}" ${fetch_parm} --output-document="${shalla_archive}" "${url}"
|
||||
"${adb_fetch}" ${fetch_parm} -O "${shalla_archive}" "${url}"
|
||||
rc=${?}
|
||||
if [ $((rc)) -eq 0 ]
|
||||
then
|
||||
@@ -126,17 +117,17 @@ do
|
||||
break
|
||||
fi
|
||||
done
|
||||
tmp_domains="$(cat "${shalla_file}")"
|
||||
tmp_domains="$(cat "${shalla_file}" | strings -n 1)"
|
||||
rm -rf "${adb_tmpdir}/BL"
|
||||
rm -f "${shalla_archive}"
|
||||
rm -f "${shalla_file}"
|
||||
fi
|
||||
else
|
||||
tmp_domains="$(${adb_fetch} ${fetch_parm} --output-document=- "${url}")"
|
||||
tmp_domains="$(${adb_fetch} ${fetch_parm} -O- "${url}" | strings -n 1)"
|
||||
fi
|
||||
rc=${?}
|
||||
else
|
||||
f_log " source doesn't change, no update required"
|
||||
f_log " source doesn't change, skipped"
|
||||
continue
|
||||
fi
|
||||
|
||||
@@ -154,9 +145,23 @@ do
|
||||
unset tmp_domains
|
||||
elif [ $((rc)) -eq 0 ] && [ -z "${tmp_domains}" ]
|
||||
then
|
||||
"${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=empty download"
|
||||
f_log " empty source download finished"
|
||||
continue
|
||||
if [ "${backup_ok}" = "true" ] && [ -r "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz" ]
|
||||
then
|
||||
gunzip -cf "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz" > "${adb_tmpfile}"
|
||||
count="$(wc -l < "${adb_tmpfile}")"
|
||||
"${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=list restored"
|
||||
f_log " empty source download, restored (${count} entries)"
|
||||
else
|
||||
if [ -r "${adb_dnsdir}/${adb_dnsprefix}.${src_name}" ]
|
||||
then
|
||||
rm -f "${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
|
||||
rm_done="true"
|
||||
fi
|
||||
"${adb_uci}" -q delete "adblock.${src_name}.adb_src_count"
|
||||
"${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=empty download"
|
||||
f_log " empty source download, skipped"
|
||||
continue
|
||||
fi
|
||||
else
|
||||
rc=0
|
||||
if [ "${backup_ok}" = "true" ] && [ -r "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz" ]
|
||||
@@ -164,10 +169,16 @@ do
|
||||
gunzip -cf "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz" > "${adb_tmpfile}"
|
||||
count="$(wc -l < "${adb_tmpfile}")"
|
||||
"${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=list restored"
|
||||
f_log " source download failed, list restored (${count} entries)"
|
||||
f_log " source download failed, restored (${count} entries)"
|
||||
else
|
||||
if [ -r "${adb_dnsdir}/${adb_dnsprefix}.${src_name}" ]
|
||||
then
|
||||
rm -f "${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
|
||||
rm_done="true"
|
||||
fi
|
||||
"${adb_uci}" -q delete "adblock.${src_name}.adb_src_count"
|
||||
"${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=download failed"
|
||||
f_log " source download failed"
|
||||
f_log " source download failed, skipped"
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
@@ -184,17 +195,9 @@ do
|
||||
sort -u "${adb_tmpfile}" | eval "${adb_dnsformat}" > "${adb_dnsfile}"
|
||||
fi
|
||||
rc=${?}
|
||||
|
||||
# finish domain processing, prepare find statement with revised adblock list source
|
||||
#
|
||||
if [ $((rc)) -eq 0 ]
|
||||
then
|
||||
if [ -z "${adb_revsrclist}" ]
|
||||
then
|
||||
adb_revsrclist="-name ${adb_dnsprefix}.${src_name}"
|
||||
else
|
||||
adb_revsrclist="${adb_revsrclist} -o -name ${adb_dnsprefix}.${src_name}"
|
||||
fi
|
||||
rev_done="true"
|
||||
f_log " domain merging finished"
|
||||
else
|
||||
rc=0
|
||||
@@ -203,7 +206,9 @@ do
|
||||
then
|
||||
rm -f "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz"
|
||||
fi
|
||||
f_log " domain merging failed, list removed"
|
||||
"${adb_uci}" -q delete "adblock.${src_name}.adb_src_count"
|
||||
"${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=domain merging failed"
|
||||
f_log " domain merging failed, skipped"
|
||||
continue
|
||||
fi
|
||||
else
|
||||
@@ -212,99 +217,58 @@ do
|
||||
then
|
||||
rm -f "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz"
|
||||
fi
|
||||
"${adb_uci}" -q delete "adblock.${src_name}.adb_src_count"
|
||||
"${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=empty domain input"
|
||||
f_log " empty domain input received, list removed"
|
||||
f_log " empty domain input, skipped"
|
||||
continue
|
||||
fi
|
||||
done
|
||||
|
||||
# remove disabled adblock lists and their backups
|
||||
# overall sort, make block list entries unique
|
||||
#
|
||||
if [ -n "${adb_srclist}" ]
|
||||
if [ "${rev_done}" = "true" ] && [ "${mem_ok}" = "true" ]
|
||||
then
|
||||
rm_done="$(find "${adb_dnsdir}" -maxdepth 1 -type f \( ${adb_srclist} \) -print -exec rm -f "{}" \;)"
|
||||
if [ "${backup_ok}" = "true" ] && [ -n "${rm_done}" ]
|
||||
then
|
||||
find "${adb_dir_backup}" -maxdepth 1 -type f \( ${adb_srclist} \) -exec rm -f "{}" \;
|
||||
fi
|
||||
else
|
||||
rm_done="$(find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}*" -print -exec rm -f "{}" \;)"
|
||||
if [ "${backup_ok}" = "true" ]
|
||||
then
|
||||
find "${adb_dir_backup}" -maxdepth 1 -type f -name "${adb_dnsprefix}*" -exec rm -f "{}" \;
|
||||
fi
|
||||
fi
|
||||
if [ -n "${rm_done}" ]
|
||||
then
|
||||
f_rmconfig "${rm_done}"
|
||||
f_log "disabled adblock lists removed"
|
||||
fi
|
||||
|
||||
# make separate adblock lists entries unique
|
||||
#
|
||||
if [ "${mem_ok}" = "true" ] && [ -n "${adb_revsrclist}" ]
|
||||
then
|
||||
f_log "remove duplicates in separate adblock lists"
|
||||
|
||||
# generate a unique overall block list
|
||||
#
|
||||
sort -u "${adb_dnsdir}/${adb_dnsprefix}."* > "${adb_tmpdir}/blocklist.overall"
|
||||
|
||||
# loop through all separate lists, ordered by size (ascending)
|
||||
#
|
||||
f_log "remove duplicates in separate block lists"
|
||||
for list in $(ls -ASr "${adb_dnsdir}/${adb_dnsprefix}"*)
|
||||
do
|
||||
# check overall block list vs. separate block list,
|
||||
# write all duplicate entries to separate list
|
||||
#
|
||||
list="${list/*./}"
|
||||
sort "${adb_tmpdir}/blocklist.overall" "${adb_dnsdir}/${adb_dnsprefix}.${list}" | uniq -d > "${adb_tmpdir}/tmp.${list}"
|
||||
mv -f "${adb_tmpdir}/tmp.${list}" "${adb_dnsdir}/${adb_dnsprefix}.${list}"
|
||||
|
||||
# write all unique entries back to overall block list
|
||||
#
|
||||
sort "${adb_tmpdir}/blocklist.overall" "${adb_dnsdir}/${adb_dnsprefix}.${list}" | uniq -u > "${adb_tmpdir}/tmp.overall"
|
||||
mv -f "${adb_tmpdir}/tmp.overall" "${adb_tmpdir}/blocklist.overall"
|
||||
if [ -s "${adb_tmpdir}/blocklist.overall" ]
|
||||
then
|
||||
sort "${adb_tmpdir}/blocklist.overall" "${adb_tmpdir}/blocklist.overall" "${adb_dnsdir}/${adb_dnsprefix}.${list}" | uniq -u > "${adb_tmpdir}/tmp.blocklist"
|
||||
cat "${adb_tmpdir}/tmp.blocklist" > "${adb_dnsdir}/${adb_dnsprefix}.${list}"
|
||||
fi
|
||||
cat "${adb_dnsdir}/${adb_dnsprefix}.${list}" >> "${adb_tmpdir}/blocklist.overall"
|
||||
done
|
||||
rm -f "${adb_tmpdir}/blocklist.overall"
|
||||
fi
|
||||
|
||||
# restart & check dnsmasq with newly generated set of adblock lists
|
||||
# restart & check dnsmasq with generated set of block lists
|
||||
#
|
||||
f_cntconfig
|
||||
adb_count="$(${adb_uci} -q get "adblock.global.adb_overall_count")"
|
||||
if [ -n "${adb_revsrclist}" ] || [ -n "${rm_done}" ]
|
||||
if [ "${rev_done}" = "true" ] || [ "${rm_done}" = "true" ] || [ -n "${mv_done}" ]
|
||||
then
|
||||
"${adb_uci}" -q set "adblock.global.adb_dnstoggle=on"
|
||||
"${adb_uci}" -q delete "adblock.global.adb_dnstoggle"
|
||||
/etc/init.d/dnsmasq restart
|
||||
sleep 1
|
||||
check="$(pgrep -f "dnsmasq")"
|
||||
if [ -n "${check}" ]
|
||||
then
|
||||
f_log "adblock lists with overall ${adb_count} domains loaded"
|
||||
f_cntconfig
|
||||
f_log "block lists with overall ${adb_count} domains loaded"
|
||||
else
|
||||
f_log "dnsmasq restart failed, retry without newly generated block lists"
|
||||
rm_done="$(find "${adb_dnsdir}" -maxdepth 1 -type f \( ${adb_revsrclist} \) -print -exec rm -f "{}" \;)"
|
||||
if [ -n "${rm_done}" ]
|
||||
f_rmdns
|
||||
sleep 1
|
||||
check="$(pgrep -f "dnsmasq")"
|
||||
if [ -n "${check}" ]
|
||||
then
|
||||
f_log "bogus adblock lists removed"
|
||||
f_rmconfig "${rm_done}"
|
||||
/etc/init.d/dnsmasq restart
|
||||
sleep 1
|
||||
check="$(pgrep -f "dnsmasq")"
|
||||
if [ -n "${check}" ]
|
||||
then
|
||||
f_cntconfig
|
||||
f_log "adblock lists with overall ${adb_count} domains loaded"
|
||||
else
|
||||
rc=100
|
||||
f_log "dnsmasq restart failed, please check 'logread' output" "${rc}"
|
||||
f_exit
|
||||
fi
|
||||
f_log "dnsmasq restart without block lists succeeded, please check your configuration"
|
||||
else
|
||||
f_log "dnsmasq restart without block lists failed, please check your configuration"
|
||||
fi
|
||||
rc=100
|
||||
f_exit
|
||||
fi
|
||||
else
|
||||
f_log "adblock lists with overall ${adb_count} domains are still valid, no update required"
|
||||
f_cntconfig
|
||||
f_log "block lists with overall ${adb_count} domains are still valid, no update required"
|
||||
fi
|
||||
|
||||
# remove temporary files and exit
|
||||
|
||||
Reference in New Issue
Block a user