mirror of
https://github.com/novatiq/packages.git
synced 2026-04-30 07:28:39 +01:00
adblock: release 3.6.0
* add adblock dns query reporting via tcpdump (see readme for details) * fix tld compression on low memory systems (< 64 MB) * fix various small issues Signed-off-by: Dirk Brenken <dev@brenken.org>
This commit is contained in:
Dirk Brenken
+204
-27
@@ -10,7 +10,7 @@
|
||||
#
|
||||
LC_ALL=C
|
||||
PATH="/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
adb_ver="3.5.5-4"
|
||||
adb_ver="3.6.0"
|
||||
adb_sysver="unknown"
|
||||
adb_enabled=0
|
||||
adb_debug=0
|
||||
@@ -18,7 +18,7 @@ adb_backup_mode=0
|
||||
adb_forcesrt=0
|
||||
adb_forcedns=0
|
||||
adb_jail=0
|
||||
adb_maxqueue=4
|
||||
adb_maxqueue=8
|
||||
adb_notify=0
|
||||
adb_notifycnt=0
|
||||
adb_triggerdelay=0
|
||||
@@ -35,6 +35,12 @@ adb_rtfile="/tmp/adb_runtime.json"
|
||||
adb_hashutil="$(command -v sha256sum)"
|
||||
adb_hashold=""
|
||||
adb_hashnew=""
|
||||
adb_report=0
|
||||
adb_repiface="br-lan"
|
||||
adb_repdir="/tmp"
|
||||
adb_reputil="$(command -v tcpdump)"
|
||||
adb_repchunkcnt="5"
|
||||
adb_repchunksize="1"
|
||||
adb_cnt=""
|
||||
adb_rc=0
|
||||
adb_action="${1:-"start"}"
|
||||
@@ -300,7 +306,7 @@ f_rmdns()
|
||||
fi
|
||||
f_rmtemp
|
||||
fi
|
||||
f_log "debug" "f_rmdns::: dns: ${adb_dns}, dns_dir: ${adb_dnsdir}, dns_prefix: ${adb_dnsprefix}, dns_file: ${adb_dnsfile}, rt_file: ${adb_rtfile}, backup_dir: ${adb_backupdir}"
|
||||
f_log "debug" "f_rmdns ::: dns: ${adb_dns}, dns_dir: ${adb_dnsdir}, dns_prefix: ${adb_dnsprefix}, dns_file: ${adb_dnsfile}, rt_file: ${adb_rtfile}, backup_dir: ${adb_backupdir}"
|
||||
}
|
||||
|
||||
# commit uci changes
|
||||
@@ -325,7 +331,7 @@ f_uci()
|
||||
esac
|
||||
fi
|
||||
fi
|
||||
f_log "debug" "f_uci ::: config: ${config}, change: ${change}"
|
||||
f_log "debug" "f_uci ::: config: ${config}, change: ${change}"
|
||||
}
|
||||
|
||||
# list/overall count
|
||||
@@ -337,7 +343,7 @@ f_count()
|
||||
adb_cnt=0
|
||||
if [ -s "${adb_dnsdir}/${adb_dnsfile}" ] && ([ -z "${mode}" ] || [ "${mode}" = "final" ])
|
||||
then
|
||||
adb_cnt="$(( $(wc -l 2>/dev/null < "${adb_dnsdir}/${adb_dnsfile}") ))"
|
||||
adb_cnt="$(wc -l 2>/dev/null < "${adb_dnsdir}/${adb_dnsfile}")"
|
||||
if [ -s "${adb_tmpdir}/tmp.add_whitelist" ]
|
||||
then
|
||||
adb_cnt="$(( ${adb_cnt} - $(wc -l 2>/dev/null < "${adb_tmpdir}/tmp.add_whitelist") ))"
|
||||
@@ -488,7 +494,7 @@ f_dnsup()
|
||||
cnt=$((cnt+1))
|
||||
sleep 1
|
||||
done
|
||||
f_log "debug" "f_dnsup::: cache_util: ${cache_util:-"-"}, cache_rc: ${cache_rc:-"-"}, cache_flush: ${adb_dnsflush}, cache_cnt: ${cnt}, rc: ${adb_rc}"
|
||||
f_log "debug" "f_dnsup ::: cache_util: ${cache_util:-"-"}, cache_rc: ${cache_rc:-"-"}, cache_flush: ${adb_dnsflush}, cache_cnt: ${cnt}, rc: ${adb_rc}"
|
||||
return ${adb_rc}
|
||||
}
|
||||
|
||||
@@ -557,7 +563,7 @@ f_list()
|
||||
;;
|
||||
esac
|
||||
f_count "${mode}"
|
||||
f_log "debug" "f_list ::: name: ${src_name:-"-"}, mode: ${mode}, cnt: ${adb_cnt}, in_rc: ${in_rc}, out_rc: ${adb_rc}"
|
||||
f_log "debug" "f_list ::: name: ${src_name:-"-"}, mode: ${mode}, cnt: ${adb_cnt}, in_rc: ${in_rc}, out_rc: ${adb_rc}"
|
||||
}
|
||||
|
||||
# top level domain compression
|
||||
@@ -581,6 +587,7 @@ f_tld()
|
||||
if [ ${?} -eq 0 ]
|
||||
then
|
||||
cnt_tld="$(wc -l 2>/dev/null < "${source}")"
|
||||
rm -f "${temp}"
|
||||
else
|
||||
mv -f "${temp}" > "${source}"
|
||||
fi
|
||||
@@ -589,7 +596,7 @@ f_tld()
|
||||
mv -f "${temp}" "${source}"
|
||||
fi
|
||||
fi
|
||||
f_log "debug" "f_tld ::: source: ${source}, cnt: ${cnt:-"-"}, cnt_srt: ${cnt_srt:-"-"}, cnt_tld: ${cnt_tld:-"-"}"
|
||||
f_log "debug" "f_tld ::: source: ${source}, cnt: ${cnt:-"-"}, cnt_srt: ${cnt_srt:-"-"}, cnt_tld: ${cnt_tld:-"-"}"
|
||||
}
|
||||
|
||||
# blocklist hash compare
|
||||
@@ -618,7 +625,7 @@ f_hash()
|
||||
adb_hashnew=""
|
||||
fi
|
||||
fi
|
||||
f_log "debug" "f_hash ::: hash_util: ${adb_hashutil}, hash: ${hash}, out_rc: ${hash_rc}"
|
||||
f_log "debug" "f_hash ::: hash_util: ${adb_hashutil}, hash: ${hash}, out_rc: ${hash_rc}"
|
||||
return ${hash_rc}
|
||||
}
|
||||
|
||||
@@ -628,6 +635,8 @@ f_switch()
|
||||
{
|
||||
local status cnt mode="${1}"
|
||||
|
||||
json_load_file "${adb_rtfile}" >/dev/null 2>&1
|
||||
json_select "data"
|
||||
json_get_var status "adblock_status"
|
||||
json_get_var cnt "overall_domains"
|
||||
|
||||
@@ -739,6 +748,8 @@ f_jsnup()
|
||||
mode="backup mode"
|
||||
fi
|
||||
|
||||
json_load_file "${adb_rtfile}" >/dev/null 2>&1
|
||||
json_select "data"
|
||||
if [ -z "${adb_fetchinfo}" ]
|
||||
then
|
||||
json_get_var adb_fetchinfo "fetch_utility"
|
||||
@@ -752,6 +763,11 @@ f_jsnup()
|
||||
then
|
||||
json_get_var run_time "last_rundate"
|
||||
fi
|
||||
|
||||
> "${adb_rtfile}"
|
||||
json_load_file "${adb_rtfile}" >/dev/null 2>&1
|
||||
json_init
|
||||
json_add_object "data"
|
||||
json_add_string "adblock_status" "${status}"
|
||||
json_add_string "adblock_version" "${adb_ver}"
|
||||
json_add_string "overall_domains" "${adb_cnt:-0} (${mode})"
|
||||
@@ -759,15 +775,16 @@ f_jsnup()
|
||||
json_add_string "dns_backend" "${adb_dns} (${adb_dnsdir})"
|
||||
json_add_string "last_rundate" "${run_time:-"-"}"
|
||||
json_add_string "system_release" "${adb_sysver}"
|
||||
json_close_object
|
||||
json_dump > "${adb_rtfile}"
|
||||
|
||||
if [ ${adb_notify} -eq 1 ] && [ ${no_mail} -eq 0 ] && [ -x /etc/adblock/adblock.notify ] && \
|
||||
([ "${status}" = "error" ] || ([ "${status}" = "enabled" ] && [ ${adb_cnt} -le ${adb_notifycnt} ]))
|
||||
then
|
||||
(/etc/adblock/adblock.notify >/dev/null 2>&1) &
|
||||
(/etc/adblock/adblock.notify >/dev/null 2>&1)&
|
||||
bg_pid=${!}
|
||||
fi
|
||||
f_log "debug" "f_jsnup::: status: ${status}, mode: ${mode}, cnt: ${adb_cnt}, notify: ${adb_notify}, notify_cnt: ${adb_notifycnt}, notify_pid: ${bg_pid:-"-"}"
|
||||
f_log "debug" "f_jsnup ::: status: ${status}, mode: ${mode}, cnt: ${adb_cnt}, notify: ${adb_notify}, notify_cnt: ${adb_notifycnt}, notify_pid: ${bg_pid:-"-"}"
|
||||
}
|
||||
|
||||
# write to syslog
|
||||
@@ -803,7 +820,7 @@ f_main()
|
||||
> "${adb_tmpdir}/tmp.raw_whitelist"
|
||||
> "${adb_tmpdir}/tmp.add_whitelist"
|
||||
> "${adb_tmpdir}/tmp.rem_whitelist"
|
||||
f_log "debug" "f_main ::: dns: ${adb_dns}, fetch_util: ${adb_fetchinfo}, backup: ${adb_backup}, backup_mode: ${adb_backup_mode}, dns_jail: ${adb_jail}, force_srt: ${adb_forcesrt}, force_dns: ${adb_forcedns}, mem_total: ${mem_total:-0}, mem_free: ${mem_free:-0}, max_queue: ${adb_maxqueue}"
|
||||
f_log "debug" "f_main ::: dns: ${adb_dns}, fetch_util: ${adb_fetchinfo}, backup: ${adb_backup}, backup_mode: ${adb_backup_mode}, dns_jail: ${adb_jail}, force_srt: ${adb_forcesrt}, force_dns: ${adb_forcedns}, mem_total: ${mem_total:-0}, mem_free: ${mem_free:-0}, max_queue: ${adb_maxqueue}"
|
||||
|
||||
# prepare whitelist entries
|
||||
#
|
||||
@@ -848,7 +865,7 @@ f_main()
|
||||
|
||||
# basic pre-checks
|
||||
#
|
||||
f_log "debug" "f_main ::: name: ${src_name}, enabled: ${enabled}"
|
||||
f_log "debug" "f_main ::: name: ${src_name}, enabled: ${enabled}"
|
||||
if [ "${enabled}" != "1" ] || [ -z "${src_url}" ] || [ -z "${src_rset}" ]
|
||||
then
|
||||
f_list remove
|
||||
@@ -894,7 +911,7 @@ f_main()
|
||||
fi
|
||||
else
|
||||
src_log="$(printf '%s' "${src_log}" | awk '{ORS=" ";print $0}')"
|
||||
f_log "debug" "f_main ::: name: ${src_name}, url: ${src_url}, rc: ${adb_rc}, log: ${src_log:-"-"}"
|
||||
f_log "debug" "f_main ::: name: ${src_name}, url: ${src_url}, rc: ${adb_rc}, log: ${src_log:-"-"}"
|
||||
fi
|
||||
) &
|
||||
else
|
||||
@@ -925,7 +942,7 @@ f_main()
|
||||
done
|
||||
else
|
||||
src_log="$(printf '%s' "${src_log}" | awk '{ORS=" ";print $0}')"
|
||||
f_log "debug" "f_main ::: name: ${src_name}, url: ${src_url}, rc: ${adb_rc}, log: ${src_log:-"-"}"
|
||||
f_log "debug" "f_main ::: name: ${src_name}, url: ${src_url}, rc: ${adb_rc}, log: ${src_log:-"-"}"
|
||||
fi
|
||||
if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpload}" ]
|
||||
then
|
||||
@@ -979,7 +996,7 @@ f_main()
|
||||
fi
|
||||
else
|
||||
src_log="$(printf '%s' "${src_log}" | awk '{ORS=" ";print $0}')"
|
||||
f_log "debug" "f_main ::: name: ${src_name}, url: ${src_url}, rc: ${adb_rc}, log: ${src_log:-"-"}"
|
||||
f_log "debug" "f_main ::: name: ${src_name}, url: ${src_url}, rc: ${adb_rc}, log: ${src_log:-"-"}"
|
||||
if [ ${adb_backup} -eq 1 ]
|
||||
then
|
||||
f_list restore
|
||||
@@ -1032,6 +1049,171 @@ f_main()
|
||||
exit ${adb_rc}
|
||||
}
|
||||
|
||||
# trace dns queries via tcpdump and prepare a report
|
||||
#
|
||||
f_report()
|
||||
{
|
||||
local bg_pid total blocked percent rep_clients rep_domains rep_blocked index hold cnt=0 print="${1:-"true"}"
|
||||
|
||||
if [ ! -x "${adb_reputil}" ]
|
||||
then
|
||||
f_log "info" "Please install the package 'tcpdump-mini' manually to use the adblock reporting feature!"
|
||||
return 0
|
||||
fi
|
||||
|
||||
bg_pid="$(pgrep -f "^${adb_reputil}.*adb_report\.pcap$" | awk '{ORS=" "; print $1}')"
|
||||
if [ ${adb_report} -eq 0 ] || ([ -n "${bg_pid}" ] && [ "${adb_action}" = "stop" ])
|
||||
then
|
||||
if [ -n "${bg_pid}" ]
|
||||
then
|
||||
kill -HUP ${bg_pid}
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
if [ -z "${bg_pid}" ] && [ "${adb_action}" != "report" ]
|
||||
then
|
||||
("${adb_reputil}" -nn -s0 -l -i ${adb_repiface} port 53 -C${adb_repchunksize} -W${adb_repchunkcnt} -w "${adb_repdir}/adb_report.pcap" >/dev/null 2>&1 &)
|
||||
fi
|
||||
if [ "${adb_action}" = "report" ]
|
||||
then
|
||||
> "${adb_repdir}/adb_report.raw"
|
||||
for file in "${adb_repdir}"/adb_report.pcap*
|
||||
do
|
||||
(
|
||||
"${adb_reputil}" -nn -tttt -r $file 2>/dev/null | \
|
||||
awk -v cnt=${cnt} '!/\.lan\. /&&/ A[\? ]+|NXDomain/{a=$1;b=substr($2,0,8);c=$4;sub(/\.[0-9]+$/,"",c); \
|
||||
d=cnt $7;e=$(NF-1);sub(/[0-9]\/[0-9]\/[0-9]/,"NX",e);sub(/\.$/,"",e);sub(/([0-9]{1,3}\.){3}[0-9]{1,3}/,"OK",e);printf("%s\t%s\t%s\t%s\t%s\n", a,b,c,d,e)}' >> "${adb_repdir}/adb_report.raw"
|
||||
)&
|
||||
hold=$(( cnt % adb_maxqueue ))
|
||||
if [ ${hold} -eq 0 ]
|
||||
then
|
||||
wait
|
||||
fi
|
||||
cnt=$(( cnt + 1 ))
|
||||
done
|
||||
wait
|
||||
|
||||
if [ -s "${adb_repdir}/adb_report.raw" ]
|
||||
then
|
||||
awk '{printf("%s\t%s\t%s\t%s\t%s\t%s\n", $4,$5,$1,$2,$3,$4)}' "${adb_repdir}/adb_report.raw" | \
|
||||
sort -ur | uniq -uf2 | awk '{currA=($6+0);currB=$6;currC=substr($6,length($6),1); \
|
||||
if(reqA==currB){reqA=0;printf("%s\t%s\n",d,$2)}else if(currC=="+"){reqA=currA;d=$3"\t"$4"\t"$5"\t"$2}}' | sort -ur > "${adb_repdir}/adb_report"
|
||||
fi
|
||||
|
||||
if [ -s "${adb_repdir}/adb_report" ]
|
||||
then
|
||||
total="$(wc -l < ${adb_repdir}/adb_report)"
|
||||
blocked="$(awk '{if($5=="NX")print $4}' ${adb_repdir}/adb_report | wc -l)"
|
||||
percent="$(awk -v t=${total} -v b=${blocked} 'BEGIN{printf("%.2f %s\n",b/t*100, "%")}')"
|
||||
rep_clients="$(awk '{print $3}' ${adb_repdir}/adb_report | sort | uniq -c | sort -r | awk '{ORS=" ";if(NR<=10) printf("%s_%s ",$1,$2)}')"
|
||||
rep_domains="$(awk '{if($5!="NX")print $4}' ${adb_repdir}/adb_report | sort | uniq -c | sort -r | awk '{ORS=" ";if(NR<=10)printf("%s_%s ",$1,$2)}')"
|
||||
rep_blocked="$(awk '{if($5=="NX")print $4}' ${adb_repdir}/adb_report | sort | uniq -c | sort -r | awk '{ORS=" ";if(NR<=10)printf("%s_%s ",$1,$2)}')"
|
||||
|
||||
> "${adb_repdir}/adb_report.json"
|
||||
json_load_file "${adb_repdir}/adb_report.json" >/dev/null 2>&1
|
||||
json_init
|
||||
json_add_object "data"
|
||||
json_add_string "start_date" "$(awk 'END{printf("%s",$1)}' ${adb_repdir}/adb_report)"
|
||||
json_add_string "start_time" "$(awk 'END{printf("%s",$2)}' ${adb_repdir}/adb_report)"
|
||||
json_add_string "end_date" "$(awk 'NR==1{printf("%s",$1)}' ${adb_repdir}/adb_report)"
|
||||
json_add_string "end_time" "$(awk 'NR==1{printf("%s",$2)}' ${adb_repdir}/adb_report)"
|
||||
json_add_string "total" "${total}"
|
||||
json_add_string "blocked" "${blocked}"
|
||||
json_add_string "percent" "${percent}"
|
||||
json_close_array
|
||||
json_add_array "top_clients"
|
||||
for client in ${rep_clients}
|
||||
do
|
||||
json_add_object
|
||||
json_add_string "count" "${client%_*}"
|
||||
json_add_string "address" "${client#*_}"
|
||||
json_close_object
|
||||
done
|
||||
json_close_array
|
||||
json_add_array "top_domains"
|
||||
for domain in ${rep_domains}
|
||||
do
|
||||
json_add_object
|
||||
json_add_string "count" "${domain%_*}"
|
||||
json_add_string "address" "${domain#*_}"
|
||||
json_close_object
|
||||
done
|
||||
json_close_array
|
||||
json_add_array "top_blocked"
|
||||
for block in ${rep_blocked}
|
||||
do
|
||||
json_add_object
|
||||
json_add_string "count" "${block%_*}"
|
||||
json_add_string "address" "${block#*_}"
|
||||
json_close_object
|
||||
done
|
||||
json_close_object
|
||||
json_dump > "${adb_repdir}/adb_report.json"
|
||||
fi
|
||||
rm -f "${adb_repdir}/adb_report.raw"
|
||||
|
||||
if [ "${print}" = "true" ]
|
||||
then
|
||||
if [ -s "${adb_repdir}/adb_report.json" ]
|
||||
then
|
||||
printf "%s\n%s\n%s\n" ":::" "::: Adblock DNS-Query Report" ":::"
|
||||
json_load_file "${adb_repdir}/adb_report.json"
|
||||
json_select "data"
|
||||
json_get_keys keylist
|
||||
for key in ${keylist}
|
||||
do
|
||||
json_get_var value "${key}"
|
||||
eval "${key}=\"${value}\""
|
||||
done
|
||||
printf " + %s\n + %s\n" "Start ::: ${start_date}, ${start_time}" "End ::: ${end_date}, ${end_time}"
|
||||
printf " + %s\n + %s %s\n" "Total ::: ${total}" "Blocked ::: ${blocked}" "(${percent})"
|
||||
json_select ".."
|
||||
if json_get_type Status "top_clients" && [ "${Status}" = "array" ]
|
||||
then
|
||||
printf "%s\n%s\n" ":::" "::: Top 10 Clients"
|
||||
json_select "top_clients"
|
||||
index=1
|
||||
while json_get_type Status ${index} && [ "${Status}" = "object" ]
|
||||
do
|
||||
json_get_values client ${index}
|
||||
printf " + %-9s::: %s\n" ${client}
|
||||
index=$((index + 1))
|
||||
done
|
||||
fi
|
||||
json_select ".."
|
||||
if json_get_type Status "top_domains" && [ "${Status}" = "array" ]
|
||||
then
|
||||
printf "%s\n%s\n" ":::" "::: Top 10 Domains"
|
||||
json_select "top_domains"
|
||||
index=1
|
||||
while json_get_type Status ${index} && [ "${Status}" = "object" ]
|
||||
do
|
||||
json_get_values domain ${index}
|
||||
printf " + %-9s::: %s\n" ${domain}
|
||||
index=$((index + 1))
|
||||
done
|
||||
fi
|
||||
json_select ".."
|
||||
if json_get_type Status "top_blocked" && [ "${Status}" = "array" ]
|
||||
then
|
||||
printf "%s\n%s\n" ":::" "::: Top 10 Blocked Domains"
|
||||
json_select "top_blocked"
|
||||
index=1
|
||||
while json_get_type Status ${index} && [ "${Status}" = "object" ]
|
||||
do
|
||||
json_get_values blocked ${index}
|
||||
printf " + %-9s::: %s\n" ${blocked}
|
||||
index=$((index + 1))
|
||||
done
|
||||
fi
|
||||
else
|
||||
printf "%s\n" "::: no reporting data available yet"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
f_log "debug" "f_report ::: action: ${adb_action}, report: ${adb_report}, print: ${print}, reputil: ${adb_reputil}, repdir: ${adb_repdir}, repiface: ${adb_repiface}, repchunksize: ${adb_repchunksize}, repchunkcnt: ${adb_repchunkcnt}, bg_pid: ${bg_pid}"
|
||||
}
|
||||
|
||||
# source required system libraries
|
||||
#
|
||||
if [ -r "/lib/functions.sh" ] && [ -r "/usr/share/libubox/jshn.sh" ]
|
||||
@@ -1042,25 +1224,16 @@ else
|
||||
f_log "err" "system libraries not found"
|
||||
fi
|
||||
|
||||
# initialize json runtime file
|
||||
#
|
||||
json_load_file "${adb_rtfile}" >/dev/null 2>&1
|
||||
json_select data >/dev/null 2>&1
|
||||
if [ ${?} -ne 0 ]
|
||||
then
|
||||
> "${adb_rtfile}"
|
||||
json_init
|
||||
json_add_object "data"
|
||||
fi
|
||||
|
||||
# handle different adblock actions
|
||||
#
|
||||
f_envload
|
||||
case "${adb_action}" in
|
||||
stop)
|
||||
f_report false
|
||||
f_rmdns
|
||||
;;
|
||||
restart)
|
||||
f_report false
|
||||
f_rmdns
|
||||
f_envcheck
|
||||
f_main
|
||||
@@ -1071,10 +1244,14 @@ case "${adb_action}" in
|
||||
resume)
|
||||
f_switch resume
|
||||
;;
|
||||
report)
|
||||
f_report "${2}"
|
||||
;;
|
||||
query)
|
||||
f_query "${2}"
|
||||
;;
|
||||
start|reload)
|
||||
f_report false
|
||||
f_envcheck
|
||||
f_main
|
||||
;;
|
||||
|
||||
Reference in New Issue
Block a user