cubixle/packages
1
0
Fork 0
mirror of https://github.com/novatiq/packages.git synced 2026-04-30 07:28:39 +01:00
Code Issues Packages Projects Releases Wiki Activity

adblock: backport updates upto 2.4.0-2

Browse Source 
Signed-off-by: Hannu Nyman <hannu.nyman@iki.fi>

Original commit messages:

adblock: update 2.3.2
* optimize memory consumption &
  enable overall sort only on devices with > 64MB RAM,
  this prevents sort related kernel dumps
(cherry picked from commit 8c5b9a0802)

adblock: release 2.4.0
* add tld compression,
  this new "top level domain compression" removes up to 40 thousand
  needless host entries from the block lists and
  lowers the memory footprint for the dns backends by 8-10 MByte
* optimize restart behavior in case of an error
* cosmetics
(cherry picked from commit ed470f0dcc)

adblock: release 2.4.0 (release 2)
* add missing sort step if tld compression was disabled
(cherry picked from commit b3b9972eac)
This commit is contained in:
Dirk Brenken
2017-03-04 18:20:24 +02:00
committed by Hannu Nyman
parent d98b545bf9
commit afe6be45e3
4 changed files with 75 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
net/adblock/Makefile
+2 -2
View File
@@ -6,8 +6,8 @@
include $(TOPDIR)/rules.mk include $(TOPDIR)/rules.mk
PKG_NAME:=adblock PKG_NAME:=adblock
PKG_VERSION:=2.3.1 PKG_VERSION:=2.4.0
PKG_RELEASE:=1 PKG_RELEASE:=2
PKG_LICENSE:=GPL-3.0+ PKG_LICENSE:=GPL-3.0+
PKG_MAINTAINER:=Dirk Brenken <dev@brenken.org> PKG_MAINTAINER:=Dirk Brenken <dev@brenken.org>
net/adblock/files/README.md
+6 -4
View File
@@ -57,6 +57,7 @@ A lot of people already use adblocker plugins within their desktop browsers, but
* automatically selects uclient-fetch or wget as download utility (other tools like curl or aria2c are supported as well) * automatically selects uclient-fetch or wget as download utility (other tools like curl or aria2c are supported as well)
* automatically supports a wide range of router modes, even AP modes are supported * automatically supports a wide range of router modes, even AP modes are supported
* full IPv4 and IPv6 support * full IPv4 and IPv6 support
* supports tld compression (top level domain compression), this feature removes thousands of needless host entries from the block lists and lowers the memory footprint for the dns backends
* each block list source will be updated and processed separately * each block list source will be updated and processed separately
* block list source parsing by fast & flexible regex rulesets * block list source parsing by fast & flexible regex rulesets
* overall duplicate removal in separate block lists * overall duplicate removal in separate block lists
@@ -120,6 +121,7 @@ A lot of people already use adblocker plugins within their desktop browsers, but
* adb\_iface => restrict the procd interface trigger to a (list of) certain wan interface(s) or disable it at all (default: not set, disabled) * adb\_iface => restrict the procd interface trigger to a (list of) certain wan interface(s) or disable it at all (default: not set, disabled)
* adb\_fetch => full path to a different download utility, see example below (default: not set, use wget) * adb\_fetch => full path to a different download utility, see example below (default: not set, use wget)
* adb\_fetchparm => options for the download utility, see example below (default: not set, use wget options) * adb\_fetchparm => options for the download utility, see example below (default: not set, use wget options)
* adb\_tldcomp => enable/disable tld compression (default: '1', enabled)
## Examples ## Examples
@@ -136,19 +138,19 @@ If you use manual configuration for unbound, then just include the following lin
<pre><code> <pre><code>
wget (default): wget (default):
option adb_fetch="/usr/bin/wget" option adb_fetch="/usr/bin/wget"
option adb_fetchparm="--no-config --quiet --tries=1 --no-cache --no-cookies --max-redirect=0 --timeout=5 --no-check-certificate -O" option adb_fetchparm="--no-config --quiet --no-cache --no-cookies --max-redirect=0 --timeout=10 --no-check-certificate -O"
aria2c: aria2c:
option adb_fetch '/usr/bin/aria2c' option adb_fetch '/usr/bin/aria2c'
option adb_fetchparm '-q --max-tries=1 --timeout=5 --allow-overwrite=true --auto-file-renaming=false --check-certificate=false -o' option adb_fetchparm '-q --timeout=10 --allow-overwrite=true --auto-file-renaming=false --check-certificate=false -o'
uclient-fetch: uclient-fetch:
option adb_fetch '/bin/uclient-fetch' option adb_fetch '/bin/uclient-fetch'
option adb_fetchparm '-q --timeout=5 --no-check-certificate -O' option adb_fetchparm '-q --timeout=10 --no-check-certificate -O'
curl: curl:
option adb_fetch '/usr/bin/curl' option adb_fetch '/usr/bin/curl'
option adb_fetchparm '-s --retry 1 --connect-timeout 5 --insecure -o' option adb_fetchparm '-s --connect-timeout 10 --insecure -o'
</code></pre> </code></pre>
**receive adblock statistics via ubus:** **receive adblock statistics via ubus:**
net/adblock/files/adblock.init
+1 -6
View File
@@ -37,7 +37,6 @@ start_service()
if [ $(/etc/init.d/adblock enabled; printf ${?}) -eq 0 ] if [ $(/etc/init.d/adblock enabled; printf ${?}) -eq 0 ]
then then
procd_open_instance "adblock" procd_open_instance "adblock"
procd_set_param env adb_procd="true"
procd_set_param command "${adb_script}" "${@}" procd_set_param command "${adb_script}" "${@}"
procd_set_param stdout 1 procd_set_param stdout 1
procd_set_param stderr 1 procd_set_param stderr 1
@@ -52,7 +51,6 @@ reload_service()
stop_service() stop_service()
{ {
export adb_procd="true"
rc_procd "${adb_script}" stop rc_procd "${adb_script}" stop
} }
@@ -73,7 +71,6 @@ resume()
query() query()
{ {
export adb_procd="true"
rc_procd "${adb_script}" query "${1}" rc_procd "${adb_script}" query "${1}"
} }
@@ -81,8 +78,6 @@ service_triggers()
{ {
local iface="$(uci -q get adblock.global.adb_iface)" local iface="$(uci -q get adblock.global.adb_iface)"
procd_open_trigger
procd_add_config_trigger "config.change" "adblock" /etc/init.d/adblock start
if [ -z "${iface}" ] if [ -z "${iface}" ]
then then
procd_add_raw_trigger "interface.*.up" 1000 /etc/init.d/adblock start procd_add_raw_trigger "interface.*.up" 1000 /etc/init.d/adblock start
@@ -92,5 +87,5 @@ service_triggers()
procd_add_interface_trigger "interface.*.up" "${name}" /etc/init.d/adblock start procd_add_interface_trigger "interface.*.up" "${name}" /etc/init.d/adblock start
done done
fi fi
procd_close_trigger procd_add_config_trigger "config.change" "adblock" /etc/init.d/adblock start
} }
net/adblock/files/adblock.sh
+66 -52
View File
@@ -10,10 +10,11 @@
# #
LC_ALL=C LC_ALL=C
PATH="/usr/sbin:/usr/bin:/sbin:/bin" PATH="/usr/sbin:/usr/bin:/sbin:/bin"
adb_ver="2.3.1" adb_ver="2.4.0-2"
adb_enabled=1 adb_enabled=1
adb_debug=0 adb_debug=0
adb_backup=0 adb_backup=0
adb_tldcomp=1
adb_backupdir="/mnt" adb_backupdir="/mnt"
adb_whitelist="/etc/adblock/adblock.whitelist" adb_whitelist="/etc/adblock/adblock.whitelist"
adb_whitelist_rset="\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\"^\"\$1\"\\\|[.]\"\$1)}" adb_whitelist_rset="\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\"^\"\$1\"\\\|[.]\"\$1)}"
@@ -169,9 +170,12 @@ f_envcheck()
# #
f_rmtemp() f_rmtemp()
{ {
rm -f "${adb_tmpload}" if [ -d "${adb_tmpdir}" ]
rm -f "${adb_tmpfile}" then
rm -rf "${adb_tmpdir}" rm -f "${adb_tmpload}"
rm -f "${adb_tmpfile}"
rm -rf "${adb_tmpdir}"
fi
} }
# f_rmdns: remove dns related files & directories # f_rmdns: remove dns related files & directories
@@ -321,7 +325,11 @@ f_log()
then then
logger -t "adblock-[${adb_ver}] ${class}" "Please check the online documentation 'https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md'" logger -t "adblock-[${adb_ver}] ${class}" "Please check the online documentation 'https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md'"
f_rmtemp f_rmtemp
f_rmdns if [ "$(ls -dA "${adb_dnsdir}/${adb_dnsprefix}"* >/dev/null 2>&1)" ]
then
f_rmdns
f_dnsrestart
fi
exit 255 exit 255
fi fi
fi fi
@@ -331,9 +339,10 @@ f_log()
# #
f_main() f_main()
{ {
local enabled url cnt sum_cnt=0 local enabled url cnt sum_cnt=0 mem_total=0
local src_name src_rset shalla_file shalla_archive list active_lists local src_name src_rset shalla_archive list active_lists
local sysver="$(ubus -S call system board | jsonfilter -e '@.release.description')" local sysver="$(ubus -S call system board | jsonfilter -e '@.release.description')"
mem_total="$(awk '$1 ~ /^MemTotal/ {printf $2}' "/proc/meminfo" 2>/dev/null)"
f_log "info " "start adblock processing ..." f_log "info " "start adblock processing ..."
for src_name in ${adb_sources} for src_name in ${adb_sources}
@@ -356,7 +365,7 @@ f_main()
# download block list # download block list
# #
f_log "debug" "name: ${src_name}, enabled: ${enabled}, backup: ${adb_backup}, dns: ${adb_dns}, fetch: ${adb_fetch}" f_log "debug" "name: ${src_name}, enabled: ${enabled}, backup: ${adb_backup}, dns: ${adb_dns}, fetch: ${adb_fetch}, memory: ${mem_total}"
if [ "${src_name}" = "blacklist" ] if [ "${src_name}" = "blacklist" ]
then then
cat "${url}" 2>/dev/null > "${adb_tmpload}" cat "${url}" 2>/dev/null > "${adb_tmpload}"
@@ -364,23 +373,19 @@ f_main()
elif [ "${src_name}" = "shalla" ] elif [ "${src_name}" = "shalla" ]
then then
shalla_archive="${adb_tmpdir}/shallalist.tar.gz" shalla_archive="${adb_tmpdir}/shallalist.tar.gz"
shalla_file="${adb_tmpdir}/shallalist.txt"
"${adb_fetch}" ${adb_fetchparm} "${shalla_archive}" "${url}" 2>/dev/null "${adb_fetch}" ${adb_fetchparm} "${shalla_archive}" "${url}" 2>/dev/null
adb_rc=${?} adb_rc=${?}
if [ ${adb_rc} -eq 0 ] if [ ${adb_rc} -eq 0 ]
then then
> "${shalla_file}"
for category in ${adb_src_cat_shalla} for category in ${adb_src_cat_shalla}
do do
tar -xOzf "${shalla_archive}" BL/${category}/domains >> "${shalla_file}" tar -xOzf "${shalla_archive}" BL/${category}/domains >> "${adb_tmpload}"
adb_rc=${?} adb_rc=${?}
if [ ${adb_rc} -ne 0 ] if [ ${adb_rc} -ne 0 ]
then then
break break
fi fi
done done
cat "${shalla_file}" 2>/dev/null > "${adb_tmpload}"
rm -f "${shalla_file}"
fi fi
rm -f "${shalla_archive}" rm -f "${shalla_archive}"
rm -rf "${adb_tmpdir}/BL" rm -rf "${adb_tmpdir}/BL"
@@ -389,13 +394,22 @@ f_main()
adb_rc=${?} adb_rc=${?}
fi fi
# check download result and prepare domain output (incl. list backup/restore) # check download result and prepare domain output (incl. tld compression, list backup & restore)
# #
if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpload}" ] if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpload}" ]
then then
awk "${src_rset}" "${adb_tmpload}" > "${adb_tmpfile}" awk "${src_rset}" "${adb_tmpload}" > "${adb_tmpfile}"
if [ -s "${adb_tmpfile}" ] if [ -s "${adb_tmpfile}" ]
then then
if [ ${adb_tldcomp} -eq 1 ]
then
awk -F "." '{for(f=NF;f > 1;f--) printf "%s.", $f;print $1}' "${adb_tmpfile}" | sort -u > "${adb_tmpload}"
awk '{if(NR==1){tld=$NF};while(getline){if($NF !~ tld"\\."){print tld;tld=$NF}}print tld}' "${adb_tmpload}" > "${adb_tmpfile}"
awk -F "." '{for(f=NF;f > 1;f--) printf "%s.", $f;print $1}' "${adb_tmpfile}" > "${adb_tmpload}"
else
sort -u "${adb_tmpfile}" > "${adb_tmpload}"
fi
mv -f "${adb_tmpload}" "${adb_tmpfile}"
f_list backup f_list backup
else else
f_list restore f_list restore
@@ -410,9 +424,9 @@ f_main()
then then
if [ -s "${adb_tmpdir}/tmp.whitelist" ] if [ -s "${adb_tmpdir}/tmp.whitelist" ]
then then
grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpfile}" | sort -u | eval "${adb_dnsformat}" > "${adb_dnsfile}" grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpfile}" | eval "${adb_dnsformat}" > "${adb_dnsfile}"
else else
sort -u "${adb_tmpfile}" | eval "${adb_dnsformat}" > "${adb_dnsfile}" cat "${adb_tmpfile}" | eval "${adb_dnsformat}" > "${adb_dnsfile}"
fi fi
adb_rc=${?} adb_rc=${?}
if [ ${adb_rc} -ne 0 ] if [ ${adb_rc} -ne 0 ]
@@ -424,16 +438,19 @@ f_main()
fi fi
done done
# sort/unique overall # overall sort
# #
for src_name in $(ls -dASr "${adb_tmpdir}/${adb_dnsprefix}"* 2>/dev/null) for src_name in $(ls -dASr "${adb_tmpdir}/${adb_dnsprefix}"* 2>/dev/null)
do do
if [ -s "${adb_tmpdir}/blocklist.overall" ] if [ ${mem_total} -ge 64000 ]
then then
sort "${adb_tmpdir}/blocklist.overall" "${adb_tmpdir}/blocklist.overall" "${src_name}" | uniq -u > "${adb_tmpdir}/tmp.blocklist" if [ -s "${adb_tmpdir}/blocklist.overall" ]
cat "${adb_tmpdir}/tmp.blocklist" > "${src_name}" then
sort "${adb_tmpdir}/blocklist.overall" "${adb_tmpdir}/blocklist.overall" "${src_name}" | uniq -u > "${adb_tmpdir}/tmp.blocklist"
mv -f "${adb_tmpdir}/tmp.blocklist" "${src_name}"
fi
cat "${src_name}" >> "${adb_tmpdir}/blocklist.overall"
fi fi
cat "${src_name}" >> "${adb_tmpdir}/blocklist.overall"
cnt="$(wc -l < "${src_name}")" cnt="$(wc -l < "${src_name}")"
sum_cnt=$((sum_cnt + cnt)) sum_cnt=$((sum_cnt + cnt))
list="${src_name/*./}" list="${src_name/*./}"
@@ -449,6 +466,7 @@ f_main()
# #
mv -f "${adb_tmpdir}/${adb_dnsprefix}"* "${adb_dnsdir}" 2>/dev/null mv -f "${adb_tmpdir}/${adb_dnsprefix}"* "${adb_dnsdir}" 2>/dev/null
chown "${adb_dns}":"${adb_dns}" "${adb_dnsdir}/${adb_dnsprefix}"* 2>/dev/null chown "${adb_dns}":"${adb_dns}" "${adb_dnsdir}/${adb_dnsprefix}"* 2>/dev/null
f_rmtemp
f_dnsrestart f_dnsrestart
if [ "${adb_dnsup}" = "true" ] if [ "${adb_dnsup}" = "true" ]
then then
@@ -461,7 +479,6 @@ f_main()
\"dns_backend\":\"${adb_dns}\", \"dns_backend\":\"${adb_dns}\",
\"last_rundate\":\"$(/bin/date "+%d.%m.%Y %H:%M:%S")\", \"last_rundate\":\"$(/bin/date "+%d.%m.%Y %H:%M:%S")\",
\"system\":\"${sysver}\"}}}}" \"system\":\"${sysver}\"}}}}"
f_rmtemp
return 0 return 0
fi fi
f_log "error" "dns backend restart with active block lists failed (${sysver})" f_log "error" "dns backend restart with active block lists failed (${sysver})"
@@ -469,34 +486,31 @@ f_main()
# handle different adblock actions # handle different adblock actions
# #
if [ "${adb_procd}" = "true" ] f_envload
then case "${1}" in
f_envload stop)
case "${1}" in f_rmtemp
stop) f_rmdns
f_rmtemp f_dnsrestart
f_rmdns ;;
f_dnsrestart restart)
;; f_rmtemp
restart) f_rmdns
f_rmtemp f_envcheck
f_rmdns f_main
f_envcheck ;;
f_main suspend)
;; f_switch suspend
suspend) ;;
f_switch suspend resume)
;; f_switch resume
resume) ;;
f_switch resume query)
;; f_query "${2}"
query) ;;
f_query "${2}" *)
;; f_envcheck
*) f_main
f_envcheck ;;
f_main esac
;;
esac
fi
exit 0 exit 0