adblock.sh (15928B)
1 #!/bin/sh 2 # dns based ad/abuse domain blocking 3 # written by Dirk Brenken (dev@brenken.org) 4 5 # This is free software, licensed under the GNU General Public License v3. 6 # You should have received a copy of the GNU General Public License 7 # along with this program. If not, see <http://www.gnu.org/licenses/>. 8 9 # set initial defaults 10 # 11 LC_ALL=C 12 PATH="/usr/sbin:/usr/bin:/sbin:/bin" 13 adb_ver="2.5.0" 14 adb_sysver="$(ubus -S call system board | jsonfilter -e '@.release.description')" 15 adb_enabled=1 16 adb_debug=0 17 adb_backup=0 18 adb_backupdir="/mnt" 19 adb_whitelist="/etc/adblock/adblock.whitelist" 20 adb_whitelist_rset="\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\"^\"\$1\"\\\|[.]\"\$1)}" 21 adb_fetch="/usr/bin/wget" 22 adb_fetchparm="--no-config --quiet --no-cache --no-cookies --max-redirect=0 --timeout=10 --no-check-certificate -O" 23 adb_dnslist="dnsmasq unbound" 24 adb_dnsprefix="adb_list" 25 26 # f_envload: load adblock environment 27 # 28 f_envload() 29 { 30 local dns_up cnt=0 31 32 # source in system library 33 # 34 if [ -r "/lib/functions.sh" ] 35 then 36 . "/lib/functions.sh" 37 else 38 f_log "error" "system library not found" 39 fi 40 41 # set dns backend environment 42 # 43 while [ ${cnt} -le 20 ] 44 do 45 for dns in ${adb_dnslist} 46 do 47 dns_up="$(ubus -S call service list "{\"name\":\"${dns}\"}" | jsonfilter -l1 -e "@.${dns}.instances.*.running")" 48 if [ "${dns_up}" = "true" ] 49 then 50 case "${dns}" in 51 dnsmasq) 52 adb_dns="dnsmasq" 53 adb_dnsdir="/tmp/dnsmasq.d" 54 adb_dnshidedir="${adb_dnsdir}/.adb_hidden" 55 adb_dnsformat="awk '{print \"local=/\"\$0\"/\"}'" 56 break 2 57 ;; 58 unbound) 59 adb_dns="unbound" 60 adb_dnsdir="/var/lib/unbound" 61 adb_dnshidedir="${adb_dnsdir}/.adb_hidden" 62 adb_dnsformat="awk '{print \"local-zone: \042\"\$0\"\042 static\"}'" 63 break 2 64 ;; 65 esac 66 fi 67 done 68 sleep 1 69 cnt=$((cnt+1)) 70 done 71 if [ -z "${adb_dns}" ] 72 then 73 f_log "error" "no active/supported DNS backend found" 74 fi 75 76 # parse global section by callback 77 # 78 config_cb() 79 { 80 local type="${1}" 81 if [ "${type}" = "adblock" ] 82 then 83 option_cb() 84 { 85 local option="${1}" 86 local value="${2}" 87 eval "${option}=\"${value}\"" 88 } 89 else 90 reset_cb 91 fi 92 } 93 94 # parse 'source' section 95 # 96 parse_config() 97 { 98 local value opt section="${1}" options="enabled adb_src adb_src_rset adb_src_cat" 99 eval "adb_sources=\"${adb_sources} ${section}\"" 100 for opt in ${options} 101 do 102 config_get value "${section}" "${opt}" 103 if [ -n "${value}" ] 104 then 105 eval "${opt}_${section}=\"${value}\"" 106 fi 107 done 108 } 109 110 # load adblock config 111 # 112 config_load adblock 113 config_foreach parse_config source 114 } 115 116 # f_envcheck: check/set environment prerequisites 117 # 118 f_envcheck() 119 { 120 # check 'enabled' option 121 # 122 if [ ${adb_enabled} -ne 1 ] 123 then 124 if [ "$(ls -dA "${adb_dnsdir}/${adb_dnsprefix}"* >/dev/null 2>&1)" ] 125 then 126 f_rmdns 127 f_dnsrestart 128 fi 129 f_log "info " "adblock is currently disabled, please set adb_enabled to '1' to use this service" 130 exit 0 131 fi 132 133 # check fetch utility 134 # 135 if [ ! -x "${adb_fetch}" ] && [ "$(readlink -fn "/bin/wget")" = "/bin/uclient-fetch" ] 136 then 137 adb_fetch="/bin/uclient-fetch" 138 adb_fetchparm="-q --timeout=10 --no-check-certificate -O" 139 fi 140 if [ -z "${adb_fetch}" ] || [ -z "${adb_fetchparm}" ] || [ ! -x "${adb_fetch}" ] || [ "$(readlink -fn "${adb_fetch}")" = "/bin/busybox" ] 141 then 142 f_log "error" "required download utility with ssl support not found, e.g. install full 'wget' package" 143 fi 144 145 # create dns hideout directory 146 # 147 if [ ! -d "${adb_dnshidedir}" ] 148 then 149 mkdir -p -m 660 "${adb_dnshidedir}" 150 chown -R "${adb_dns}":"${adb_dns}" "${adb_dnshidedir}" 2>/dev/null 151 else 152 rm -f "${adb_dnshidedir}/${adb_dnsprefix}"* 153 fi 154 155 # create adblock temp file/directory 156 # 157 adb_tmpload="$(mktemp -tu)" 158 adb_tmpfile="$(mktemp -tu)" 159 adb_tmpdir="$(mktemp -p /tmp -d)" 160 161 # prepare whitelist entries 162 # 163 if [ -s "${adb_whitelist}" ] 164 then 165 awk "${adb_whitelist_rset}" "${adb_whitelist}" > "${adb_tmpdir}/tmp.whitelist" 166 fi 167 } 168 169 # f_rmtemp: remove temporary files & directories 170 # 171 f_rmtemp() 172 { 173 if [ -d "${adb_tmpdir}" ] 174 then 175 rm -f "${adb_tmpload}" 176 rm -f "${adb_tmpfile}" 177 rm -rf "${adb_tmpdir}" 178 fi 179 } 180 181 # f_rmdns: remove dns related files & directories 182 # 183 f_rmdns() 184 { 185 if [ -n "${adb_dns}" ] 186 then 187 rm -f "${adb_dnsdir}/${adb_dnsprefix}"* 188 rm -f "${adb_backupdir}/${adb_dnsprefix}"*.gz 189 rm -rf "${adb_dnshidedir}" 190 fi 191 } 192 193 # f_dnsrestart: restart the dns backend 194 # 195 f_dnsrestart() 196 { 197 local cnt=0 198 199 "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 200 while [ ${cnt} -le 10 ] 201 do 202 adb_dnsup="$(ubus -S call service list "{\"name\":\"${adb_dns}\"}" | jsonfilter -l1 -e "@.${adb_dns}.instances.*.running")" 203 if [ "${adb_dnsup}" = "true" ] 204 then 205 break 206 fi 207 cnt=$((cnt+1)) 208 sleep 1 209 done 210 } 211 212 # f_list: backup/restore/remove block lists 213 # 214 f_list() 215 { 216 local mode="${1}" in_rc="${adb_rc}" cnt=0 217 218 case "${mode}" in 219 backup) 220 cnt="$(wc -l < "${adb_tmpfile}")" 221 if [ ${adb_backup} -eq 1 ] && [ -d "${adb_backupdir}" ] 222 then 223 gzip -cf "${adb_tmpfile}" > "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" 224 adb_rc=${?} 225 fi 226 ;; 227 restore) 228 if [ ${adb_backup} -eq 1 ] && [ -d "${adb_backupdir}" ] 229 then 230 rm -f "${adb_dnsdir}/${adb_dnsprefix}.${src_name}" 231 if [ -f "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" ] 232 then 233 gunzip -cf "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" > "${adb_tmpfile}" 234 adb_rc=${?} 235 fi 236 fi 237 ;; 238 remove) 239 rm -f "${adb_dnsdir}/${adb_dnsprefix}.${src_name}" 240 if [ -d "${adb_backupdir}" ] 241 then 242 rm -f "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" 243 fi 244 adb_rc=${?} 245 ;; 246 esac 247 f_log "debug" "name: ${src_name}, mode: ${mode}, count: ${cnt}, in_rc: ${in_rc}, out_rc: ${adb_rc}" 248 } 249 250 # f_switch: suspend/resume adblock processing 251 # 252 f_switch() 253 { 254 if [ -d "${adb_dnshidedir}" ] 255 then 256 local source target status mode="${1}" 257 local dns_active="$(find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}*" -print)" 258 local dns_passive="$(find "${adb_dnshidedir}" -maxdepth 1 -type f -name "${adb_dnsprefix}*" -print)" 259 260 if [ -n "${dns_active}" ] && [ "${mode}" = "suspend" ] 261 then 262 source="${adb_dnsdir}/${adb_dnsprefix}" 263 target="${adb_dnshidedir}" 264 status="suspended" 265 elif [ -n "${dns_passive}" ] && [ "${mode}" = "resume" ] 266 then 267 source="${adb_dnshidedir}/${adb_dnsprefix}" 268 target="${adb_dnsdir}" 269 status="resumed" 270 fi 271 if [ -n "${status}" ] 272 then 273 mv -f "${source}"* "${target}" 274 f_dnsrestart 275 f_log "info " "adblock processing ${status}" 276 fi 277 fi 278 } 279 280 # f_query: query block lists for certain (sub-)domains 281 # 282 f_query() 283 { 284 local search result cnt 285 local domain="${1}" 286 local tld="${domain#*.}" 287 local dns_active="$(find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}*" -print)" 288 289 if [ -z "${dns_active}" ] 290 then 291 printf "%s\n" "::: no active block lists found, please start adblock first" 292 elif [ -z "${domain}" ] || [ "${domain}" = "${tld}" ] 293 then 294 printf "%s\n" "::: invalid domain input, please submit a specific (sub-)domain, e.g. 'www.abc.xyz'" 295 else 296 cd "${adb_dnsdir}" 297 while [ "${domain}" != "${tld}" ] 298 do 299 search="${domain//./\.}" 300 result="$(grep -Hm1 "[/\"\.]${search}[/\"]" "${adb_dnsprefix}"* | awk -F ':|=|/|\"' '{printf(" %-20s : %s\n",$1,$4)}')" 301 printf "%s\n" "::: distinct results for domain '${domain}'" 302 if [ -z "${result}" ] 303 then 304 printf "%s\n" " no match" 305 else 306 printf "%s\n" "${result}" 307 fi 308 domain="${tld}" 309 tld="${domain#*.}" 310 done 311 fi 312 } 313 314 # f_log: write to syslog, exit on error 315 # 316 f_log() 317 { 318 local class="${1}" log_msg="${2}" 319 320 if [ -n "${log_msg}" ] && ([ "${class}" != "debug" ] || [ ${adb_debug} -eq 1 ]) 321 then 322 logger -t "adblock-[${adb_ver}] ${class}" "${log_msg}" 323 if [ "${class}" = "error" ] 324 then 325 logger -t "adblock-[${adb_ver}] ${class}" "Please check 'https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md' (${adb_sysver})" 326 f_rmtemp 327 if [ "$(ls -dA "${adb_dnsdir}/${adb_dnsprefix}"* >/dev/null 2>&1)" ] 328 then 329 f_rmdns 330 f_dnsrestart 331 fi 332 exit 255 333 fi 334 fi 335 } 336 337 # main function for block list processing 338 # 339 f_main() 340 { 341 local enabled url cnt sum_cnt=0 mem_total=0 342 local src_name src_rset shalla_archive list active_lists active_triggers 343 mem_total="$(awk '$1 ~ /^MemTotal/ {printf $2}' "/proc/meminfo" 2>/dev/null)" 344 345 f_log "info " "start adblock processing ..." 346 for src_name in ${adb_sources} 347 do 348 eval "enabled=\"\${enabled_${src_name}}\"" 349 eval "url=\"\${adb_src_${src_name}}\"" 350 eval "src_rset=\"\${adb_src_rset_${src_name}}\"" 351 adb_dnsfile="${adb_tmpdir}/${adb_dnsprefix}.${src_name}" 352 > "${adb_tmpload}" 353 > "${adb_tmpfile}" 354 adb_rc=0 355 356 # basic pre-checks 357 # 358 if [ "${enabled}" != "1" ] || [ -z "${url}" ] || [ -z "${src_rset}" ] 359 then 360 f_list remove 361 continue 362 fi 363 364 # download block list 365 # 366 f_log "debug" "name: ${src_name}, enabled: ${enabled}, backup: ${adb_backup}, dns: ${adb_dns}, fetch: ${adb_fetch}, memory: ${mem_total}" 367 if [ "${src_name}" = "blacklist" ] 368 then 369 cat "${url}" 2>/dev/null > "${adb_tmpload}" 370 adb_rc=${?} 371 elif [ "${src_name}" = "shalla" ] 372 then 373 shalla_archive="${adb_tmpdir}/shallalist.tar.gz" 374 "${adb_fetch}" ${adb_fetchparm} "${shalla_archive}" "${url}" 2>/dev/null 375 adb_rc=${?} 376 if [ ${adb_rc} -eq 0 ] 377 then 378 for category in ${adb_src_cat_shalla} 379 do 380 tar -xOzf "${shalla_archive}" BL/${category}/domains >> "${adb_tmpload}" 381 adb_rc=${?} 382 if [ ${adb_rc} -ne 0 ] 383 then 384 break 385 fi 386 done 387 fi 388 rm -f "${shalla_archive}" 389 rm -rf "${adb_tmpdir}/BL" 390 else 391 "${adb_fetch}" ${adb_fetchparm} "${adb_tmpload}" "${url}" 2>/dev/null 392 adb_rc=${?} 393 fi 394 395 # check download result and prepare domain output (incl. tld compression, list backup & restore) 396 # 397 if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpload}" ] 398 then 399 awk "${src_rset}" "${adb_tmpload}" 2>/dev/null > "${adb_tmpfile}" 400 if [ -s "${adb_tmpfile}" ] 401 then 402 awk -F "." '{for(f=NF;f > 1;f--) printf "%s.", $f;print $1}' "${adb_tmpfile}" 2>/dev/null | sort -u > "${adb_tmpload}" 403 awk '{if(NR==1){tld=$NF};while(getline){if($NF !~ tld"\\."){print tld;tld=$NF}}print tld}' "${adb_tmpload}" 2>/dev/null > "${adb_tmpfile}" 404 awk -F "." '{for(f=NF;f > 1;f--) printf "%s.", $f;print $1}' "${adb_tmpfile}" 2>/dev/null > "${adb_tmpload}" 405 mv -f "${adb_tmpload}" "${adb_tmpfile}" 406 f_list backup 407 else 408 f_list restore 409 fi 410 else 411 f_list restore 412 fi 413 414 # remove whitelist domains, final list preparation 415 # 416 if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpfile}" ] 417 then 418 if [ -s "${adb_tmpdir}/tmp.whitelist" ] 419 then 420 grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpfile}" 2>/dev/null | eval "${adb_dnsformat}" > "${adb_dnsfile}" 421 else 422 cat "${adb_tmpfile}" 2>/dev/null | eval "${adb_dnsformat}" > "${adb_dnsfile}" 423 fi 424 adb_rc=${?} 425 if [ ${adb_rc} -ne 0 ] 426 then 427 f_list remove 428 fi 429 else 430 f_list remove 431 fi 432 done 433 434 # overall sort 435 # 436 for src_name in $(ls -dASr "${adb_tmpdir}/${adb_dnsprefix}"* 2>/dev/null) 437 do 438 if [ ${mem_total} -ge 64000 ] 439 then 440 if [ -s "${adb_tmpdir}/blocklist.overall" ] 441 then 442 sort "${adb_tmpdir}/blocklist.overall" "${adb_tmpdir}/blocklist.overall" "${src_name}" | uniq -u > "${adb_tmpdir}/tmp.blocklist" 443 mv -f "${adb_tmpdir}/tmp.blocklist" "${src_name}" 444 fi 445 cat "${src_name}" >> "${adb_tmpdir}/blocklist.overall" 446 fi 447 cnt="$(wc -l < "${src_name}")" 448 sum_cnt=$((sum_cnt + cnt)) 449 list="${src_name/*./}" 450 if [ -z "${active_lists}" ] 451 then 452 active_lists="\"${list}\":\"${cnt}\"" 453 else 454 active_lists="${active_lists},\"${list}\":\"${cnt}\"" 455 fi 456 done 457 458 # restart the dns backend and write statistics to procd service instance 459 # 460 mv -f "${adb_tmpdir}/${adb_dnsprefix}"* "${adb_dnsdir}" 2>/dev/null 461 chown "${adb_dns}":"${adb_dns}" "${adb_dnsdir}/${adb_dnsprefix}"* 2>/dev/null 462 f_rmtemp 463 f_dnsrestart 464 if [ "${adb_dnsup}" = "true" ] 465 then 466 f_log "info " "block lists with overall ${sum_cnt} domains loaded successfully (${adb_sysver})" 467 for name in ${adb_iface} 468 do 469 active_triggers="${active_triggers}[\"interface.*.up\",[\"if\",[\"eq\",\"interface\",\"${name}\"],[\"run_script\",\"/etc/init.d/adblock\",\"start\"],1000]]," 470 done 471 active_triggers="${active_triggers}[\"config.change\",[\"if\",[\"eq\",\"package\",\"adblock\"],[\"run_script\",\"/etc/init.d/adblock\",\"start\"],1000]]" 472 ubus call service set "{\"name\":\"adblock\", 473 \"instances\":{\"adblock\":{\"command\":[\"/usr/bin/adblock.sh\"], 474 \"data\":{\"active_lists\":[{${active_lists}}], 475 \"adblock_version\":\"${adb_ver}\", 476 \"blocked_domains\":\"${sum_cnt}\", 477 \"dns_backend\":\"${adb_dns}\", 478 \"last_rundate\":\"$(/bin/date "+%d.%m.%Y %H:%M:%S")\", 479 \"system\":\"${adb_sysver}\"}}}, 480 \"triggers\":[${active_triggers}]}" 481 else 482 f_log "error" "dns backend restart with active block lists failed" 483 fi 484 } 485 486 # handle different adblock actions 487 # 488 f_envload 489 case "${1}" in 490 stop) 491 f_rmtemp 492 f_rmdns 493 f_dnsrestart 494 ;; 495 restart) 496 f_rmtemp 497 f_rmdns 498 f_envcheck 499 f_main 500 ;; 501 suspend) 502 f_switch suspend 503 ;; 504 resume) 505 f_switch resume 506 ;; 507 query) 508 f_query "${2}" 509 ;; 510 *) 511 f_envcheck 512 f_main 513 ;; 514 esac 515 exit 0