public_suffix_list.sh (1034B)
1 #!/bin/sh 2 3 URL="https://publicsuffix.org/list/public_suffix_list.dat" 4 TMPFILE=$(dirname $0)/public_suffix_list.tmp 5 DATFILE=$(dirname $0)/public_suffix_list.dat 6 7 wget -O $TMPFILE $URL || exit 1 8 9 # there might be backslashes (at line end they produce problems) 10 sed -i 's/\\//g' $TMPFILE 11 12 # clear DATFILE if exist 13 printf %s "" > $DATFILE 14 L=0; M=0 15 export CHARSET=UTF-8 # needed for idn 16 cat ${TMPFILE} | while read LINE; do 17 L=$(( L + 1 )) 18 printf "\\r\\t%s\\t%s" "in: $L " "out: $(( $L + $M )) " 19 printf %s\\n "$LINE" | grep -E "^\/\/" >/dev/null 2>&1 && { 20 # do not modify lines beginning with "//" 21 printf %s\\n "$LINE" >> $DATFILE 22 continue 23 } 24 printf %s\\n "$LINE" | grep -E "^$" >/dev/null 2>&1 && { 25 # do not modify empty lines 26 printf %s\\n "$LINE" >> $DATFILE 27 continue 28 } 29 ASCII=$(idn -a "$LINE") # write ASCII and UTF-8 30 if [ "$ASCII" != "$LINE" ]; then 31 printf %s\\n "$ASCII" >> $DATFILE 32 printf "\\t%s\\n" "add: $ASCII" 33 M=$(( M + 1 )) 34 fi 35 printf %s\\n "$LINE" >> $DATFILE 36 done 37 rm -f $TMPFILE 38 gzip -f9 $DATFILE 39