GoWiki | openbsd | Ads List

# OpenBSD Ads List

Scripts used to create curated Ads filter lists:

https://dl.giftfish.de/ads

## Prepare

```
mkdir -p /var/www/htdocs/dl/ads
chown nobody /var/www/htdocs/dl/ads
```

## The Script

```
cat <<"EOF">/usr/local/sbin/ads
#!/bin/sh

# source:
# https://firebog.net/

tmp_file="$(mktemp)"
chmod 644 ${tmp_file}
ads_domains="/var/www/htdocs/dl/ads/domains"
ads_hosts="/var/www/htdocs/dl/ads/hosts"
ads_unbound="/var/www/htdocs/dl/ads/unbound_ads_filter.conf"
ads_bind="/var/www/htdocs/dl/ads/bind_ads_filter.conf"

set -A filter \
supportxmr.com \
getmonero.org \
pr0gramm.com \
onesignal.com \
api.onesignal.com \
p2pool.io \
t.ly \
www.bit.ly \
bit.ly \
ow.ly \
tinyurl.com \
s.shopify.com

set -A domains \
https://v.firebog.net/hosts/AdguardDNS.txt \
https://v.firebog.net/hosts/Admiral.txt \
https://v.firebog.net/hosts/Easylist.txt \
https://v.firebog.net/hosts/Easyprivacy.txt \
https://v.firebog.net/hosts/Prigent-Ads.txt \
https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt \
https://osint.digitalside.it/Threat-Intel/lists/latestdomains.txt \
https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt \
https://bitbucket.org/ethanr/dns-blacklists/raw/8575c9f96e5b4a1308f2f12394abd86d0927a4a0/bad_lists/Mandiant_APT1_Report_Appendix_D.txt \
https://phishing.army/download/phishing_army_blocklist_extended.txt \
https://v.firebog.net/hosts/RPiList-Malware.txt \
https://v.firebog.net/hosts/RPiList-Phishing.txt \
https://raw.githubusercontent.com/Spam404/lists/master/main-blacklist.txt \
https://raw.githubusercontent.com/AssoEchap/stalkerware-indicators/master/generated/hosts

# https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-malware.txt

set -A hosts \
https://adaway.org/hosts.txt \
https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt \
https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext \
https://raw.githubusercontent.com/FadeMind/hosts.extras/master/UncheckyAds/hosts \
https://raw.githubusercontent.com/bigdargon/hostsVN/master/hosts \
https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.2o7Net/hosts \
https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt \
https://hostfiles.frogeye.fr/firstparty-trackers-hosts.txt \
https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareHosts.txt \
https://v.firebog.net/hosts/Prigent-Crypto.txt \
https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts \
https://urlhaus.abuse.ch/downloads/hostfile/ \
https://zerodot1.gitlab.io/CoinBlockerLists/hosts_browser

for url in ${domains[@]}; do
  ftp -M -o- ${url} | awk '!/#/&&!/^$/{print}' | sed 's/[[:space:]]//g' | grep -v -E "^[^.]*$|^\.|localdomain|0.0.0.0" | tr -d '\r' | tr -d '"' | tr '[:upper:]' '[:lower:]' >> ${tmp_file}
done

for url in ${hosts[@]}; do
  ftp -M -o- ${url} | awk '!/#/&&!/^$/{print $2}' | sed 's/[[:space:]]//g' | grep -v -E "^[^.]*$|^\.|localdomain|localhost|loopback|0.0.0.0" | tr -d '\r' | tr -d '"' | tr '[:upper:]' '[:lower:]' >> ${tmp_file}
done

sort -u -o ${tmp_file} ${tmp_file}

for url in ${filter[@]}; do
  sed -i "/${url}/d" ${tmp_file}
done
sed -i '/.*\.\{0,1\}.\{63,\}\..*/d' ${tmp_file}

cat <<EOF>${ads_hosts}
127.0.0.1 localhost
127.0.0.1 localhost.localdomain
127.0.0.1 local
255.255.255.255 broadcasthost
::1 localhost
::1 ip6-localhost
::1 ip6-loopback
fe80::1%lo0 localhost
ff00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
ff02::3 ip6-allhosts
0.0.0.0 0.0.0.0
EOF
perl -ne 'while(/^(.*)$/g){print "0.0.0.0 $1\n";}' ${tmp_file} >>${ads_hosts}

echo 'local-zone: "use-application-dns.net" static' >${ads_unbound}
perl -ne 'while(/^(.*)$/g){print "local-zone: \"$1\" static\n";}' ${tmp_file} >>${ads_unbound}

echo 'zone "use-application-dns.net" { type master; notify no; file "null.zone.file"; };' >${ads_bind}
perl -ne 'while(/^(.*)$/g){print "zone \"$1\" { type master; notify no; file \"null.zone.file\"; };\n";}' ${tmp_file} >>${ads_bind}

gzip -c ${ads_hosts} >${ads_hosts}.gz
gzip -c ${ads_unbound} >${ads_unbound}.gz
gzip -c ${ads_bind} >${ads_bind}.gz
cp -f ${tmp_file} ${ads_domains}
gzip -c ${ads_domains} >${ads_domains}.gz
rm -f ${tmp_file}
EOF

chmod 755 /usr/local/sbin/ads

echo "0 */6 * * * /usr/local/sbin/ads >/dev/null 2>&1" >> /var/cron/tabs/nobody
```

## Unbound Updater Script

```
cat <<'EOF'> /usr/local/sbin/update-ads
#!/bin/sh

ads_conf="/var/unbound/etc/unbound_ads_filter.conf"
ads_conf_gz="${ads_conf}.gz"
set -A filter \
getmonero.org \
t.co

ftp -M -o ${ads_conf_gz} https://dl.giftfish.de/ads/unbound_ads_filter.conf.gz

gunzip -f ${ads_conf_gz}

for url in ${filter[@]}; do
  sed -i "/\"${url}\"/d" ${ads_conf}
done

unbound-control reload

rm -f ${ads_conf_gz}
EOF
chmod 755 /usr/local/sbin/update-ads
echo "30 1 * * * /usr/local/sbin/update-ads >/dev/null 2>&1" >> /var/cron/tabs/root
```

Run the updater script once manually: `/usr/local/sbin/update-ads`