arno/pi-hole
1
0
Fork 0
mirror of https://github.com/pi-hole/pi-hole synced 2025-08-04 05:18:20 +00:00
Code Issues Releases Wiki Activity

revamp of code using a for loop

Browse Source 
-source URLs are in an array that can be looped through
-echo commands grab domain name so there is no need to maintain them
-case statement allows additional formatting of the curl command if
necessary
-all folders are now in variable names
-reverse domain sorting from pizzafritta
This commit is contained in:
jacobsalmela 2015-04-13 12:37:57 -05:00
parent a8874fb606
commit a2ad93b445
1 changed files with 94 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

138
gravity-adv.sh
View File

@ -1,57 +1,107 @@
#!/bin/bash #!/bin/bash
# The Pi-hole now blocks over 120,000 ad domains
# Address to send ads to (the RPi) # Address to send ads to (the RPi)
piholeIP="127.0.0.1" piholeIP="127.0.0.1"
# Optionally, uncomment to automatically detect the address. Thanks Gregg # Optionally, uncomment to automatically detect the local IP address.
#piholeIP=$(ifconfig eth0 | awk '/inet addr/{print substr($2,6)}') #piholeIP=$(hostname -I)
# Config file to hold URL rules # Ad-list sources--one per line in single quotes
eventHorizion="/etc/dnsmasq.d/adList.conf" sources=('http://pgl.yoyo.org/adservers/serverlist.php?='
whitelist=/etc/pihole/whitelist.txt 'http://winhelp2002.mvps.org/hosts.txt'
'https://adaway.org/hosts.txt'
'http://hosts-file.net/.%5Cad_servers.txt'
'http://www.malwaredomainlist.com/hostslist/hosts.txt'
'http://someonewhocares.org/hosts/hosts'
'http://adblock.gjtech.net/?format=unix-hosts'
'http://adblock.mahakala.is/')
# Create the pihole resource directory if it doesn't exist. Future files will be stored here # Variables for various stages of downloading and formatting the list
if [[ -d /etc/pihole/ ]];then origin=/tmp
piholeDir=/etc/pihole
justDomainsExtension=domains
matter=pihole.0.matter.txt
andLight=pihole.1.andLight.txt
eventHorizion=pihole.2.eventHorizon.txt
accretionDisc=/etc/dnsmasq.d/adList.conf
blacklist=$piholeDir/blacklist.txt
whitelist=$piholeDir/whitelist.txt
# Create the Pi-Hole directory if it doesn't exist
if [[ -d $piholeDir ]];then
: :
else else
echo "Forming pihole directory..." echo "** Forming Pi-hole directory..."
sudo mkdir /etc/pihole sudo mkdir $piholeDir
fi fi
echo "Getting yoyo ad list..." # Approximately 2452 domains at the time of writing # Loop through domain list. Download each one and remove commented lines (lines beginning with '# 'or '/') and blank lines
curl -s -d mimetype=plaintext -d hostformat=unixhosts http://pgl.yoyo.org/adservers/serverlist.php? | sort > /tmp/matter.txt for ((i = 0; i < "${#sources[@]}"; i++))
echo "Getting winhelp2002 ad list..." # 12985 domains do
curl -s http://winhelp2002.mvps.org/hosts.txt | grep -v "#" | grep -v "127.0.0.1" | sed '/^$/d' | sed 's/\ /\\ /g' | awk '{print $2}' | sort >> /tmp/matter.txt # Get just the domain from the URL
echo "Getting adaway ad list..." # 445 domains domain=$(echo "${sources[$i]}" | cut -d'/' -f3)
curl -s https://adaway.org/hosts.txt | grep -v "#" | grep -v "::1" | sed '/^$/d' | sed 's/\ /\\ /g' | awk '{print $2}' | grep -v '^\\' | grep -v '\\$' | sort >> /tmp/matter.txt
echo "Getting hosts-file ad list..." # 28050 domains # Save the file as list.#.domain
curl -s http://hosts-file.net/.%5Cad_servers.txt | grep -v "#" | grep -v "::1" | sed '/^$/d' | sed 's/\ /\\ /g' | awk '{print $2}' | grep -v '^\\' | grep -v '\\$' | sort >> /tmp/matter.txt saveLocation=$origin/"list"."$i"."$domain"
echo "Getting malwaredomainlist ad list..." # 1352 domains
curl -s http://www.malwaredomainlist.com/hostslist/hosts.txt | grep -v "#" | sed '/^$/d' | sed 's/\ /\\ /g' | awk '{print $3}' | grep -v '^\\' | grep -v '\\$' | sort >> /tmp/matter.txt # Use a case statement for the domains that need extra options with the curl command. If it doesn't need anything special, just download and format it.
echo "Getting adblock.gjtech ad list..." # 696 domains case "$domain" in
curl -s http://adblock.gjtech.net/?format=unix-hosts | grep -v "#" | sed '/^$/d' | sed 's/\ /\\ /g' | awk '{print $2}' | grep -v '^\\' | grep -v '\\$' | sort >> /tmp/matter.txt
echo "Getting someone who cares ad list..." # 10600 "pgl.yoyo.org")
curl -s http://someonewhocares.org/hosts/hosts | grep -v "#" | sed '/^$/d' | sed 's/\ /\\ /g' | grep -v '^\\' | grep -v '\\$' | awk '{print $2}' | grep -v '^\\' | grep -v '\\$' | sort >> /tmp/matter.txt echo "Getting $domain list...";
echo "Getting Mother of All Ad Blocks list..." # 102168 domains!! Thanks Kacy curl -s -o "$saveLocation" -d mimetype=plaintext -d hostformat=unixhosts "${sources[$i]}";
curl -A 'Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' -e http://forum.xda-developers.com/ http://adblock.mahakala.is/ | grep -v "#" | awk '{print $2}' | sort >> /tmp/matter.txt cat "$saveLocation" > $saveLocation.$justDomainsExtension;;
"adblock.mahakala.is")
echo "Getting $domain list...";
curl -o "$saveLocation" -A 'Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' -e http://forum.xda-developers.com/ "${sources[$i]}";
cat "$saveLocation" | awk '{if ($1 !~ "#" && $1 !~ "/" && $2 !~ "#" && $2 !~ "/" && $0 != "^$" && $2 != "") { print $2}}' > $saveLocation."$justDomainsExtension";;
*) # Runs if the domain doesn't need a specialized curl command
echo "Getting $domain list...";
curl -s -o "$saveLocation" "${sources[$i]}";
# Remove comments and blank lines. Print on the domain (the $2nd field)
cat "$saveLocation" | awk '{if ($1 !~ "#" && $1 !~ "/" && $2 !~ "#" && $2 !~ "/" && $0 != "^$" && $2 != "") { print $2}}' > $saveLocation."$justDomainsExtension";;
esac
done
# Sort the aggregated results and remove any duplicates # Find all files with the .domains extension and compile them into one file
# Remove entries from the whitelist file if it exists at the root of the current user's home folder echo "Aggregating list of domains..."
if [[ -f $whitelist ]];then find $origin/ -type f -name "*.$justDomainsExtension" -exec cat {} \; > $origin/$matter
echo "Removing duplicates, whitelisting, and formatting the list of domains..."
cat /tmp/matter.txt | sed $'s/\r$//' | sort | uniq | sed '/^$/d' | grep -v -x -f $whitelist | awk -v "IP=$piholeIP" '{sub(/\r$/,""); print "address=/"$0"/"IP}' > /tmp/andLight.txt # Append entries from the blacklist file if it exists
numberOfSitesWhitelisted=$(cat $whitelist | wc -l | sed 's/^[ \t]*//') if [[ -f $blacklist ]];then
echo "$numberOfSitesWhitelisted domains whitelisted." numberOf=$(cat $blacklist | wc -l | sed 's/^[ \t]*//')
echo "** Appending $numberOf blacklist entries..."
cat $blacklist >> $origin/$matter
else else
echo "Removing duplicates and formatting the list of domains..." :
cat /tmp/matter.txt | sed $'s/\r$//' | sort | uniq | sed '/^$/d' | awk -v "IP=$piholeIP" '{sub(/\r$/,""); print "address=/"$0"/"IP}' > /tmp/andLight.txt
fi fi
# Count how many domains/whitelists were added so it can be displayed to the user function gravity_advanced()
numberOfAdsBlocked=$(cat /tmp/andLight.txt | wc -l | sed 's/^[ \t]*//') ###########################
echo "$numberOfAdsBlocked ad domains blocked." {
# Sort domains by TLD and remove duplicates
# Turn the file into a dnsmasq config file numberOf=$(cat $origin/$andLight | wc -l | sed 's/^[ \t]*//')
sudo mv /tmp/andLight.txt $eventHorizion echo "$numberOf domains being pulled in by gravity..."
cat $origin/$andLight | awk -F. '{for (i=NF; i>1; --i) printf "%s.",$i;print $1}' | sort -t'.' -k1,2 | awk -F. '{for (i=NF; i>1; --i) printf "%s.",$i;print $1}' | uniq > $origin/$eventHorizion
# Restart DNS numberOf=$(cat $origin/$eventHorizion | wc -l | sed 's/^[ \t]*//')
sudo service dnsmasq restart echo "$numberOf unique domains trapped in the event horizon."
# Format domain list as address=/example.com/127.0.0.1
echo "** Formatting domains into a dnsmasq file..."
cat $origin/$eventHorizion | awk -v "IP=$piholeIP" '{sub(/\r$/,""); print "address=/"$0"/"IP}' > $accretionDisc
sudo service dnsmasq restart
}
# Whitelist (if applicable) then remove duplicates and format for dnsmasq
if [[ -f $whitelist ]];then
# Remove whitelist entries
numberOf=$(cat $whitelist | wc -l | sed 's/^[ \t]*//')
echo "** Whitelisting $numberOf domain(s)..."
cat $origin/$matter | grep -vwf $whitelist > $origin/$andLight
gravity_advanced
else
cat $origin/$matter > $origin/$andLight
gravity_advanced
fi