diff --git a/advanced/Scripts/blacklist.sh b/advanced/Scripts/blacklist.sh new file mode 100644 index 00000000..9eac0780 --- /dev/null +++ b/advanced/Scripts/blacklist.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +# (c) 2015 by Jacob Salmela +# This file is part of Pi-hole. +# +# Pi-hole is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. + +if [[ $# = 0 ]]; then + echo "Immediately blacklists one or more domains in the hosts file" + echo "Usage: blacklist.sh domain1 [domain2 ...]" + echo "Options:" + echo " -d, --delmode Remove domains from the blacklist" + echo " -nr, --noreload Update blacklist without refreshing dnsmasq" + echo " -f, --force Force updating of the hosts files, even if there are no changes" + + exit 1 +fi + +#globals +blacklist=/etc/pihole/blacklist.txt +adList=/etc/pihole/gravity.list +reload=true +addmode=true +force=false +versbose=true +domList=() +domToRemoveList=() + + +piholeIPfile=/tmp/piholeIP +piholeIPv6file=/etc/pihole/.useIPv6 + +# Otherwise, the IP address can be taken directly from the machine, which will happen when the script is run by the user and not the installation script +IPv4dev=$(ip route get 8.8.8.8 | awk '{for(i=1;i<=NF;i++)if($i~/dev/)print $(i+1)}') +piholeIPCIDR=$(ip -o -f inet addr show dev $IPv4dev | awk '{print $4}' | awk 'END {print}') +piholeIP=${piholeIPCIDR%/*} + +modifyHost=false + + +if [[ -f $piholeIPv6file ]];then + # If the file exists, then the user previously chose to use IPv6 in the automated installer + piholeIPv6=$(ip -6 route get 2001:4860:4860::8888 | awk -F " " '{ for(i=1;i<=NF;i++) if ($i == "src") print $(i+1) }') +fi + + +function HandleOther(){ + #check validity of domain + validDomain=$(echo $1 | perl -ne'print if /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/') + + if [ -z "$validDomain" ]; then + echo $1 is not a valid argument or domain name + else + domList=("${domList[@]}" $validDomain) + fi +} + +function PopBlacklistFile(){ + #check whitelist file exists, and if not, create it + if [[ ! -f $blacklist ]];then + touch $blacklist + fi + for dom in "${domList[@]}" + do + if $addmode; then + AddDomain $dom + else + RemoveDomain $dom + fi + done +} + +function AddDomain(){ +#| sed 's/\./\\./g' + bool=false + grep -Ex -q "$1" $blacklist || bool=true + if $bool; then + #domain not found in the blacklist file, add it! + if $versbose; then + echo "** Adding $1 to blacklist file" + fi + echo $1 >> $blacklist + modifyHost=true + else + if $versbose; then + echo "** $1 already blacklisted! No need to add" + fi + fi +} + +function RemoveDomain(){ + + bool=false + grep -Ex -q "$1" $blacklist || bool=true + if $bool; then + #Domain is not in the blacklist file, no need to Remove + if $versbose; then + echo "** $1 is NOT blacklisted! No need to remove" + fi + else + #Domain is in the blacklist file, add to a temporary array + if $versbose; then + echo "** Un-blacklisting $dom..." + fi + domToRemoveList=("${domToRemoveList[@]}" $1) + modifyHost=true + fi +} + +function ModifyHostFile(){ + if $addmode; then + #add domains to the hosts file + if [[ -r $blacklist ]];then + numberOf=$(cat $blacklist | sed '/^\s*$/d' | wc -l) + plural=; [[ "$numberOf" != "1" ]] && plural=s + echo "** blacklisting a total of $numberOf domain${plural}..." + if [[ -n $piholeIPv6 ]];then + cat $blacklist | awk -v ipv4addr="$piholeIP" -v ipv6addr="$piholeIPv6" '{sub(/\r$/,""); print ipv4addr" "$0"\n"ipv6addr" "$0}' >> $adList + else + cat $blacklist | awk -v ipv4addr="$piholeIP" '{sub(/\r$/,""); print ipv4addr" "$0}' >>$adList + fi + + fi + else + + for dom in "${domToRemoveList[@]}" + do + #we need to remove the domains from the blacklist file and the host file + echo $dom | sed 's/\./\\./g' | xargs -I {} perl -i -ne'print unless /[^.]'{}'(?!.)/;' $adList + echo $dom | sed 's/\./\\./g' | xargs -I {} perl -i -ne'print unless /'{}'(?!.)/;' $blacklist + done + fi + +} + +function Reload() { + # Reload hosts file + echo "** Refresh lists in dnsmasq..." + + dnsmasqPid=$(pidof dnsmasq) + + if [[ $dnsmasqPid ]]; then + # service already running - reload config + sudo kill -HUP $dnsmasqPid + else + # service not running, start it up + sudo service dnsmasq start + fi +} + +################################################### + +for var in "$@" +do + case "$var" in + "-nr"| "--noreload" ) reload=false;; + "-d" | "--delmode" ) addmode=false;; + "-f" | "--force" ) force=true;; + "-q" | "--quiet" ) versbose=false;; + * ) HandleOther $var;; + esac +done + +PopBlacklistFile + +if $modifyHost || $force; then + echo "** Modifying Hosts File" + ModifyHostFile +else + if $versbose; then + echo "** No changes need to be made" + fi + exit 1 +fi + +if $reload; then + Reload +fi diff --git a/advanced/Scripts/whitelist.sh b/advanced/Scripts/whitelist.sh index 7b964cdf..fa699be0 100755 --- a/advanced/Scripts/whitelist.sh +++ b/advanced/Scripts/whitelist.sh @@ -7,83 +7,174 @@ # the Free Software Foundation, either version 2 of the License, or # (at your option) any later version. +if [[ $# = 0 ]]; then + echo "Immediately whitelists one or more domains in the hosts file" + echo " " + echo "Usage: whitelist.sh domain1 [domain2 ...]" + echo " " + echo "Options:" + echo " -d, --delmode Remove domains from the whitelist" + echo " -nr, --noreload Update Whitelist without refreshing dnsmasq" + echo " -f, --force Force updating of the hosts files, even if there are no changes" + + exit 1 +fi + +#globals whitelist=/etc/pihole/whitelist.txt adList=/etc/pihole/gravity.list -webInterfaceEchos=/tmp/whitelistEchoFile +reload=true +addmode=true +force=false +versbose=true +domList=() +domToRemoveList=() -if [[ ! -f $whitelist ]];then - touch $whitelist -fi +piholeIPfile=/tmp/piholeIP +piholeIPv6file=/etc/pihole/.useIPv6 -formatEchoes() -{ -if [[ "$(whoami)" = "www-data" ]];then - echo "$1" >> $webInterfaceEchos -else - echo "$1" -fi -} +# Otherwise, the IP address can be taken directly from the machine, which will happen when the script is run by the user and not the installation script +IPv4dev=$(ip route get 8.8.8.8 | awk '{for(i=1;i<=NF;i++)if($i~/dev/)print $(i+1)}') +piholeIPCIDR=$(ip -o -f inet addr show dev $IPv4dev | awk '{print $4}' | awk 'END {print}') +piholeIP=${piholeIPCIDR%/*} -if [[ $# = 0 ]]; then - # echoes go to a file for showing in the Web interface - echo "Immediately whitelists one or more domains." - echo "Usage: whitelist.sh domain1 [domain2 ...]" - if [[ "$(whoami)" = "www-data" ]];then - formatEchoes "Enter one or more space-separated FQDN." - # If the user is www-data, the script is probably being called from the Web interface - # Since the Web interface only displays the last echo in the script (I'm still a n00b with PHP) - webInterfaceDisplay=$(cat $webInterfaceEchos) - # The last echo needs to be delimited by a semi-colon so I translate newlines into semi-colons so it displays properly - # Someone better in PHP might be able to come up with a better solution, but this is a highly-requested feature - # This is also used later in the script, too - echo "$webInterfaceDisplay" | tr "\n" ";" - fi +modifyHost=false + + +if [[ -f $piholeIPv6file ]];then + # If the file exists, then the user previously chose to use IPv6 in the automated installer + piholeIPv6=$(ip -6 route get 2001:4860:4860::8888 | awk -F " " '{ for(i=1;i<=NF;i++) if ($i == "src") print $(i+1) }') fi -combopattern="" -# Overwrite any previously existing file so the output is always correct -echo "" > $webInterfaceEchos +function HandleOther(){ + #check validity of domain + validDomain=$(echo $1 | perl -ne'print if /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/') + + if [ -z "$validDomain" ]; then + echo $1 is not a valid argument or domain name + else + domList=("${domList[@]}" $validDomain) + fi +} -# For each argument passed to this script -for var in "$@" -do - # Start appending the echoes into the file for display in the Web interface later - formatEchoes "Whitelisting $var..." +function PopWhitelistFile(){ + #check whitelist file exists, and if not, create it + if [[ ! -f $whitelist ]];then + touch $whitelist + fi + for dom in "${domList[@]}" + do + if $addmode; then + AddDomain $dom + else + RemoveDomain $dom + fi + done +} - # Construct basic pattern to match domain name. - basicpattern=$(echo $var | awk -F '[# \t]' 'NF>0&&$1!="" {print ""$1""}' | sed 's/\./\\./g') +function AddDomain(){ +#| sed 's/\./\\./g' + bool=false + grep -Ex -q "$1" $whitelist || bool=true + if $bool; then + #domain not found in the whitelist file, add it! + if $versbose; then + echo "** Adding $1 to whitelist file" + fi + echo $1 >> $whitelist + modifyHost=true + else + if $versbose; then + echo "** $1 already whitelisted! No need to add" + fi + fi +} - if [[ "$basicpattern" != "" ]]; then - # Add to the combination pattern that will be used below - if [[ "$combopattern" != "" ]]; then combopattern="$combopattern|"; fi - combopattern="$combopattern$basicpattern" +function RemoveDomain(){ + + bool=false + grep -Ex -q "$1" $whitelist || bool=true + if $bool; then + #Domain is not in the whitelist file, no need to Remove + if $versbose; then + echo "** $1 is NOT whitelisted! No need to remove" + fi + else + #Domain is in the whitelist file, add to a temporary array and remove from whitelist file + if $versbose; then + echo "** Un-whitelisting $dom..." + fi + domToRemoveList=("${domToRemoveList[@]}" $1) + modifyHost=true + fi +} - # Also add the domain to the whitelist but only if it's not already present - grep -E -q "^$basicpattern$" $whitelist \ - || echo "$var" >> $whitelist - fi -done +function ModifyHostFile(){ + if $addmode; then + #remove domains in from hosts file + if [[ -r $whitelist ]];then + # Remove whitelist entries + numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l) + plural=; [[ "$numberOf" != "1" ]] && plural=s + echo "** Whitelisting a total of $numberOf domain${plural}..." + awk -F':' '{ print $1 }' $whitelist | sed 's/\./\\./g' | xargs -I {} perl -i -ne'print unless /[^.]'{}'(?!.)/;' $adList + fi + else + #we need to add the removed domains to the hosts file + for rdom in "${domToRemoveList[@]}" + do + if [[ -n $piholeIPv6 ]];then + echo "**Blacklisting $rdom on IPv4 and IPv6" + echo $rdom | awk -v ipv4addr="$piholeIP" -v ipv6addr="$piholeIPv6" '{sub(/\r$/,""); print ipv4addr" "$0"\n"ipv6addr" "$0}' >> $adList + else + echo "**Blacklisting $rdom on IPv4" + echo $rdom | awk -v ipv4addr="$piholeIP" '{sub(/\r$/,""); print ipv4addr" "$0}' >>$adList + fi + echo $rdom| sed 's/\./\\./g' | xargs -I {} perl -i -ne'print unless /'{}'(?!.)/;' $whitelist + done + fi +} -# Now report on and remove matched domains -if [[ "$combopattern" != "" ]]; then - formatEchoes "Modifying hosts file..." +function Reload() { + # Reload hosts file + echo "** Refresh lists in dnsmasq..." + dnsmasqPid=$(pidof dnsmasq) - # Construct pattern to match entry in hosts file. - # This consists of one or more IP addresses followed by the domain name. - pattern=$(echo $combopattern | awk -F '[# \t]' '{printf "%s", "^(([0-9]+\.){3}[0-9]+ +)+("$1")$"}') + if [[ $dnsmasqPid ]]; then + # service already running - reload config + sudo kill -HUP $dnsmasqPid + else + # service not running, start it up + sudo service dnsmasq start + fi +} - # Output what will be removed and then actually remove - sed -r -n 's/'"$pattern"'/ Removed: \3/p' $adList - sed -r -i '/'"$pattern"'/d' $adList +################################################### - formatEchoes "** $# domain(s) whitelisted." +for var in "$@" +do + case "$var" in + "-nr"| "--noreload" ) reload=false;; + "-d" | "--delmode" ) addmode=false;; + "-f" | "--force" ) force=true;; + "-q" | "--quiet" ) versbose=false;; + * ) HandleOther $var;; + esac +done + +PopWhitelistFile + +if $modifyHost || $force; then + echo "** Modifying Hosts File" + ModifyHostFile +else + if $versbose; then + echo "** No changes need to be made" + exit 1 + fi +fi - # Only echo the semi-colon delimited echoes if the user running the script is www-data (meaning it is run the from Web interface) - if [[ "$(whoami)" = "www-data" ]];then - webInterfaceDisplay=$(cat $webInterfaceEchos) - echo "$webInterfaceDisplay" | tr "\n" ";" - fi - # Force dnsmasq to reload /etc/pihole/gravity.list - kill -HUP $(pidof dnsmasq) +if $reload; then + Reload fi diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index b80180cb..20570f22 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -207,8 +207,9 @@ installScripts(){ sudo curl -o /usr/local/bin/gravity.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/gravity.sh sudo curl -o /usr/local/bin/chronometer.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/chronometer.sh sudo curl -o /usr/local/bin/whitelist.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/whitelist.sh +sudo curl -o /usr/local/bin/blacklist.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/blacklist.sh sudo curl -o /usr/local/bin/piholeLogFlush.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/piholeLogFlush.sh -sudo chmod 755 /usr/local/bin/{gravity,chronometer,whitelist,piholeLogFlush}.sh +sudo chmod 755 /usr/local/bin/{gravity,chronometer,whitelist,blacklist,piholeLogFlush}.sh } installConfigs(){ diff --git a/gravity.sh b/gravity.sh index fbecb3ad..fa94aabe 100755 --- a/gravity.sh +++ b/gravity.sh @@ -48,12 +48,11 @@ blacklist=$piholeDir/blacklist.txt whitelist=$piholeDir/whitelist.txt latentWhitelist=$piholeDir/latentWhitelist.txt justDomainsExtension=domains -matter=$basename.0.matter.txt -andLight=$basename.1.andLight.txt -supernova=$basename.2.supernova.txt -eventHorizon=$basename.3.eventHorizon.txt -accretionDisc=$basename.4.accretionDisc.txt -eyeOfTheNeedle=$basename.5.wormhole.txt +matterandlight=$basename.0.matterandlight.txt +supernova=$basename.1.supernova.txt +eventHorizon=$basename.2.eventHorizon.txt +accretionDisc=$basename.3.accretionDisc.txt +eyeOfTheNeedle=$basename.4.wormhole.txt # After setting defaults, check if there's local overrides if [[ -r $piholeDir/pihole.conf ]];then @@ -160,48 +159,35 @@ function gravity_Schwarzchild() { # Find all active domains and compile them into one file and remove CRs echo "** Aggregating list of domains..." - truncate -s 0 $piholeDir/$matter + truncate -s 0 $piholeDir/$matterandlight for i in "${activeDomains[@]}" do - cat $i |tr -d '\r' >> $piholeDir/$matter + cat $i |tr -d '\r' >> $piholeDir/$matterandlight done } -# Pulsar - White/blacklist application -function gravity_pulsar() { +function gravity_Blacklist(){ # Append blacklist entries if they exist - if [[ -r $blacklist ]];then - numberOf=$(cat $blacklist | sed '/^\s*$/d' | wc -l) - echo "** Blacklisting $numberOf domain(s)..." - cat $blacklist >> $piholeDir/$matter - fi + blacklist.sh -f -nr -q +} - # Whitelist (if applicable) domains - if [[ -r $whitelist ]];then - # Remove whitelist entries - numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l) - plural=; [[ "$numberOf" != "1" ]] && plural=s - echo "** Whitelisting $numberOf domain${plural}..." - - # Append a "$" to the end, prepend a "^" to the beginning, and - # replace "." with "\." of each line to turn each entry into a - # regexp so it can be parsed out with grep -x - awk -F '[# \t]' 'NF>0&&$1!="" {print "^"$1"$"}' $whitelist | sed 's/\./\\./g' > $latentWhitelist - else - rm $latentWhitelist 2>/dev/null - fi +function gravity_Whitelist() { # Prevent our sources from being pulled into the hole - plural=; [[ "${#sources[@]}" != "1" ]] && plural=s + plural=; [[ "${sources[@]}" != "1" ]] && plural=s echo "** Whitelisting ${#sources[@]} ad list source${plural}..." + + urls=() for url in ${sources[@]} do - echo "$url" | awk -F '/' '{print "^"$3"$"}' | sed 's/\./\\./g' >> $latentWhitelist + tmp=$(echo "$url" | awk -F '/' '{print $3}') + urls=("${urls[@]}" $tmp) done + + whitelist.sh -f -nr -q ${urls[@]} - # Remove whitelist entries from list - grep -vxf $latentWhitelist $piholeDir/$matter > $piholeDir/$andLight + } function gravity_unique() { @@ -240,12 +226,13 @@ function gravity_blackbody() { } function gravity_advanced() { + + # Remove comments and print only the domain name # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious # This helps with that and makes it easier to read # It also helps with debugging so each stage of the script can be researched more in depth - awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $piholeDir/$andLight | \ - sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $piholeDir/$supernova + awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $piholeDir/$matterandlight | sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $piholeDir/$supernova numberOf=$(wc -l < $piholeDir/$supernova) echo "** $numberOf domains being pulled in by gravity..." @@ -256,7 +243,6 @@ function gravity_advanced() { function gravity_reload() { # Reload hosts file echo "** Refresh lists in dnsmasq..." - dnsmasqPid=$(pidof dnsmasq) if [[ $dnsmasqPid ]]; then @@ -268,11 +254,13 @@ function gravity_reload() { fi } + gravity_collapse gravity_spinup gravity_Schwarzchild -gravity_pulsar gravity_advanced gravity_hostFormat gravity_blackbody +gravity_Whitelist +gravity_Blacklist gravity_reload