Merge pull request #180 from jacobsalmela/whitelist

Whitelist/blacklist improvements

advanced/Scripts/blacklist.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# (c) 2015 by Jacob Salmela
+# This file is part of Pi-hole.
+#
+# Pi-hole is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+
+if [[ $# = 0 ]]; then
+    echo "Immediately blacklists one or more domains in the hosts file"
+    echo " "
+    echo "Usage: blacklist.sh domain1 [domain2 ...]"
+    echo "  "
+    echo "Options:"
+    echo "  -d, --delmode		Remove domains from the blacklist"
+    echo "  -nr, --noreload	Update blacklist without refreshing dnsmasq"
+    echo "  -f, --force		Force updating of the hosts files, even if there are no changes"
+    echo "  -q, --quiet		output is less verbose"
+    exit 1
+fi
+
+#globals
+blacklist=/etc/pihole/blacklist.txt
+adList=/etc/pihole/gravity.list
+reload=true
+addmode=true
+force=false
+versbose=true
+domList=()
+domToRemoveList=()
+
+
+piholeIPfile=/tmp/piholeIP
+piholeIPv6file=/etc/pihole/.useIPv6
+
+# Otherwise, the IP address can be taken directly from the machine, which will happen when the script is run by the user and not the installation script
+IPv4dev=$(ip route get 8.8.8.8 | awk '{for(i=1;i<=NF;i++)if($i~/dev/)print $(i+1)}')
+piholeIPCIDR=$(ip -o -f inet addr show dev $IPv4dev | awk '{print $4}' | awk 'END {print}')
+piholeIP=${piholeIPCIDR%/*}
+
+modifyHost=false
+
+
+if [[ -f $piholeIPv6file ]];then
+    # If the file exists, then the user previously chose to use IPv6 in the automated installer
+    piholeIPv6=$(ip -6 route get 2001:4860:4860::8888 | awk -F " " '{ for(i=1;i<=NF;i++) if ($i == "src") print $(i+1) }')
+fi
+
+
+function HandleOther(){	
+  #check validity of domain
+	validDomain=$(echo $1 | perl -ne'print if /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/')
+	
+	if [ -z "$validDomain" ]; then
+		echo $1 is not a valid argument or domain name
+	else	  
+	  domList=("${domList[@]}" $validDomain)
+	fi
+}
+
+function PopBlacklistFile(){
+	#check blacklist file exists, and if not, create it
+	if [[ ! -f $blacklist ]];then
+  	  touch $blacklist
+	fi	
+	for dom in "${domList[@]}"
+	do	  
+	  if $addmode; then
+	  	AddDomain $dom
+	  else
+	    RemoveDomain $dom
+	  fi
+	done
+}
+
+function AddDomain(){
+#| sed 's/\./\\./g'
+	bool=false
+	grep -Ex -q "$1" $blacklist || bool=true
+	if $bool; then
+	  #domain not found in the blacklist file, add it!
+	  if $versbose; then
+	  echo "** Adding $1 to blacklist file"
+	  fi
+		echo $1 >> $blacklist
+		modifyHost=true
+	else
+	if $versbose; then
+		echo "** $1 already blacklisted! No need to add"
+		fi
+	fi
+}
+
+function RemoveDomain(){
+  
+  bool=false
+  grep -Ex -q "$1" $blacklist || bool=true
+  if $bool; then
+  	#Domain is not in the blacklist file, no need to Remove
+  	if $versbose; then
+  	echo "** $1 is NOT blacklisted! No need to remove"
+  	fi
+  else
+    #Domain is in the blacklist file, add to a temporary array
+    if $versbose; then
+    echo "** Un-blacklisting $dom..."
+    fi
+    domToRemoveList=("${domToRemoveList[@]}" $1)    
+    modifyHost=true	  	
+  fi  	
+}
+
+function ModifyHostFile(){	
+	 if $addmode; then
+	    #add domains to the hosts file
+	    if [[ -r $blacklist ]];then
+	      numberOf=$(cat $blacklist | sed '/^\s*$/d' | wc -l)
+        plural=; [[ "$numberOf" != "1" ]] && plural=s
+        echo "** blacklisting a total of $numberOf domain${plural}..."	   		    
+	    	if [[ -n $piholeIPv6 ]];then	    	  
+	    	  cat $blacklist | awk -v ipv4addr="$piholeIP" -v ipv6addr="$piholeIPv6" '{sub(/\r$/,""); print ipv4addr" "$0"\n"ipv6addr" "$0}' >> $adList
+	      else	        
+	      	cat $blacklist | awk -v ipv4addr="$piholeIP" '{sub(/\r$/,""); print ipv4addr" "$0}' >>$adList
+	      fi		    
+		   
+	  	fi
+	  else
+
+	  for dom in "${domToRemoveList[@]}"
+		do	  
+	      #we need to remove the domains from the blacklist file and the host file	     
+	      echo $dom | sed 's/\./\\./g' | xargs -I {} perl -i -ne'print unless /[^.]'{}'(?!.)/;' $adList  
+	      echo $dom | sed 's/\./\\./g' | xargs -I {} perl -i -ne'print unless /'{}'(?!.)/;' $blacklist
+		done	  
+		fi
+	  
+}
+
+function Reload() {
+	# Reload hosts file
+	echo "** Refresh lists in dnsmasq..."
+
+	dnsmasqPid=$(pidof dnsmasq)
+
+	if [[ $dnsmasqPid ]]; then
+		# service already running - reload config
+		sudo kill -HUP $dnsmasqPid
+	else
+		# service not running, start it up
+		sudo service dnsmasq start
+	fi
+}
+
+###################################################
+
+for var in "$@"
+do
+  case "$var" in
+    "-nr"| "--noreload"  ) reload=false;;        			
+    "-d" | "--delmode"   ) addmode=false;;
+    "-f" | "--force"     ) force=true;;
+    "-q" | "--quiet"     ) versbose=false;;  			
+    *                    ) HandleOther $var;;
+  esac
+done
+
+PopBlacklistFile
+
+if $modifyHost || $force; then
+	echo "** Modifying Hosts File"
+	ModifyHostFile
+else
+  if $versbose; then
+	echo "** No changes need to be made"
+	fi
+	exit 1
+fi
+
+if $reload; then
+	Reload
+fi

advanced/Scripts/whitelist.sh

@@ -7,51 +7,174 @@
 # the Free Software Foundation, either version 2 of the License, or
 # (at your option) any later version.
 
+if [[ $# = 0 ]]; then
+    echo "Immediately whitelists one or more domains in the hosts file"
+    echo " "
+    echo "Usage: whitelist.sh domain1 [domain2 ...]"
+    echo "  "
+    echo "Options:"
+    echo "  -d, --delmode		Remove domains from the whitelist"
+    echo "  -nr, --noreload	Update Whitelist without refreshing dnsmasq"
+    echo "  -f, --force		Force updating of the hosts files, even if there are no changes"
+    echo "  -q, --quiet		output is less verbose"
+    exit 1
+fi
+
+#globals
 whitelist=/etc/pihole/whitelist.txt
 adList=/etc/pihole/gravity.list
-if [[ ! -f $whitelist ]];then
+reload=true
+addmode=true
+force=false
+versbose=true
+domList=()
+domToRemoveList=()
+
+piholeIPfile=/tmp/piholeIP
+piholeIPv6file=/etc/pihole/.useIPv6
+
+# Otherwise, the IP address can be taken directly from the machine, which will happen when the script is run by the user and not the installation script
+IPv4dev=$(ip route get 8.8.8.8 | awk '{for(i=1;i<=NF;i++)if($i~/dev/)print $(i+1)}')
+piholeIPCIDR=$(ip -o -f inet addr show dev $IPv4dev | awk '{print $4}' | awk 'END {print}')
+piholeIP=${piholeIPCIDR%/*}
+
+modifyHost=false
+
+
+if [[ -f $piholeIPv6file ]];then
+    # If the file exists, then the user previously chose to use IPv6 in the automated installer
+    piholeIPv6=$(ip -6 route get 2001:4860:4860::8888 | awk -F " " '{ for(i=1;i<=NF;i++) if ($i == "src") print $(i+1) }')
+fi
+
+
+function HandleOther(){	
+  #check validity of domain
+	validDomain=$(echo $1 | perl -ne'print if /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/')
+	
+	if [ -z "$validDomain" ]; then
+		echo $1 is not a valid argument or domain name
+	else	  
+	  domList=("${domList[@]}" $validDomain)
+	fi
+}
+
+function PopWhitelistFile(){
+	#check whitelist file exists, and if not, create it
+	if [[ ! -f $whitelist ]];then
   	  touch $whitelist
-fi
+	fi	
+	for dom in "${domList[@]}"
+	do	  
+	  if $addmode; then
+	  	AddDomain $dom
+	  else
+	    RemoveDomain $dom
+	  fi
+	done
+}
 
-if [[ $# = 0 ]]; then
+function AddDomain(){
-    echo "Immediately whitelists one or more domains."
+#| sed 's/\./\\./g'
-    echo "Usage: whitelist.sh domain1 [domain2 ...]"
+	bool=false
-fi
+	grep -Ex -q "$1" $whitelist || bool=true
+	if $bool; then
+	  #domain not found in the whitelist file, add it!
+	  if $versbose; then
+	  echo "** Adding $1 to whitelist file"
+	  fi
+		echo $1 >> $whitelist
+		modifyHost=true
+	else
+		if $versbose; then
+			echo "** $1 already whitelisted! No need to add"
+		fi
+	fi
+}
 
-combopattern=""
+function RemoveDomain(){
+  
+  bool=false
+  grep -Ex -q "$1" $whitelist || bool=true
+  if $bool; then
+  	#Domain is not in the whitelist file, no need to Remove
+  	if $versbose; then
+  	echo "** $1 is NOT whitelisted! No need to remove"
+  	fi
+  else
+    #Domain is in the whitelist file, add to a temporary array and remove from whitelist file
+    if $versbose; then
+    echo "** Un-whitelisting $dom..."
+    fi
+    domToRemoveList=("${domToRemoveList[@]}" $1)
+    modifyHost=true	  	
+  fi  	
+}
+
+function ModifyHostFile(){	
+	 if $addmode; then
+	    #remove domains in  from hosts file
+	    if [[ -r $whitelist ]];then
+        # Remove whitelist entries
+        numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l)
+        plural=; [[ "$numberOf" != "1" ]] && plural=s
+        echo "** Whitelisting a total of $numberOf domain${plural}..."	   
+	  	  awk -F':' '{ print $1 }' $whitelist | sed 's/\./\\./g' | xargs -I {} perl -i -ne'print unless /[^.]'{}'(?!.)/;' $adList
+	  	fi
+	  else
+	    #we need to add the removed domains to the hosts file
+	    for rdom in "${domToRemoveList[@]}"
+	    do
+	    	if [[ -n $piholeIPv6 ]];then
+	    	  echo "**Blacklisting $rdom on IPv4 and IPv6"
+	    	  echo $rdom | awk -v ipv4addr="$piholeIP" -v ipv6addr="$piholeIPv6" '{sub(/\r$/,""); print ipv4addr" "$0"\n"ipv6addr" "$0}' >> $adList
+	      else
+	        echo "**Blacklisting $rdom on IPv4"
+	      	echo $rdom | awk -v ipv4addr="$piholeIP" '{sub(/\r$/,""); print ipv4addr" "$0}' >>$adList
+	      fi	      	      
+	      echo $rdom| sed 's/\./\\./g' | xargs -I {} perl -i -ne'print unless /'{}'(?!.)/;' $whitelist
+	    done
+	  fi	
+}
+
+function Reload() {
+	# Reload hosts file
+	echo "** Refresh lists in dnsmasq..."
+	dnsmasqPid=$(pidof dnsmasq)
+
+	if [[ $dnsmasqPid ]]; then
+		# service already running - reload config
+		sudo kill -HUP $dnsmasqPid
+	else
+		# service not running, start it up
+		sudo service dnsmasq start
+	fi
+}
+
+###################################################
 
-# For each argument passed to this script
 for var in "$@"
 do
-  echo "Whitelisting $var..."
+  case "$var" in
-
+    "-nr"| "--noreload"  ) reload=false;;        			
-  # Construct basic pattern to match domain name.
+    "-d" | "--delmode"   ) addmode=false;;
-  basicpattern=$(echo $var | awk -F '[# \t]' 'NF>0&&$1!="" {print ""$1""}' | sed 's/\./\\./g')
+    "-f" | "--force"     ) force=true;;
-
+    "-q" | "--quiet"     ) versbose=false;;  			
-  if [[ "$basicpattern" != "" ]]; then
+    *                    ) HandleOther $var;;
-    # Add to the combination pattern that will be used below
+  esac
-    if [[ "$combopattern" != "" ]]; then combopattern="$combopattern|"; fi
-    combopattern="$combopattern$basicpattern"
-
-    # Also add the domain to the whitelist but only if it's not already present
-    grep -E -q "^$basicpattern$" $whitelist \
-    || echo "$var" >> $whitelist
-  fi
 done
 
-# Now report on and remove matched domains
+PopWhitelistFile
-if [[ "$combopattern" != "" ]]; then
-  echo "Modifying hosts file..."
 
-  # Construct pattern to match entry in hosts file.
+if $modifyHost || $force; then
-  # This consists of one or more IP addresses followed by the domain name.
+	echo "** Modifying Hosts File"
-  pattern=$(echo $combopattern | awk -F '[# \t]' '{printf "%s", "^(([0-9]+\.){3}[0-9]+ +)+("$1")$"}')
+	ModifyHostFile
-
+else
-  # Output what will be removed and then actually remove
+  if $versbose; then
-  sed -r -n 's/'"$pattern"'/  Removed: \3/p' $adList
+	echo "** No changes need to be made"
-  sed -r -i '/'"$pattern"'/d' $adList
+	exit 1
-
+	fi
-  echo "** $# domain(s) whitelisted."
+fi
-  # Force dnsmasq to reload /etc/pihole/gravity.list
+
-  kill -HUP $(pidof dnsmasq)
+if $reload; then
+	Reload
 fi

automated install/basic-install.sh

@@ -207,8 +207,9 @@ installScripts(){
 sudo curl -o /usr/local/bin/gravity.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/gravity.sh
 sudo curl -o /usr/local/bin/chronometer.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/chronometer.sh
 sudo curl -o /usr/local/bin/whitelist.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/whitelist.sh
+sudo curl -o /usr/local/bin/blacklist.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/blacklist.sh
 sudo curl -o /usr/local/bin/piholeLogFlush.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/piholeLogFlush.sh
-sudo chmod 755 /usr/local/bin/{gravity,chronometer,whitelist,piholeLogFlush}.sh
+sudo chmod 755 /usr/local/bin/{gravity,chronometer,whitelist,blacklist,piholeLogFlush}.sh
 }
 
 installConfigs(){

gravity.sh

@@ -48,12 +48,11 @@ blacklist=$piholeDir/blacklist.txt
 whitelist=$piholeDir/whitelist.txt
 latentWhitelist=$piholeDir/latentWhitelist.txt
 justDomainsExtension=domains
-matter=$basename.0.matter.txt
+matterandlight=$basename.0.matterandlight.txt
-andLight=$basename.1.andLight.txt
+supernova=$basename.1.supernova.txt
-supernova=$basename.2.supernova.txt
+eventHorizon=$basename.2.eventHorizon.txt
-eventHorizon=$basename.3.eventHorizon.txt
+accretionDisc=$basename.3.accretionDisc.txt
-accretionDisc=$basename.4.accretionDisc.txt
+eyeOfTheNeedle=$basename.4.wormhole.txt
-eyeOfTheNeedle=$basename.5.wormhole.txt
 
 # After setting defaults, check if there's local overrides
 if [[ -r $piholeDir/pihole.conf ]];then
@@ -160,48 +159,35 @@ function gravity_Schwarzchild() {
 
 	# Find all active domains and compile them into one file and remove CRs
 	echo "** Aggregating list of domains..."
-	truncate -s 0 $piholeDir/$matter
+	truncate -s 0 $piholeDir/$matterandlight
 	for i in "${activeDomains[@]}"
 	do
-   		cat $i |tr -d '\r' >> $piholeDir/$matter
+   		cat $i |tr -d '\r' >> $piholeDir/$matterandlight
 	done
 }
 
-# Pulsar - White/blacklist application
-function gravity_pulsar() {
 
+function gravity_Blacklist(){
 	# Append blacklist entries if they exist
-	if [[ -r $blacklist ]];then
+	blacklist.sh -f -nr -q
-        numberOf=$(cat $blacklist | sed '/^\s*$/d' | wc -l)
+}
-        echo "** Blacklisting $numberOf domain(s)..."
-        cat $blacklist >> $piholeDir/$matter
-	fi
 
-	# Whitelist (if applicable) domains
-	if [[ -r $whitelist ]];then
-        # Remove whitelist entries
-        numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l)
-        plural=; [[ "$numberOf" != "1" ]] && plural=s
-        echo "** Whitelisting $numberOf domain${plural}..."
-
-        # Append a "$" to the end, prepend a "^" to the beginning, and
-        # replace "." with "\." of each line to turn each entry into a
-        # regexp so it can be parsed out with grep -x
-        awk -F '[# \t]' 'NF>0&&$1!="" {print "^"$1"$"}' $whitelist | sed 's/\./\\./g' > $latentWhitelist
-	else
-        rm $latentWhitelist 2>/dev/null
-	fi
 
+function gravity_Whitelist() {
 	# Prevent our sources from being pulled into the hole
-	plural=; [[ "${#sources[@]}" != "1" ]] && plural=s
+	plural=; [[ "${sources[@]}" != "1" ]] && plural=s
 	echo "** Whitelisting ${#sources[@]} ad list source${plural}..."
+	
+	urls=()
 	for url in ${sources[@]}
 	do
-        echo "$url" | awk -F '/' '{print "^"$3"$"}' | sed 's/\./\\./g' >> $latentWhitelist
+        tmp=$(echo "$url" | awk -F '/' '{print $3}')
+        urls=("${urls[@]}" $tmp)
 	done
 	
-	# Remove whitelist entries from list
+	whitelist.sh -f -nr -q ${urls[@]}
-	grep -vxf $latentWhitelist $piholeDir/$matter > $piholeDir/$andLight
+
+		
 }
 
 function gravity_unique() {
@@ -240,12 +226,13 @@ function gravity_blackbody() {
 }
 
 function gravity_advanced() {
+
+
 	# Remove comments and print only the domain name
 	# Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious
 	# This helps with that and makes it easier to read
 	# It also helps with debugging so each stage of the script can be researched more in depth
-	awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $piholeDir/$andLight | \
+	awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $piholeDir/$matterandlight | sed -nr -e 's/\.{2,}/./g' -e '/\./p' >  $piholeDir/$supernova
-		sed -nr -e 's/\.{2,}/./g' -e '/\./p' >  $piholeDir/$supernova
 
 	numberOf=$(wc -l < $piholeDir/$supernova)
 	echo "** $numberOf domains being pulled in by gravity..."
@@ -256,7 +243,6 @@ function gravity_advanced() {
 function gravity_reload() {
 	# Reload hosts file
 	echo "** Refresh lists in dnsmasq..."
-
 	dnsmasqPid=$(pidof dnsmasq)
 
 	if [[ $dnsmasqPid ]]; then
@@ -268,11 +254,13 @@ function gravity_reload() {
 	fi
 }
 
+
 gravity_collapse
 gravity_spinup
 gravity_Schwarzchild
-gravity_pulsar
 gravity_advanced
 gravity_hostFormat
 gravity_blackbody
+gravity_Whitelist
+gravity_Blacklist
 gravity_reload