Merge branch 'development' into developmentNew

advanced/Scripts/webpage.sh

@@ -221,18 +221,19 @@ Reboot() {
 }
 
 RestartDNS() {
-  local str="Restarting dnsmasq"
+  local str="Restarting DNS service"
-  echo -ne "  ${INFO} ${str}..."
+  [[ -t 1 ]] && echo -ne "  ${INFO} ${str}"
-  if [[ -x "$(command -v systemctl)" ]]; then
+  if command -v systemctl &> /dev/null; then
-    systemctl restart dnsmasq
+    output=$( { systemctl restart dnsmasq; } 2>&1 )
   else
-    service dnsmasq restart
+    output=$( { service dnsmasq restart; } 2>&1 )
   fi
 
-  if [[ "$?" == 0 ]]; then
+ if [[ -z "${output}" ]]; then
-    echo -e "${OVER}  ${TICK} ${str}"
+    [[ -t 1 ]] && echo -e "${OVER}  ${TICK} ${str}"
   else
-    echo -e "${OVER}  ${CROSS} ${str}"
+    [[ ! -t 1 ]] && OVER=""
+    echo -e "${OVER}  ${CROSS} ${output}"
   fi
 }
 

automated install/basic-install.sh

@@ -1485,8 +1485,7 @@ finalExports() {
 
   # If the setup variable file exists,
   if [ -e "${setupVars}" ]; then
-    # update the variables in the file
+    sed -i.update.bak '/PIHOLE_INTERFACE/d;/IPV4_ADDRESS/d;/IPV6_ADDRESS/d;/PIHOLE_DNS_1/d;/PIHOLE_DNS_2/d;/QUERY_LOGGING/d;/INSTALL_WEB/d;/LIGHTTPD_ENABLED/d;' "${setupVars}"
-    sed -i.update.bak '/PIHOLE_INTERFACE/d;/IPV4_ADDRESS/d;/IPV6_ADDRESS/d;/PIHOLE_DNS_1/d;/PIHOLE_DNS_2/d;/QUERY_LOGGING/d;/INSTALL_WEB/d;' "${setupVars}"
   fi
   # echo the information to the user
     {
@@ -1497,6 +1496,7 @@ finalExports() {
   echo "PIHOLE_DNS_2=${PIHOLE_DNS_2}"
   echo "QUERY_LOGGING=${QUERY_LOGGING}"
   echo "INSTALL_WEB=${INSTALL_WEB}"
+  echo "LIGHTTPD_ENABLED=${LIGHTTPD_ENABLED}"
     }>> "${setupVars}"
 
   # Look for DNS server settings which would have to be reapplied
@@ -1585,9 +1585,6 @@ installPihole() {
   FTLdetect || echo -e "  ${CROSS} FTL Engine not installed."
   # Configure the firewall
   configureFirewall
-  # Run the final exports
-  finalExports
-  #runGravity
 }
 
 # At some point in the future this list can be pruned, for now we'll need it to ensure updates don't break.
@@ -1621,8 +1618,8 @@ updatePihole() {
   installLogrotate
   # Detect if FTL is installed
   FTLdetect || echo -e "  ${CROSS} FTL Engine not installed."
-  finalExports #re-export setupVars.conf to account for any new vars added in new versions
+
-  #runGravity
+
 }
 
 
@@ -2052,10 +2049,24 @@ main() {
   enable_service dnsmasq
 
   # If the Web server was installed,
-  if [[ ${INSTALL_WEB} == true ]]; then
+  if [[ "${INSTALL_WEB}" == true ]]; then
-    # enable it
+    # Check to see if lighttpd was already set to run on reboot
-    start_service lighttpd
+    if [[ "${useUpdateVars}" == true ]]; then
-    enable_service lighttpd
+      if [[ -x "$(command -v systemctl)" ]]; then
+        # Value will either be 1, if true, or 0
+        LIGHTTPD_ENABLED=$(systemctl is-enabled lighttpd | grep -c 'enabled' || true)
+      else
+        # Value will either be 1, if true, or 0
+        LIGHTTPD_ENABLED=$(service lighttpd status | awk '/Loaded:/ {print $0}' | grep -c 'enabled' || true)
+      fi
+    fi
+
+    if [[ "${LIGHTTPD_ENABLED}" == "1" ]]; then
+      start_service lighttpd
+      enable_service lighttpd
+    else
+      echo -e "  ${INFO} Lighttpd is disabled, skipping service restart"
+    fi
   fi
 
   # Download and compile the aggregated block list
@@ -2103,6 +2114,8 @@ main() {
   # Display where the log file is
   echo -e "\n  ${INFO} The install log is located at: /etc/pihole/install.log
   ${COL_LIGHT_GREEN}${INSTALL_TYPE} Complete! ${COL_NC}"
+  #update setupvars.conf with any variables that may or may not have been changed during the install
+  finalExports
 }
 
 #

pihole

@@ -87,10 +87,14 @@ scanList(){
   domain="${1}"
   list="${2}"
   method="${3}"
-  if [[ ${method} == "-exact" ]] ; then
+
-    grep -i -E "(^|\s)${domain}($|\s)" "${list}"
+  # Switch folder, preventing grep from printing file path
+  cd "/etc/pihole" || return 1
+
+  if [[ -n "${method}" ]]; then
+    grep -i -E -l "(^|\s|\/)${domain}($|\s|\/)" ${list} /dev/null 2> /dev/null
   else
-    grep -i "${domain}" "${list}"
+    grep -i "${domain}" ${list} /dev/null 2> /dev/null
   fi
 }
 
@@ -110,46 +114,210 @@ processWildcards() {
 }
 
 queryFunc() {
-  domain="${2}"
+  options="$*"
+  options="${options/-q /}"
 
-  if [[ -z "${domain}" ]]; then
+  if [[ "${options}" == "-h" ]] || [[ "${options}" == "--help" ]]; then
-    echo -e "  ${COL_LIGHT_RED}Invalid option${COL_NC}
+    echo "Usage: pihole -q [option] <domain>
-  Try 'pihole query --help' for more information."
+Example: 'pihole -q -exact domain.com'
+Query the adlists for a specified domain
+
+Options:
+  -adlist             Print the name of the block list URL
+  -exact              Search the block lists for exact domain matches
+  -all                Return all query matches within a block list
+  -h, --help          Show this help dialog"
+    exit 0
+  fi
+
+  if [[ "${options}" == *"-exact"* ]]; then
+    method="exact"
+    exact=true
+  fi
+
+  if [[ "${options}" == *"-adlist"* ]]; then
+    adlist=true
+  fi
+
+  if [[ "${options}" == *"-bp"* ]]; then
+    method="exact"
+    blockpage=true
+  fi
+
+  if [[ "${options}" == *"-all"* ]]; then
+    all=true
+  fi
+
+  # Strip valid options, leaving only the domain and invalid options
+  options=$(sed 's/ \?-\(exact\|adlist\|bp\|all\) \?//g' <<< "$options")
+
+  # Handle errors
+  if [[ "${options}" == *" "* ]]; then
+    error=true
+    str="Unknown option specified"
+  elif [[ "${options}" == "-q" ]]; then
+    error=true
+    str="No domain specified"
+  fi
+
+  if [[ -n "${error}" ]]; then
+    echo -e "  ${COL_LIGHT_RED}${str}${COL_NC}
+  Try 'pihole -q --help' for more information."
     exit 1
   fi
 
-  method="${3}"
+  # If domain contains non ASCII characters, convert domain to punycode if python is available
-  lists=( /etc/pihole/list.* /etc/pihole/blacklist.txt)
+  # Cr: https://serverfault.com/a/335079
-  for list in ${lists[@]}; do
+  if [[ "$options" = *[![:ascii:]]* ]]; then
-    if [ -e "${list}" ]; then
+    if command -v python &> /dev/null; then
-      result=$(scanList ${domain} ${list} ${method})
+      query=$(python -c 'import sys;print sys.argv[1].decode("utf-8").encode("idna")' "${options}")
-      # Remove empty lines before couting number of results
-      count=$(sed '/^\s*$/d' <<< "$result" | wc -l)
-      echo "${list} (${count} results)"
-      if [[ ${count} > 0 ]]; then
-        echo "${result}"
-      fi
-      echo ""
-    else
-      echo -e "  ${CROSS} List does not exist"
-      echo ""
     fi
-  done
+  else
+    query="${options}"
+  fi
 
-  # Scan for possible wildcard matches
+  # Scan Whitelist and Blacklist
-  if [ -e "${wildcardlist}" ]; then
+  lists="whitelist.txt blacklist.txt"
-    local wildcards=($(processWildcards "${domain}"))
+  results=($(scanList "${query}" "${lists}" "${method}"))
-    for domain in ${wildcards[@]}; do
+
-      result=$(scanList "\/${domain}\/" ${wildcardlist})
+  if [[ -n "${results[*]}" ]]; then
-      # Remove empty lines before couting number of results
+    # Loop through each scanList line to print appropriate title
-      count=$(sed '/^\s*$/d' <<< "$result" | wc -l)
+    for result in "${results[@]}"; do
-      if [[ ${count} > 0 ]]; then
+      filename="${result/:*/}"
-        echo -e "  ${TICK} Wildcard blocking ${domain} (${count} results)"
+      if [[ -n "$exact" ]]; then
-        echo "${result}"
+        printf "  Exact result in %s\n" "${filename}"
-        echo ""
+      elif [[ -n "$blockpage" ]]; then
+        printf "  [i] %s\n" "${filename}"
+      else
+        domain="${result/*:/}"
+        if [[ ! "${filename}" == "${filename_prev:-}" ]]; then
+          printf "  Result from %s\n" "${filename}"
+        fi
+        printf "    %s\n" "${domain}"
+        filename_prev="${filename}"
       fi
     done
   fi
+
+  # Scan Wildcards
+  if [[ -e "${wildcardlist}" ]]; then
+    wildcards=($(processWildcards "${query}"))
+
+    for match in "${wildcards[@]}"; do
+      results=($(scanList "\/${match}\/" ${wildcardlist}))
+
+      if [[ -n "${results[*]}" ]]; then
+        # Remove empty lines before couting number of results
+        count=$(sed '/^\s*$/d' <<< "${results[@]}" | wc -l)
+        if [[ "${count}" -ge 0 ]]; then
+          blResult=true
+          if [[ -z "${blockpage}" ]]; then
+            printf "  Wildcard result in %s\n" "${wildcardlist/*dnsmasq.d\/}"
+          fi
+
+          if [[ -n "${blockpage}" ]]; then
+            echo "  ${INFO} ${match}"
+          else
+            echo "    *.${match}"
+          fi
+        fi
+      fi
+    done
+
+    [[ -n "${blResult}" ]] && [[ -n "${blockpage}" ]] && exit 0
+  fi
+
+  # Glob *.domains file names, remove file paths and sort by list number
+  lists_raw=(/etc/pihole/*.domains)
+  IFS_OLD=$IFS
+  IFS=$'\n'
+  lists=$(sort -t . -k 2 -g <<< "${lists_raw[*]//\/etc\/pihole\//}")
+
+  # Scan Domains files
+  results=($(scanList "${query}" "${lists}" "${method}"))
+
+  # Handle notices
+  if [[ -z "${blResult}" ]] && [[ -z "${results[*]}" ]]; then
+    notice=true
+    str="No ${method/t/t }results found for ${query} found within block lists"
+  elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 16000 ]]; then
+    # 16000 chars is 15 chars X 1000 lines worth of results
+    notice=true
+    str="Hundreds of ${method/t/t }results found for ${query}
+    This can be overriden using the -all option"
+  fi
+
+  if [[ -n "${notice}" ]]; then
+    echo -e "  ${INFO} ${str}"
+    exit
+  fi
+
+  # Remove unwanted content from results
+  if [[ -z "${method}" ]]; then
+    results=($(sed "/:#/d" <<< "${results[*]}")) # Lines starting with comments
+    results=($(sed "s/[ \t]#.*//g" <<< "${results[*]}")) # Comments after domain
+    results=($(sed "s/:.*[ \t]/:/g" <<< "${results[*]}")) # IP address
+  fi
+  IFS=$IFS_OLD
+
+  # Get adlist content as array
+  if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then
+    if [[ -f "/etc/pihole/adlists.list" ]]; then
+      for url in $(< /etc/pihole/adlists.list); do
+        if [[ "${url:0:4}" == "http" ]] || [[ "${url:0:3}" == "www" ]]; then
+          adlists+=("$url")
+        fi
+      done
+    else
+      echo -e "  ${COL_LIGHT_RED}The file '/etc/pihole/adlists.list' was not found${COL_NC}"
+      exit 1
+    fi
+  fi
+
+  if [[ -n "${results[*]}" ]]; then
+    if [[ -n "${exact}" ]]; then
+      echo "  Exact result(s) for ${query} found in:"
+    fi
+
+    for result in "${results[@]}"; do
+      filename="${result/:*/}"
+
+      # Convert file name to URL name for -adlist or -bp options
+      if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then
+        filenum=("${filename/list./}")
+        filenum=("${filenum/.*/}")
+        filename="${adlists[$filenum]}"
+      fi
+
+      if [[ -n "${exact}" ]]; then
+        printf "    %s\n" "${filename}"
+      elif [[ -n "${blockpage}" ]]; then
+        printf "  [%s] %s\n" "${filenum}" "${filename}"
+      else # Standard query output
+
+        # Print filename heading once per file, not for every match
+        if [[ ! "${filename}" == "${filename_prev:-}" ]]; then
+          unset count
+          printf "  Result from %s\n" "${filename}"
+        else
+          let count++
+        fi
+
+        # Print matching domain if $max_count has not been reached
+        [[ -z "${all}" ]] && max_count="20"
+        if [[ -z "${all}" ]] && [[ "${count}" -eq "${max_count}" ]]; then
+          echo "    Over $count results found, skipping rest of file"
+        elif [[ -z "${all}" ]] && [[ "${count}" -gt "${max_count}" ]]; then
+          continue
+        else
+          domain="${result/*:/}"
+          printf "    %s\n" "${domain}"
+        fi
+        filename_prev="${filename}"
+      fi
+    done
+  fi
+
   exit 0
 }
 
@@ -173,24 +341,32 @@ versionFunc() {
 
 restartDNS() {
   dnsmasqPid=$(pidof dnsmasq)
+  local str="Restarting DNS service"
+  echo -ne "  ${INFO} ${str}"
   if [[ "${dnsmasqPid}" ]]; then
     # Service already running - reload config
-    echo -ne "  ${INFO} Restarting dnsmasq"
     if [[ -x "$(command -v systemctl)" ]]; then
-      systemctl restart dnsmasq
+      output=$( { systemctl restart dnsmasq; } 2>&1 )
     else
-      service dnsmasq restart
+      output=$( { service dnsmasq restart; } 2>&1 )
+    fi
+    if [[ -z "${output}" ]]; then
+      echo -e "${OVER}  ${TICK} ${str}"
+    else
+      echo -e "${OVER}  ${CROSS} ${output}"
     fi
-    [[ "$?" == 0 ]] && echo -e "${OVER}  ${TICK} Restarted dnsmasq" || echo -e "${OVER}  ${CROSS} Failed to restart dnsmasq"
   else
     # Service not running, start it up
-    echo -ne "  ${INFO} Starting dnsmasq"
     if [[ -x "$(command -v systemctl)" ]]; then
-      systemctl start dnsmasq
+      output=$( { systemctl start dnsmasq; } 2>&1 )
     else
-      service dnsmasq start
+      output=$( { service dnsmasq start; } 2>&1 )
+    fi
+    if [[ -z "${output}" ]]; then
+      echo -e "${OVER}  ${TICK} ${str}"
+    else
+      echo -e "${OVER}  ${CROSS} ${output}"
     fi
-    [[ "$?" == 0 ]] && echo -e "${OVER}  ${TICK} Restarted dnsmasq" || echo -e "${OVER}  ${CROSS} Failed to restart dnsmasq"
   fi
 }
 
@@ -430,7 +606,7 @@ Options:
   -l, logging         Specify whether the Pi-hole log should be used
                         Add '-h' for more info on logging usage
   -q, query           Query the adlists for a specified domain
-                        Add '-exact' AFTER a specified domain for exact match
+                        Add '-h' for more info on query usage
   -up, updatePihole   Update Pi-hole subsystems
   -v, version         Show installed versions of Pi-hole, Admin Console & FTL
                         Add '-h' for more info on version usage