Merge pull request #1694 from pi-hole/fix/qf-opt

Slow query fix & queryFunc optimisation

automated install/basic-install.sh

@@ -164,7 +164,7 @@ if command -v apt-get &> /dev/null; then
   # These programs are stored in an array so they can be looped through later
   INSTALLER_DEPS=(apt-utils dialog debconf dhcpcd5 git ${iproute_pkg} whiptail)
   # Pi-hole itself has several dependencies that also need to be installed
-  PIHOLE_DEPS=(bc cron curl dnsmasq dnsutils iputils-ping lsof netcat sudo unzip wget)
+  PIHOLE_DEPS=(bc cron curl dnsmasq dnsutils iputils-ping lsof netcat sudo unzip wget idn2)
   # The Web dashboard has some that also need to be installed
   # It's useful to separate the two since our repos are also setup as "Core" code and "Web" code
   PIHOLE_WEB_DEPS=(lighttpd ${phpVer}-common ${phpVer}-cgi ${phpVer}-${phpSqlite})
@@ -208,7 +208,7 @@ elif command -v rpm &> /dev/null; then
   PKG_INSTALL=(${PKG_MANAGER} install -y)
   PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l"
   INSTALLER_DEPS=(dialog git iproute net-tools newt procps-ng)
-  PIHOLE_DEPS=(bc bind-utils cronie curl dnsmasq findutils nmap-ncat sudo unzip wget)
+  PIHOLE_DEPS=(bc bind-utils cronie curl dnsmasq findutils nmap-ncat sudo unzip wget idn2)
   PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php php-common php-cli php-pdo)
   if ! grep -q 'Fedora' /etc/redhat-release; then
     INSTALLER_DEPS=("${INSTALLER_DEPS[@]}" "epel-release");

pihole

@@ -83,21 +83,27 @@ updateGravityFunc() {
   exit 0
 }
 
+# Scan an array of files for matching strings
 scanList(){
-  domain="${1}"
+  local domain="${1}" lists="${2}" type="${3:-}"
-  list="${2}"
-  method="${3}"
 
-  # Switch folder, preventing grep from printing file path
+  # Prevent grep from printing file path
-  cd "/etc/pihole" || return 1
+  cd "/etc/pihole" || exit 1
 
-  if [[ -n "${method}" ]]; then
+  # Prevent grep -i matching slowly: http://bit.ly/2xFXtUX
-    grep -i -E -l "(^|\s|\/)${domain}($|\s|\/)" ${list} /dev/null 2> /dev/null
+  export LC_CTYPE=C
-  else
+
-    grep -i "${domain}" ${list} /dev/null 2> /dev/null
+  # /dev/null forces filename to be printed when only one list has been generated
-  fi
+  # shellcheck disable=SC2086
+  case "${type}" in
+    "exact" ) grep -i -E -l "(^|\\s)${domain}($|\\s|#)" ${lists} /dev/null;;
+    "wc"    ) grep -i -o -m 1 "/${domain}/" ${lists};;
+    *       ) grep -i "${domain}" ${lists} /dev/null;;
+  esac
 }
 
+# Print each subdomain
+# e.g: foo.bar.baz.com = "foo.bar.baz.com bar.baz.com baz.com com"
 processWildcards() {
   IFS="." read -r -a array <<< "${1}"
   for (( i=${#array[@]}-1; i>=0; i-- )); do
@@ -114,8 +120,8 @@ processWildcards() {
 }
 
 queryFunc() {
-  options="$*"
+  shift
-  options="${options/-q /}"
+  local options="$*" adlist="" all="" exact="" blockpage="" matchType="match"
 
   if [[ "${options}" == "-h" ]] || [[ "${options}" == "--help" ]]; then
     echo "Usage: pihole -q [option] <domain>
@@ -130,200 +136,175 @@ Options:
     exit 0
   fi
 
-  if [[ "${options}" == *"-exact"* ]]; then
+  if [[ ! -e "/etc/pihole/adlists.list" ]]; then
-    method="exact"
+    echo -e "${COL_LIGHT_RED}The file '/etc/pihole/adlists.list' was not found${COL_NC}"
-    exact=true
-  fi
-
-  if [[ "${options}" == *"-adlist"* ]]; then
-    adlist=true
-  fi
-
-  if [[ "${options}" == *"-bp"* ]]; then
-    method="exact"
-    blockpage=true
-  fi
-
-  if [[ "${options}" == *"-all"* ]]; then
-    all=true
-  fi
-
-  # Strip valid options, leaving only the domain and invalid options
-  options=$(sed 's/ \?-\(exact\|adlist\(s\)\?\|bp\|all\) \?//g' <<< "$options")
-
-  # Handle errors
-  if [[ "${options}" == *" "* ]]; then
-    error=true
-    str="Unknown option specified"
-  elif [[ "${options}" == "-q" ]]; then
-    error=true
-    str="No domain specified"
-  fi
-
-  if [[ -n "${error}" ]]; then
-    echo -e "  ${COL_LIGHT_RED}${str}${COL_NC}
-  Try 'pihole -q --help' for more information."
     exit 1
   fi
 
-  # If domain contains non ASCII characters, convert domain to punycode if python is available
+  # Handle valid options
-  # Cr: https://serverfault.com/a/335079
+  if [[ "${options}" == *"-bp"* ]]; then
-  if [[ "$options" = *[![:ascii:]]* ]]; then
+    exact="exact"; blockpage=true
-    if command -v python &> /dev/null; then
-      query=$(python -c 'import sys;print sys.argv[1].decode("utf-8").encode("idna")' "${options}")
-    fi
   else
-    query="${options}"
+    [[ "${options}" == *"-adlist"* ]] && adlist=true
+    [[ "${options}" == *"-all"* ]] && all=true
+    if [[ "${options}" == *"-exact"* ]]; then
+      exact="exact"; matchType="exact ${matchType}"
+    fi
+  fi
+
+  # Strip valid options, leaving only the domain and invalid options
+  # This allows users to place the options before or after the domain
+  options=$(sed -E 's/ ?-(bp|adlists?|all|exact)//g' <<< "${options}")
+
+  # Handle remaining options
+  # If $options contain non ASCII characters, convert to punycode
+  case "${options}" in
+    ""             ) str="No domain specified";;
+    *" "*          ) str="Unknown query option specified";;
+    *[![:ascii:]]* ) domainQuery=$(idn2 "${options}");;
+    *              ) domainQuery="${options}";;
+  esac
+
+  if [[ -n "${str:-}" ]]; then
+    echo -e "${str}${COL_NC}\\nTry 'pihole -q --help' for more information."
+    exit 1
   fi
 
   # Scan Whitelist and Blacklist
   lists="whitelist.txt blacklist.txt"
-  results=($(scanList "${query}" "${lists}" "${method}"))
+  mapfile -t results <<< "$(scanList "${domainQuery}" "${lists}" "${exact}")"
 
   if [[ -n "${results[*]}" ]]; then
-    blResult=true
+    wbMatch=true
-    # Loop through each scanList line to print appropriate title
+
+    # Loop through each result in order to print unique file title once
     for result in "${results[@]}"; do
-      filename="${result/:*/}"
+      fileName="${result%%.*}"
-      if [[ -n "$exact" ]]; then
+
-        printf "  Exact result in %s\n" "${filename}"
+      if [[ -n "${blockpage}" ]]; then
-      elif [[ -n "$blockpage" ]]; then
+        echo "π ${result}"
-        printf "π %s\n" "${filename}"
+        exit 0
+      elif [[ -n "${exact}" ]]; then
+        echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}"
       else
-        domain="${result/*:/}"
+        # Only print filename title once per file
-        if [[ ! "${filename}" == "${filename_prev:-}" ]]; then
+        if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then
-          printf "  Result from %s\n" "${filename}"
+          echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}"
+          fileName_prev="${fileName}"
         fi
-        printf "    %s\n" "${domain}"
+        echo "   ${result#*:}"
-        filename_prev="${filename}"
       fi
     done
   fi
 
   # Scan Wildcards
   if [[ -e "${wildcardlist}" ]]; then
-    wildcards=($(processWildcards "${query}"))
+    # Determine all subdomains, domain and TLDs 
+    mapfile -t wildcards <<< "$(processWildcards "${domainQuery}")"
 
     for match in "${wildcards[@]}"; do
-      results=($(scanList "\/${match}\/" ${wildcardlist}))
+      # Search wildcard list for matches
+      mapfile -t results <<< "$(scanList "${match}" "${wildcardlist}" "wc")"
 
       if [[ -n "${results[*]}" ]]; then
-        # Remove empty lines before couting number of results
+        if [[ -z "${wcMatch:-}" ]] && [[ -z "${blockpage}" ]]; then
-        count=$(sed '/^\s*$/d' <<< "${results[@]}" | wc -l)
+          wcMatch=true
-        if [[ "${count}" -ge 0 ]]; then
+          echo " ${matchType^} found in ${COL_BOLD}Wildcards${COL_NC}:"
-          blResult=true
-          if [[ -z "${blockpage}" ]]; then
-            printf "  Wildcard result in %s\n" "${wildcardlist/*dnsmasq.d\/}"
         fi
 
-          if [[ -n "${blockpage}" ]]; then
+        case "${blockpage}" in
-            echo "π ${wildcardlist/*\/}"
+          true ) echo "π ${wildcardlist##*/}"; exit 0;;
-          else
+          *    ) echo "   *.${match}";;
-            echo "    *.${match}"
+        esac
-          fi
-        fi
       fi
     done
-
-    [[ -n "${blResult}" ]] && [[ -n "${blockpage}" ]] && exit 0
   fi
 
-  # Glob *.domains file names, remove file paths and sort by list number
+  # Get version sorted *.domains filenames (without dir path)
-  lists_raw=(/etc/pihole/*.domains)
+  lists=("$(cd "/etc/pihole" || exit 0; printf "%s\\n" -- *.domains | sort -V)")
-  IFS_OLD=$IFS
-  IFS=$'\n'
-  lists=$(sort -t . -k 2 -g <<< "${lists_raw[*]//\/etc\/pihole\//}")
 
-  # Scan Domains files
+  # Query blocklists for occurences of domain
-  results=($(scanList "${query}" "${lists}" "${method}"))
+  mapfile -t results <<< "$(scanList "${domainQuery}" "${lists[*]}" "${exact}")"
 
   # Handle notices
-  if [[ -z "${blResult}" ]] && [[ -z "${results[*]}" ]]; then
+  if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then
-    notice=true
+    echo -e "  ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} found within block lists"
-    str="No ${method/t/t }results found for ${query} found within block lists"
+    exit 0
-  elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 16000 ]]; then
+  elif [[ -z "${results[*]}" ]]; then
-    # 16000 chars is 15 chars X 1000 lines worth of results
+    # Result found in WL/BL/Wildcards
-    notice=true
+    exit 0
-    str="Hundreds of ${method/t/t }results found for ${query}
+  elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 100 ]]; then
-    This can be overriden using the -all option"
+    echo -e "  ${INFO} Over 100 ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC}
+      This can be overridden using the -all option"
+    exit 0
   fi
 
-  if [[ -n "${notice}" ]]; then
+  # Remove unwanted content from non-exact $results
-    echo -e "  ${INFO} ${str}"
+  if [[ -z "${exact}" ]]; then
-    exit
+    # Delete lines starting with #
+    # Remove comments after domain
+    # Remove hosts format IP address
+    mapfile -t results <<< "$(IFS=$'\n'; sed \
+      -e "/:#/d" \
+      -e "s/[ \\t]#.*//g" \
+      -e "s/:.*[ \\t]/:/g" \
+    <<< "${results[*]}")"
+
+    # Exit if result was in a comment
+    [[ -z "${results[*]}" ]] && exit 0
   fi
 
-  # Remove unwanted content from results
+  # Get adlist file content as array
-  if [[ -z "${method}" ]]; then
-    results=($(sed "/:#/d" <<< "${results[*]}")) # Lines starting with comments
-    results=($(sed "s/[ \t]#.*//g" <<< "${results[*]}")) # Comments after domain
-    results=($(sed "s/:.*[ \t]/:/g" <<< "${results[*]}")) # IP address
-  fi
-  IFS=$IFS_OLD
-
-  # Get adlist content as array
   if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then
-    if [[ -f "/etc/pihole/adlists.list" ]]; then
+    for adlistUrl in $(< "/etc/pihole/adlists.list"); do
-      for url in $(< /etc/pihole/adlists.list); do
+      if [[ "${adlistUrl:0:4}" =~ (http|www.) ]]; then
-        if [[ "${url:0:4}" == "http" ]] || [[ "${url:0:3}" == "www" ]]; then
+        adlists+=("${adlistUrl}")
-          adlists+=("$url")
       fi
     done
-    else
-      echo -e "  ${COL_LIGHT_RED}The file '/etc/pihole/adlists.list' was not found${COL_NC}"
-      exit 1
-    fi
   fi
 
-  if [[ -n "${results[*]}" ]]; then
+  # Print "Exact matches for" title
-    if [[ -n "${exact}" ]]; then
+  if [[ -n "${exact}" ]] && [[ -z "${blockpage}" ]]; then
-      echo "  Exact result(s) for ${query} found in:"
+    plural=""; [[ "${#results[*]}" -gt 1 ]] && plural="es"
+    echo " ${matchType^}${plural} for ${COL_BOLD}${domainQuery}${COL_NC} found in:"
   fi
 
   for result in "${results[@]}"; do
-      filename="${result/:*/}"
+    fileName="${result/:*/}"
 
-      # Convert file name to URL name for -adlist or -bp options
+    # Determine *.domains URL using filename's number
     if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then
-        filenum=("${filename/list./}")
+      fileNum="${fileName/list./}"; fileNum="${fileNum%%.*}"
-        filenum=("${filenum/.*/}")
+      fileName="${adlists[$fileNum]}"
-        filename="${adlists[$filenum]}"
 
-        # If gravity has generated associated .domains files
+      # Discrepency occurs when adlists has been modified, but Gravity has not been run
-        # but adlists.list has been modified since
+      if [[ -z "${fileName}" ]]; then
-        if [[ -z "${filename}" ]]; then
+        fileName="${COL_LIGHT_RED}(no associated adlists URL found)${COL_NC}"
-          filename="${COL_LIGHT_RED}Error: no associated adlists URL found${COL_NC}"
       fi
     fi
 
-      if [[ -n "${exact}" ]]; then
+    if [[ -n "${blockpage}" ]]; then
-        printf "    %s\n" "${filename}"
+      echo "${fileNum} ${fileName}"
-      elif [[ -n "${blockpage}" ]]; then
+    elif [[ -n "${exact}" ]]; then
-        printf "%s %s\n" "${filenum}" "${filename}"
+      echo "   ${fileName}"
-      else # Standard query output
-
-        # Print filename heading once per file, not for every match
-        if [[ ! "${filename}" == "${filename_prev:-}" ]]; then
-          unset count
-          printf "  Result from %s\n" "${filename}"
     else
-          let count++
+      if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then
+        count=""
+        echo " ${matchType^} found in ${COL_BOLD}${fileName}${COL_NC}:"
+        fileName_prev="${fileName}"
       fi
+      : $((count++))
 
       # Print matching domain if $max_count has not been reached
-        [[ -z "${all}" ]] && max_count="20"
+      [[ -z "${all}" ]] && max_count="50"
-        if [[ -z "${all}" ]] && [[ "${count}" -eq "${max_count}" ]]; then
+      if [[ -z "${all}" ]] && [[ "${count}" -ge "${max_count}" ]]; then
-          echo "    Over $count results found, skipping rest of file"
+        [[ "${count}" -gt "${max_count}" ]] && continue
-        elif [[ -z "${all}" ]] && [[ "${count}" -gt "${max_count}" ]]; then
+        echo "   ${COL_GRAY}Over ${count} results found, skipping rest of file${COL_NC}"
-          continue
       else
-          domain="${result/*:/}"
+        echo "   ${result#*:}"
-          printf "    %s\n" "${domain}"
       fi
-        filename_prev="${filename}"
     fi
   done
-  fi
 
   exit 0
 }