From 3aa838bbe47aee2f4ec804873fc2b267422d8f3c Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sat, 4 May 2019 12:47:25 +0200
Subject: [PATCH 01/27] Implement black- and whitelist searching with SQL
 statements. We use the ESCAPE clause in the LIKE query as the underscore "_"
 wildcard matches any single character but we want to suppress this behavior
 (underscores can be legitimate part of domains)

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/query.sh | 70 +++++++++++++++++++++++++++------------
 1 file changed, 49 insertions(+), 21 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index a4ac895b..9b7a0fab 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -102,29 +102,57 @@ if [[ -n "${str:-}" ]]; then
     exit 1
 fi
 
-# Scan Whitelist and Blacklist
-lists="whitelist.txt blacklist.txt"
-mapfile -t results <<< "$(scanList "${domainQuery}" "${lists}" "${exact}")"
-if [[ -n "${results[*]}" ]]; then
-    wbMatch=true
-    # Loop through each result in order to print unique file title once
-    for result in "${results[@]}"; do
-        fileName="${result%%.*}"
-        if [[ -n "${blockpage}" ]]; then
-            echo "π ${result}"
-            exit 0
-        elif [[ -n "${exact}" ]]; then
-            echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}"
-        else
-            # Only print filename title once per file
-            if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then
+scanDatabaseTable() {
+    local domain table type querystr result
+    domain="${1}"
+    table="${2}"
+    type="${3:-}"
+
+    # As underscores are legitimate parts of domains, we escape possible them when using the LIKE operator.
+    # Underscores are a SQLite wildcard matching exactly one character. We obviously want to suppress this
+    # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched
+    # as a literal underscore character.
+    case "${type}" in
+        "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";;
+        *       ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\'";;
+    esac
+
+    # Send prepared query to gravity database
+    result="$(sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null
+    if [[ -n "${result}" ]]; then
+        # Prepend listname (separated by a colon) if we found at least one result
+        # and output result
+        results="$(sed "s/^/${table}:/g;" <<< "${result}")"
+    else
+        # Output empty string as the database query didn't return any result
+        return
+    fi
+    mapfile -t results <<< "${results}"
+    if [[ -n "${results[*]}" ]]; then
+        wbMatch=true
+        # Loop through each result in order to print unique file title once
+        for result in "${results[@]}"; do
+            fileName="${result%%:*}"
+            if [[ -n "${blockpage}" ]]; then
+                echo "π ${result}"
+                exit 0
+            elif [[ -n "${exact}" ]]; then
                 echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}"
-                fileName_prev="${fileName}"
+            else
+                # Only print filename title once per file
+                if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then
+                    echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}"
+                    fileName_prev="${fileName}"
+                fi
+            echo "   ${result#*:}"
             fi
-        echo "   ${result#*:}"
-        fi
-    done
-fi
+        done
+    fi
+}
+
+# Scan Whitelist and Blacklist
+scanDatabaseTable "${domainQuery}" "whitelist" "${exact}"
+scanDatabaseTable "${domainQuery}" "blacklist" "${exact}"
 
 # Scan Wildcards
 if [[ -e "${wildcardlist}" ]]; then

From 6ba58896d23c5ac7bf497413192a1fdd225e03b1 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sat, 4 May 2019 13:15:30 +0200
Subject: [PATCH 02/27] Simplify code

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/query.sh | 48 +++++++++++++++++----------------------
 1 file changed, 21 insertions(+), 27 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index 9b7a0fab..66b55e87 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -103,7 +103,7 @@ if [[ -n "${str:-}" ]]; then
 fi
 
 scanDatabaseTable() {
-    local domain table type querystr result
+    local domain table type querystr result table_prev
     domain="${1}"
     table="${2}"
     type="${3:-}"
@@ -119,35 +119,29 @@ scanDatabaseTable() {
 
     # Send prepared query to gravity database
     result="$(sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null
-    if [[ -n "${result}" ]]; then
-        # Prepend listname (separated by a colon) if we found at least one result
-        # and output result
-        results="$(sed "s/^/${table}:/g;" <<< "${result}")"
-    else
-        # Output empty string as the database query didn't return any result
+    if [[ -z "${result}" ]]; then
+        # Return early when we have no results
         return
     fi
-    mapfile -t results <<< "${results}"
-    if [[ -n "${results[*]}" ]]; then
-        wbMatch=true
-        # Loop through each result in order to print unique file title once
-        for result in "${results[@]}"; do
-            fileName="${result%%:*}"
-            if [[ -n "${blockpage}" ]]; then
-                echo "π ${result}"
-                exit 0
-            elif [[ -n "${exact}" ]]; then
-                echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}"
-            else
-                # Only print filename title once per file
-                if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then
-                    echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}"
-                    fileName_prev="${fileName}"
-                fi
-            echo "   ${result#*:}"
+
+    wbMatch=true
+    mapfile -t results <<< "${result}"
+    # Loop through each result
+    for result in "${results[@]}"; do
+        if [[ -n "${blockpage}" ]]; then
+            echo "π ${result}"
+            exit 0
+        elif [[ -n "${exact}" ]]; then
+            echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}"
+        else
+            # Only print table name once
+            if [[ ! "${table}" == "${table_prev:-}" ]]; then
+                echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}"
+                table_prev="${table}"
             fi
-        done
-    fi
+        echo "   ${result}"
+        fi
+    done
 }
 
 # Scan Whitelist and Blacklist

From f80fdd7e83b4bb23edb1671316f986b9ed791b68 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sat, 4 May 2019 13:19:50 +0200
Subject: [PATCH 03/27] Improve comments

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/query.sh | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index 66b55e87..a7cc9dfb 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -108,10 +108,10 @@ scanDatabaseTable() {
     table="${2}"
     type="${3:-}"
 
-    # As underscores are legitimate parts of domains, we escape possible them when using the LIKE operator.
-    # Underscores are a SQLite wildcard matching exactly one character. We obviously want to suppress this
+    # As underscores are legitimate parts of domains, we escape them when using the LIKE operator.
+    # Underscores are SQLite wildcards matching exactly one character. We obviously want to suppress this
     # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched
-    # as a literal underscore character.
+    # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores.
     case "${type}" in
         "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";;
         *       ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\'";;
@@ -120,13 +120,14 @@ scanDatabaseTable() {
     # Send prepared query to gravity database
     result="$(sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null
     if [[ -z "${result}" ]]; then
-        # Return early when we have no results
+        # Return early when there are no matches in this table
         return
     fi
 
+    # Mark domain as having been white-/blacklist matched (global variable)
     wbMatch=true
-    mapfile -t results <<< "${result}"
     # Loop through each result
+    mapfile -t results <<< "${result}"
     for result in "${results[@]}"; do
         if [[ -n "${blockpage}" ]]; then
             echo "π ${result}"

From 5246b3e49672edb7e397418160ca2c327cef0ccc Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sat, 4 May 2019 13:24:36 +0200
Subject: [PATCH 04/27] Explicitly escape backslash in ESCAPE clause. This has
 been suggested by Stickler bot.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/query.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index a7cc9dfb..d4de380c 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -114,7 +114,7 @@ scanDatabaseTable() {
     # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores.
     case "${type}" in
         "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";;
-        *       ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\'";;
+        *       ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
     esac
 
     # Send prepared query to gravity database

From a904c183dfc2de7a503466f5ff39d1af3a4d1aca Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sat, 4 May 2019 18:25:11 +0200
Subject: [PATCH 05/27] Use printf to escape domain content. This prevents
 possible SQL injection issues

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/query.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index d4de380c..d2e16e79 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -104,7 +104,7 @@ fi
 
 scanDatabaseTable() {
     local domain table type querystr result table_prev
-    domain="${1}"
+    domain="$(printf "%q" "${1}")"
     table="${2}"
     type="${3:-}"
 

From 03d93aa19a61cbe3e23dea47c3bdd9cd2c66287a Mon Sep 17 00:00:00 2001
From: Mcat12 <newtoncat12@yahoo.com>
Date: Mon, 20 May 2019 20:58:57 -0700
Subject: [PATCH 06/27] Update debug script with gravity DB changes

Signed-off-by: Mcat12 <newtoncat12@yahoo.com>
---
 advanced/Scripts/piholeDebug.sh | 67 +++++++++++++++++----------------
 1 file changed, 35 insertions(+), 32 deletions(-)

diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh
index 1010f26c..3a5c482f 100755
--- a/advanced/Scripts/piholeDebug.sh
+++ b/advanced/Scripts/piholeDebug.sh
@@ -89,16 +89,12 @@ PIHOLE_WILDCARD_CONFIG_FILE="${DNSMASQ_D_DIRECTORY}/03-wildcard.conf"
 WEB_SERVER_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/lighttpd.conf"
 #WEB_SERVER_CUSTOM_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/external.conf"
 
-PIHOLE_DEFAULT_AD_LISTS="${PIHOLE_DIRECTORY}/adlists.default"
-PIHOLE_USER_DEFINED_AD_LISTS="${PIHOLE_DIRECTORY}/adlists.list"
-PIHOLE_BLACKLIST_FILE="${PIHOLE_DIRECTORY}/blacklist.txt"
-PIHOLE_BLOCKLIST_FILE="${PIHOLE_DIRECTORY}/gravity.list"
 PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log"
 PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*"
 PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list"
 PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate"
 PIHOLE_SETUP_VARS_FILE="${PIHOLE_DIRECTORY}/setupVars.conf"
-PIHOLE_WHITELIST_FILE="${PIHOLE_DIRECTORY}/whitelist.txt"
+PIHOLE_GRAVITY_DB_FILE="${PIHOLE_DIRECTORY}/gravity.db"
 
 PIHOLE_COMMAND="${BIN_DIRECTORY}/pihole"
 PIHOLE_COLTABLE_FILE="${BIN_DIRECTORY}/COL_TABLE"
@@ -142,16 +138,11 @@ REQUIRED_FILES=("${PIHOLE_CRON_FILE}"
 "${PIHOLE_DHCP_CONFIG_FILE}"
 "${PIHOLE_WILDCARD_CONFIG_FILE}"
 "${WEB_SERVER_CONFIG_FILE}"
-"${PIHOLE_DEFAULT_AD_LISTS}"
-"${PIHOLE_USER_DEFINED_AD_LISTS}"
-"${PIHOLE_BLACKLIST_FILE}"
-"${PIHOLE_BLOCKLIST_FILE}"
 "${PIHOLE_INSTALL_LOG_FILE}"
 "${PIHOLE_RAW_BLOCKLIST_FILES}"
 "${PIHOLE_LOCAL_HOSTS_FILE}"
 "${PIHOLE_LOGROTATE_FILE}"
 "${PIHOLE_SETUP_VARS_FILE}"
-"${PIHOLE_WHITELIST_FILE}"
 "${PIHOLE_COMMAND}"
 "${PIHOLE_COLTABLE_FILE}"
 "${FTL_PID}"
@@ -793,7 +784,7 @@ dig_at() {
     # This helps emulate queries to different domains that a user might query
     # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains
     local random_url
-    random_url=$(shuf -n 1 "${PIHOLE_BLOCKLIST_FILE}")
+    random_url=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity ORDER BY RANDOM() LIMIT 1")
 
     # First, do a dig on localhost to see if Pi-hole can use itself to block a domain
     if local_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @${local_address} +short "${record_type}"); then
@@ -975,8 +966,7 @@ list_files_in_dir() {
         if [[ -d "${dir_to_parse}/${each_file}" ]]; then
             # If it's a directoy, do nothing
             :
-        elif [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_BLOCKLIST_FILE}" ]] || \
-            [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_DEBUG_LOG}" ]] || \
+        elif [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_DEBUG_LOG}" ]] || \
             [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_RAW_BLOCKLIST_FILES}" ]] || \
             [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_INSTALL_LOG_FILE}" ]] || \
             [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_SETUP_VARS_FILE}" ]] || \
@@ -1061,31 +1051,43 @@ head_tail_log() {
     IFS="$OLD_IFS"
 }
 
-analyze_gravity_list() {
-    echo_current_diagnostic "Gravity list"
-    local head_line
-    local tail_line
-    # Put the current Internal Field Separator into another variable so it can be restored later
+show_adlists() {
+    echo_current_diagnostic "Adlists"
+
     OLD_IFS="$IFS"
-    # Get the lines that are in the file(s) and store them in an array for parsing later
     IFS=$'\r\n'
+    local adlists=()
+    mapfile -t adlists < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT address FROM vw_adlists")
+
+    for line in "${adlists[@]}"; do
+        log_write "   ${line}"
+    done
+
+    IFS="$OLD_IFS"
+}
+
+analyze_gravity_list() {
+    echo_current_diagnostic "Gravity List and Database"
+
     local gravity_permissions
-    gravity_permissions=$(ls -ld "${PIHOLE_BLOCKLIST_FILE}")
+    gravity_permissions=$(ls -ld "${PIHOLE_GRAVITY_DB_FILE}")
     log_write "${COL_GREEN}${gravity_permissions}${COL_NC}"
-    local gravity_head=()
-    mapfile -t gravity_head < <(head -n 4 ${PIHOLE_BLOCKLIST_FILE})
-    log_write "   ${COL_CYAN}-----head of $(basename ${PIHOLE_BLOCKLIST_FILE})------${COL_NC}"
-    for head_line in "${gravity_head[@]}"; do
-        log_write "   ${head_line}"
+
+    local gravity_size
+    gravity_size=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT COUNT(*) FROM vw_gravity")
+    log_write "   Size: ${COL_CYAN}${gravity_size}${COL_NC} entries"
+
+    OLD_IFS="$IFS"
+    IFS=$'\r\n'
+    local gravity_sample=()
+    mapfile -t gravity_sample < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10")
+    log_write "   ${COL_CYAN}----- First 10 Domains -----${COL_NC}"
+
+    for line in "${gravity_sample[@]}"; do
+        log_write "   ${line}"
     done
+
     log_write ""
-    local gravity_tail=()
-    mapfile -t gravity_tail < <(tail -n 4 ${PIHOLE_BLOCKLIST_FILE})
-    log_write "   ${COL_CYAN}-----tail of $(basename ${PIHOLE_BLOCKLIST_FILE})------${COL_NC}"
-    for tail_line in "${gravity_tail[@]}"; do
-        log_write "   ${tail_line}"
-    done
-    # Set the IFS back to what it was
     IFS="$OLD_IFS"
 }
 
@@ -1236,6 +1238,7 @@ process_status
 parse_setup_vars
 check_x_headers
 analyze_gravity_list
+show_adlists
 show_content_of_pihole_files
 parse_locale
 analyze_pihole_log

From 3f05efd60f4e4ff5630621d3a1d50fd81e1e1807 Mon Sep 17 00:00:00 2001
From: Mcat12 <newtoncat12@yahoo.com>
Date: Mon, 20 May 2019 21:02:31 -0700
Subject: [PATCH 07/27] Add extra newline

Signed-off-by: Mcat12 <newtoncat12@yahoo.com>
---
 advanced/Scripts/piholeDebug.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh
index 3a5c482f..82660d61 100755
--- a/advanced/Scripts/piholeDebug.sh
+++ b/advanced/Scripts/piholeDebug.sh
@@ -1076,6 +1076,7 @@ analyze_gravity_list() {
     local gravity_size
     gravity_size=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT COUNT(*) FROM vw_gravity")
     log_write "   Size: ${COL_CYAN}${gravity_size}${COL_NC} entries"
+    log_write ""
 
     OLD_IFS="$IFS"
     IFS=$'\r\n'

From 807ce0af4ed7d59830c1ae26eaffb6757c320699 Mon Sep 17 00:00:00 2001
From: Mcat12 <newtoncat12@yahoo.com>
Date: Mon, 20 May 2019 21:15:22 -0700
Subject: [PATCH 08/27] Show whitelist, blacklist, and regexlist details

Signed-off-by: Mcat12 <newtoncat12@yahoo.com>
---
 advanced/Scripts/piholeDebug.sh | 32 +++++++++++++++++++++++++++-----
 1 file changed, 27 insertions(+), 5 deletions(-)

diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh
index 82660d61..c491b1c6 100755
--- a/advanced/Scripts/piholeDebug.sh
+++ b/advanced/Scripts/piholeDebug.sh
@@ -1051,21 +1051,40 @@ head_tail_log() {
     IFS="$OLD_IFS"
 }
 
-show_adlists() {
-    echo_current_diagnostic "Adlists"
+show_db_entries() {
+    local title="${1}"
+    local query="${2}"
+
+    echo_current_diagnostic "${title}"
 
     OLD_IFS="$IFS"
     IFS=$'\r\n'
-    local adlists=()
-    mapfile -t adlists < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT address FROM vw_adlists")
+    local entries=()
+    mapfile -t entries < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" -cmd ".headers on" "${query}")
 
-    for line in "${adlists[@]}"; do
+    for line in "${entries[@]}"; do
         log_write "   ${line}"
     done
 
     IFS="$OLD_IFS"
 }
 
+show_adlists() {
+    show_db_entries "Adlists" "SELECT * FROM adlists"
+}
+
+show_whitelist() {
+    show_db_entries "Whitelist" "SELECT * FROM whitelist"
+}
+
+show_blacklist() {
+    show_db_entries "Blacklist" "SELECT * FROM blacklist"
+}
+
+show_regexlist() {
+    show_db_entries "Regexlist" "SELECT * FROM regex"
+}
+
 analyze_gravity_list() {
     echo_current_diagnostic "Gravity List and Database"
 
@@ -1240,6 +1259,9 @@ parse_setup_vars
 check_x_headers
 analyze_gravity_list
 show_adlists
+show_whitelist
+show_blacklist
+show_regexlist
 show_content_of_pihole_files
 parse_locale
 analyze_pihole_log

From 7b5fc60e003263b58fc7f1ebf8ce3bc3f6b74796 Mon Sep 17 00:00:00 2001
From: Mcat12 <newtoncat12@yahoo.com>
Date: Mon, 20 May 2019 21:20:38 -0700
Subject: [PATCH 09/27] Improve table formatting

Signed-off-by: Mcat12 <newtoncat12@yahoo.com>
---
 advanced/Scripts/piholeDebug.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh
index c491b1c6..e56d1f94 100755
--- a/advanced/Scripts/piholeDebug.sh
+++ b/advanced/Scripts/piholeDebug.sh
@@ -1060,7 +1060,7 @@ show_db_entries() {
     OLD_IFS="$IFS"
     IFS=$'\r\n'
     local entries=()
-    mapfile -t entries < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" -cmd ".headers on" "${query}")
+    mapfile -t entries < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" -cmd ".headers on" -cmd ".mode column" "${query}")
 
     for line in "${entries[@]}"; do
         log_write "   ${line}"

From a3e1473ac10411e2fc72952bb2fa7983393b3b17 Mon Sep 17 00:00:00 2001
From: Mcat12 <newtoncat12@yahoo.com>
Date: Mon, 20 May 2019 21:33:09 -0700
Subject: [PATCH 10/27] Set explicit column widths to prevent text from getting
 cut off

Signed-off-by: Mcat12 <newtoncat12@yahoo.com>
---
 advanced/Scripts/piholeDebug.sh | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh
index e56d1f94..816652e0 100755
--- a/advanced/Scripts/piholeDebug.sh
+++ b/advanced/Scripts/piholeDebug.sh
@@ -1054,13 +1054,20 @@ head_tail_log() {
 show_db_entries() {
     local title="${1}"
     local query="${2}"
+    local widths="${3}"
 
     echo_current_diagnostic "${title}"
 
     OLD_IFS="$IFS"
     IFS=$'\r\n'
     local entries=()
-    mapfile -t entries < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" -cmd ".headers on" -cmd ".mode column" "${query}")
+    mapfile -t entries < <(\
+        sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" \
+            -cmd ".headers on" \
+            -cmd ".mode column" \
+            -cmd ".width ${widths}" \
+            "${query}"\
+    )
 
     for line in "${entries[@]}"; do
         log_write "   ${line}"
@@ -1070,19 +1077,19 @@ show_db_entries() {
 }
 
 show_adlists() {
-    show_db_entries "Adlists" "SELECT * FROM adlists"
+    show_db_entries "Adlists" "SELECT * FROM adlists" "2 100 7 10 13 50"
 }
 
 show_whitelist() {
-    show_db_entries "Whitelist" "SELECT * FROM whitelist"
+    show_db_entries "Whitelist" "SELECT * FROM whitelist" "2 100 7 10 13 50"
 }
 
 show_blacklist() {
-    show_db_entries "Blacklist" "SELECT * FROM blacklist"
+    show_db_entries "Blacklist" "SELECT * FROM blacklist" "2 100 7 10 13 50"
 }
 
 show_regexlist() {
-    show_db_entries "Regexlist" "SELECT * FROM regex"
+    show_db_entries "Regexlist" "SELECT * FROM regex" "2 100 7 10 13 50"
 }
 
 analyze_gravity_list() {

From 5796054305e4b6842d95f8f1259df35862a34ba4 Mon Sep 17 00:00:00 2001
From: Mcat12 <newtoncat12@yahoo.com>
Date: Mon, 20 May 2019 21:59:18 -0700
Subject: [PATCH 11/27] Increase ID column width to 4

Signed-off-by: Mcat12 <newtoncat12@yahoo.com>
---
 advanced/Scripts/piholeDebug.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh
index 816652e0..07a11ff2 100755
--- a/advanced/Scripts/piholeDebug.sh
+++ b/advanced/Scripts/piholeDebug.sh
@@ -1077,19 +1077,19 @@ show_db_entries() {
 }
 
 show_adlists() {
-    show_db_entries "Adlists" "SELECT * FROM adlists" "2 100 7 10 13 50"
+    show_db_entries "Adlists" "SELECT * FROM adlists" "4 100 7 10 13 50"
 }
 
 show_whitelist() {
-    show_db_entries "Whitelist" "SELECT * FROM whitelist" "2 100 7 10 13 50"
+    show_db_entries "Whitelist" "SELECT * FROM whitelist" "4 100 7 10 13 50"
 }
 
 show_blacklist() {
-    show_db_entries "Blacklist" "SELECT * FROM blacklist" "2 100 7 10 13 50"
+    show_db_entries "Blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50"
 }
 
 show_regexlist() {
-    show_db_entries "Regexlist" "SELECT * FROM regex" "2 100 7 10 13 50"
+    show_db_entries "Regexlist" "SELECT * FROM regex" "4 100 7 10 13 50"
 }
 
 analyze_gravity_list() {

From bfb99c361c6db92a728760ba34edb63ffca6006c Mon Sep 17 00:00:00 2001
From: Mcat12 <newtoncat12@yahoo.com>
Date: Tue, 21 May 2019 17:12:47 -0700
Subject: [PATCH 12/27] Note that the gravity size does not include the
 blacklist entries

Signed-off-by: Mcat12 <newtoncat12@yahoo.com>
---
 advanced/Scripts/piholeDebug.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh
index 07a11ff2..b31bbdc5 100755
--- a/advanced/Scripts/piholeDebug.sh
+++ b/advanced/Scripts/piholeDebug.sh
@@ -1101,7 +1101,7 @@ analyze_gravity_list() {
 
     local gravity_size
     gravity_size=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT COUNT(*) FROM vw_gravity")
-    log_write "   Size: ${COL_CYAN}${gravity_size}${COL_NC} entries"
+    log_write "   Size (excluding blacklist): ${COL_CYAN}${gravity_size}${COL_NC} entries"
     log_write ""
 
     OLD_IFS="$IFS"

From f6213d4f4dfb2aa32dabf07619d9739116308bbd Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Thu, 30 May 2019 15:26:27 +0200
Subject: [PATCH 13/27] Use last PID in case pidof returns multiple PIDs for
 pihole-FTL

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/pihole-FTL.service | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service
index 9eb183ed..7c7e533e 100644
--- a/advanced/Templates/pihole-FTL.service
+++ b/advanced/Templates/pihole-FTL.service
@@ -13,7 +13,7 @@ FTLUSER=pihole
 PIDFILE=/var/run/pihole-FTL.pid
 
 get_pid() {
-    pidof "pihole-FTL"
+    pidof "pihole-FTL" | awk '{print $(NF)}'
 }
 
 is_running() {

From c5df104a6653ae08b5d226de345a9f2cd53cf26b Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Thu, 30 May 2019 16:41:37 +0200
Subject: [PATCH 14/27] Add dhcp-ignore-names option when enabling DHCP
 service. We currently remove anything that starts with "dhcp-" to have a
 clean configuration and removed these lines without noticing when enabling
 the DHCP server.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/01-pihole.conf     | 5 -----
 advanced/Scripts/webpage.sh | 8 ++++++++
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/advanced/01-pihole.conf b/advanced/01-pihole.conf
index 40a117fe..cd74e186 100644
--- a/advanced/01-pihole.conf
+++ b/advanced/01-pihole.conf
@@ -41,8 +41,3 @@ log-facility=/var/log/pihole.log
 local-ttl=2
 
 log-async
-
-# If a DHCP client claims that its name is "wpad", ignore that.
-# This fixes a security hole. see CERT Vulnerability VU#598349
-dhcp-name-match=set:wpad-ignore,wpad
-dhcp-ignore-names=tag:wpad-ignore
diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh
index ea699efa..583579b6 100755
--- a/advanced/Scripts/webpage.sh
+++ b/advanced/Scripts/webpage.sh
@@ -366,6 +366,14 @@ EnableDHCP() {
     delete_dnsmasq_setting "dhcp-"
     delete_dnsmasq_setting "quiet-dhcp"
 
+    # If a DHCP client claims that its name is "wpad", ignore that.
+    # This fixes a security hole. see CERT Vulnerability VU#598349
+    # We also ignore "localhost" as Windows behaves strangely if a
+    # device claims this host name
+    add_dnsmasq_setting "dhcp-name-match=set:hostname-ignore,wpad
+dhcp-name-match=set:hostname-ignore,localhost
+dhcp-ignore-names=tag:hostname-ignore"
+
     ProcessDHCPSettings
 
     RestartDNS

From 5060605626950007aa0fc35153356ea49c96a187 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Thu, 30 May 2019 21:44:47 +0200
Subject: [PATCH 15/27] Print table name before entering the loop for the sake
 of simplicity

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/query.sh | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index d2e16e79..88ec2173 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -103,7 +103,7 @@ if [[ -n "${str:-}" ]]; then
 fi
 
 scanDatabaseTable() {
-    local domain table type querystr result table_prev
+    local domain table type querystr result
     domain="$(printf "%q" "${1}")"
     table="${2}"
     type="${3:-}"
@@ -126,22 +126,20 @@ scanDatabaseTable() {
 
     # Mark domain as having been white-/blacklist matched (global variable)
     wbMatch=true
-    # Loop through each result
+
+    # Print table name
+    if [[ ! -z "${result}" ]]; then
+        echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}"
+    fi
+
+    # Loop over results and print them
     mapfile -t results <<< "${result}"
     for result in "${results[@]}"; do
         if [[ -n "${blockpage}" ]]; then
             echo "π ${result}"
             exit 0
-        elif [[ -n "${exact}" ]]; then
-            echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}"
-        else
-            # Only print table name once
-            if [[ ! "${table}" == "${table_prev:-}" ]]; then
-                echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}"
-                table_prev="${table}"
-            fi
-        echo "   ${result}"
         fi
+        echo "   ${result}"
     done
 }
 

From 44f8fcb600cc78f7b4dd8c6c9c8bba73c64c718c Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 31 May 2019 08:39:18 +0200
Subject: [PATCH 16/27] We can print the table name without any extra check as
 we already returned early in case there are no results.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/query.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index 88ec2173..9cb298df 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -128,9 +128,7 @@ scanDatabaseTable() {
     wbMatch=true
 
     # Print table name
-    if [[ ! -z "${result}" ]]; then
-        echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}"
-    fi
+    echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}"
 
     # Loop over results and print them
     mapfile -t results <<< "${result}"

From ae98fde32141154e7384bf38a3893ecf162d94ca Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 31 May 2019 09:18:12 +0200
Subject: [PATCH 17/27] Try to obtain PID from PIDFILE. If this fails (file
 does not exist or is empty), fall back to using pidof + awk

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/pihole-FTL.service | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service
index 7c7e533e..8a4c7ce6 100644
--- a/advanced/Templates/pihole-FTL.service
+++ b/advanced/Templates/pihole-FTL.service
@@ -13,6 +13,13 @@ FTLUSER=pihole
 PIDFILE=/var/run/pihole-FTL.pid
 
 get_pid() {
+    # First, try to obtain PID from PIDFILE
+    if [ -s "${PIDFILE}" ]; then
+        cat "${PIDFILE}"
+        return
+    fi
+
+    # If the PIDFILE is empty or not available, obtain the PID using pidof
     pidof "pihole-FTL" | awk '{print $(NF)}'
 }
 

From 54bfaa28c1ac41fc8839c935003a1900320195c2 Mon Sep 17 00:00:00 2001
From: MMotti <matthew.w.motti@gmail.com>
Date: Mon, 3 Jun 2019 19:23:27 +0100
Subject: [PATCH 18/27] Fix for regexp queries through pihole -q

Taking inspiration from: https://github.com/pi-hole/pi-hole/pull/2346

We will use awk to iterate through each regexp and print any that match the domain query.

Signed-off-by: MMotti <matthew.w.motti@gmail.com>
---
 advanced/Scripts/query.sh | 64 +++++++++++++++------------------------
 1 file changed, 25 insertions(+), 39 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index 9cb298df..840c03da 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -12,7 +12,7 @@
 # Globals
 piholeDir="/etc/pihole"
 gravityDBfile="${piholeDir}/gravity.db"
-wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf"
+regexlist="/etc/pihole/regex.list"
 options="$*"
 adlist=""
 all=""
@@ -23,27 +23,11 @@ matchType="match"
 colfile="/opt/pihole/COL_TABLE"
 source "${colfile}"
 
-# Print each subdomain
-# e.g: foo.bar.baz.com = "foo.bar.baz.com bar.baz.com baz.com com"
-processWildcards() {
-    IFS="." read -r -a array <<< "${1}"
-    for (( i=${#array[@]}-1; i>=0; i-- )); do
-        ar=""
-        for (( j=${#array[@]}-1; j>${#array[@]}-i-2; j-- )); do
-            if [[ $j == $((${#array[@]}-1)) ]]; then
-                ar="${array[$j]}"
-            else
-                ar="${array[$j]}.${ar}"
-            fi
-        done
-        echo "${ar}"
-    done
-}
-
+# Scan an array of files for matching strings
 # Scan an array of files for matching strings
 scanList(){
     # Escape full stops
-    local domain="${1//./\\.}" lists="${2}" type="${3:-}"
+    local domain="${1}" esc_domain="${1//./\\.}" lists="${2}" type="${3:-}"
 
     # Prevent grep from printing file path
     cd "$piholeDir" || exit 1
@@ -54,9 +38,9 @@ scanList(){
     # /dev/null forces filename to be printed when only one list has been generated
     # shellcheck disable=SC2086
     case "${type}" in
-        "exact" ) grep -i -E -l "(^|(?<!#)\\s)${domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
-        "wc"    ) grep -i -o -m 1 "/${domain}/" ${lists} 2>/dev/null;;
-        *       ) grep -i "${domain}" ${lists} /dev/null 2>/dev/null;;
+		"exact" ) grep -i -E -l "(^|(?<!#)\\s)${esc_domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
+		"rx"    ) awk 'NR==FNR{regexps[$0]}{for (r in regexps)if($0 ~ r)print r}' ${lists} <(echo "$domain") 2>/dev/null;;
+		*       ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;;
     esac
 }
 
@@ -145,24 +129,26 @@ scanDatabaseTable() {
 scanDatabaseTable "${domainQuery}" "whitelist" "${exact}"
 scanDatabaseTable "${domainQuery}" "blacklist" "${exact}"
 
-# Scan Wildcards
-if [[ -e "${wildcardlist}" ]]; then
-    # Determine all subdomains, domain and TLDs
-    mapfile -t wildcards <<< "$(processWildcards "${domainQuery}")"
-    for match in "${wildcards[@]}"; do
-        # Search wildcard list for matches
-        mapfile -t results <<< "$(scanList "${match}" "${wildcardlist}" "wc")"
-        if [[ -n "${results[*]}" ]]; then
-            if [[ -z "${wcMatch:-}" ]] && [[ -z "${blockpage}" ]]; then
-                wcMatch=true
-                echo " ${matchType^} found in ${COL_BOLD}Wildcards${COL_NC}:"
-            fi
-            case "${blockpage}" in
-                true ) echo "π ${wildcardlist##*/}"; exit 0;;
-                *    ) echo "   *.${match}";;
-            esac
+# Scan Regex
+if [[ -e "${regexlist}" ]]; then
+    # Return portion(s) of string that is found in the regex list
+    mapfile -t results <<< "$(scanList "${domainQuery}" "${regexlist}" "rx")"
+
+    if [[ -n "${results[*]}" ]]; then
+		# A result is found
+		str="Phrase ${matchType}ed within ${COL_BOLD}regex list${COL_NC}"
+		result="${COL_BOLD}$(printf '%s\n' ${results[*]})${COL_NC}"
+
+        if [[ -z "${blockpage}" ]]; then
+            wcMatch=true
+            echo " $str"
         fi
-    done
+
+        case "${blockpage}" in
+            true ) echo "π ${regexlist##*/}"; exit 0;;
+            *    ) awk '{print "   "$0}' <<< "${result}";;
+        esac
+    fi
 fi
 
 # Get version sorted *.domains filenames (without dir path)

From 97df6d7415bfc2daf10b401a8c8712a47af0ab2d Mon Sep 17 00:00:00 2001
From: MMotti <matthew.w.motti@gmail.com>
Date: Mon, 3 Jun 2019 19:55:29 +0100
Subject: [PATCH 19/27] Stickler fix

Signed-off-by: MMotti <matthew.w.motti@gmail.com>
---
 advanced/Scripts/query.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index 840c03da..827f97a3 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -137,7 +137,7 @@ if [[ -e "${regexlist}" ]]; then
     if [[ -n "${results[*]}" ]]; then
 		# A result is found
 		str="Phrase ${matchType}ed within ${COL_BOLD}regex list${COL_NC}"
-		result="${COL_BOLD}$(printf '%s\n' ${results[*]})${COL_NC}"
+		result="${COL_BOLD}$(IFS=$'\n'; echo "${results[*]}")${COL_NC}"
 
         if [[ -z "${blockpage}" ]]; then
             wcMatch=true

From 09532638d5d6cbbf82aa44ec0e04a67e64402905 Mon Sep 17 00:00:00 2001
From: MMotti <matthew.w.motti@gmail.com>
Date: Mon, 3 Jun 2019 23:59:58 +0100
Subject: [PATCH 20/27] Read from DB instead of regex.list

Signed-off-by: MMotti <matthew.w.motti@gmail.com>
---
 advanced/Scripts/query.sh | 31 +++++++++++++++++++++++--------
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index 827f97a3..a4cf02b1 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -39,7 +39,7 @@ scanList(){
     # shellcheck disable=SC2086
     case "${type}" in
 		"exact" ) grep -i -E -l "(^|(?<!#)\\s)${esc_domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
-		"rx"    ) awk 'NR==FNR{regexps[$0]}{for (r in regexps)if($0 ~ r)print r}' ${lists} <(echo "$domain") 2>/dev/null;;
+		"rx"    ) awk 'NR==FNR{regexps[$0]}{for (r in regexps)if($0 ~ r)print r}' <(echo "$lists") <(echo "$domain") 2>/dev/null;;
 		*       ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;;
     esac
 }
@@ -97,8 +97,9 @@ scanDatabaseTable() {
     # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched
     # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores.
     case "${type}" in
-        "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";;
-        *       ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
+		"exact"         ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";;
+		"retrievetable" ) querystr="SELECT domain FROM vw_${table}";;
+		*               ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
     esac
 
     # Send prepared query to gravity database
@@ -108,6 +109,13 @@ scanDatabaseTable() {
         return
     fi
 
+	# If we are only retrieving the table
+	# Just output and return
+	if [[ "${type}" == "retrievetable" ]]; then
+		echo "${result[*]}"
+		return
+	fi
+
     # Mark domain as having been white-/blacklist matched (global variable)
     wbMatch=true
 
@@ -129,14 +137,21 @@ scanDatabaseTable() {
 scanDatabaseTable "${domainQuery}" "whitelist" "${exact}"
 scanDatabaseTable "${domainQuery}" "blacklist" "${exact}"
 
-# Scan Regex
-if [[ -e "${regexlist}" ]]; then
+# Scan Regex table
+regexlist=$(scanDatabaseTable "" "regex" "retrievetable")
+
+if [[ -n "${regexlist}" ]]; then
     # Return portion(s) of string that is found in the regex list
     mapfile -t results <<< "$(scanList "${domainQuery}" "${regexlist}" "rx")"
 
-    if [[ -n "${results[*]}" ]]; then
-		# A result is found
-		str="Phrase ${matchType}ed within ${COL_BOLD}regex list${COL_NC}"
+	# If a result is found
+	if [[ -n "${results[*]}" ]]; then
+		# Count the matches
+		regexCount=${#results[@]}
+		# Determine plural string
+		[[ $regexCount -gt 1 ]] && plu="es"
+		# Form output strings
+		str="${COL_BOLD}${regexCount}${COL_NC} ${matchType}${plu:-} found in ${COL_BOLD}regex${COL_NC} table"
 		result="${COL_BOLD}$(IFS=$'\n'; echo "${results[*]}")${COL_NC}"
 
         if [[ -z "${blockpage}" ]]; then

From b49c702f331c5bfecc2b4622039740f8b32aa247 Mon Sep 17 00:00:00 2001
From: MMotti <matthew.w.motti@gmail.com>
Date: Tue, 4 Jun 2019 00:03:37 +0100
Subject: [PATCH 21/27] Consistency tweak

Signed-off-by: MMotti <matthew.w.motti@gmail.com>
---
 advanced/Scripts/query.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index a4cf02b1..f4aed05b 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -39,7 +39,7 @@ scanList(){
     # shellcheck disable=SC2086
     case "${type}" in
 		"exact" ) grep -i -E -l "(^|(?<!#)\\s)${esc_domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
-		"rx"    ) awk 'NR==FNR{regexps[$0]}{for (r in regexps)if($0 ~ r)print r}' <(echo "$lists") <(echo "$domain") 2>/dev/null;;
+		"rx"    ) awk 'NR==FNR{regexps[$0]}{for (r in regexps)if($0 ~ r)print r}' <(echo "${lists}") <(echo "${domain}") 2>/dev/null;;
 		*       ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;;
     esac
 }

From 7613e94ef6f987d12203dc13e26b5637a5e31d48 Mon Sep 17 00:00:00 2001
From: MMotti <matthew.w.motti@gmail.com>
Date: Tue, 4 Jun 2019 06:06:17 +0100
Subject: [PATCH 22/27] Minor tweaks Mainly for consistency

Signed-off-by: MMotti <matthew.w.motti@gmail.com>
---
 advanced/Scripts/query.sh | 46 +++++++++++++++++----------------------
 1 file changed, 20 insertions(+), 26 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index f4aed05b..5b8baa63 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -12,7 +12,6 @@
 # Globals
 piholeDir="/etc/pihole"
 gravityDBfile="${piholeDir}/gravity.db"
-regexlist="/etc/pihole/regex.list"
 options="$*"
 adlist=""
 all=""
@@ -23,7 +22,6 @@ matchType="match"
 colfile="/opt/pihole/COL_TABLE"
 source "${colfile}"
 
-# Scan an array of files for matching strings
 # Scan an array of files for matching strings
 scanList(){
     # Escape full stops
@@ -39,7 +37,12 @@ scanList(){
     # shellcheck disable=SC2086
     case "${type}" in
 		"exact" ) grep -i -E -l "(^|(?<!#)\\s)${esc_domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
-		"rx"    ) awk 'NR==FNR{regexps[$0]}{for (r in regexps)if($0 ~ r)print r}' <(echo "${lists}") <(echo "${domain}") 2>/dev/null;;
+		# Create array of regexps
+		# Iterate through each regexp and check whether it matches the domainQuery
+		# If it does, print the matching regexp and continue looping
+		# Input 1 - regexps | Input 2 - domainQuery
+		"regex" ) awk 'NR==FNR{regexps[$0]}{for (r in regexps)if($0 ~ r)print r}' \
+					<(echo "${lists}") <(echo "${domain}") 2>/dev/null;;
 		*       ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;;
     esac
 }
@@ -97,9 +100,8 @@ scanDatabaseTable() {
     # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched
     # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores.
     case "${type}" in
-		"exact"         ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";;
-		"retrievetable" ) querystr="SELECT domain FROM vw_${table}";;
-		*               ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
+		"exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";;
+		*       ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
     esac
 
     # Send prepared query to gravity database
@@ -109,13 +111,6 @@ scanDatabaseTable() {
         return
     fi
 
-	# If we are only retrieving the table
-	# Just output and return
-	if [[ "${type}" == "retrievetable" ]]; then
-		echo "${result[*]}"
-		return
-	fi
-
     # Mark domain as having been white-/blacklist matched (global variable)
     wbMatch=true
 
@@ -138,20 +133,19 @@ scanDatabaseTable "${domainQuery}" "whitelist" "${exact}"
 scanDatabaseTable "${domainQuery}" "blacklist" "${exact}"
 
 # Scan Regex table
-regexlist=$(scanDatabaseTable "" "regex" "retrievetable")
-
-if [[ -n "${regexlist}" ]]; then
-    # Return portion(s) of string that is found in the regex list
-    mapfile -t results <<< "$(scanList "${domainQuery}" "${regexlist}" "rx")"
-
-	# If a result is found
+mapfile -t regexlist <<< "$(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex" 2> /dev/null)"
+# Split results over new line and store in a string
+# ready for processing
+str_regexlist=$(IFS=$'\n'; echo "${regexlist[*]}")
+# If there are regexps in the DB
+if [[ -n "${str_regexlist}" ]]; then
+    # Return any regexps that match the domainQuery
+    mapfile -t results <<< "$(scanList "${domainQuery}" "${str_regexlist}" "regex")"
+
+	# If there are matches to the domain query
 	if [[ -n "${results[*]}" ]]; then
-		# Count the matches
-		regexCount=${#results[@]}
-		# Determine plural string
-		[[ $regexCount -gt 1 ]] && plu="es"
 		# Form output strings
-		str="${COL_BOLD}${regexCount}${COL_NC} ${matchType}${plu:-} found in ${COL_BOLD}regex${COL_NC} table"
+		str="${matchType^} found in ${COL_BOLD}regex list${COL_NC}"
 		result="${COL_BOLD}$(IFS=$'\n'; echo "${results[*]}")${COL_NC}"
 
         if [[ -z "${blockpage}" ]]; then
@@ -160,7 +154,7 @@ if [[ -n "${regexlist}" ]]; then
         fi
 
         case "${blockpage}" in
-            true ) echo "π ${regexlist##*/}"; exit 0;;
+            true ) echo "π regex list"; exit 0;;
             *    ) awk '{print "   "$0}' <<< "${result}";;
         esac
     fi

From cf21efa10350cfbb257d0428c6be08594e8139ae Mon Sep 17 00:00:00 2001
From: MMotti <matthew.w.motti@gmail.com>
Date: Wed, 5 Jun 2019 14:36:43 +0100
Subject: [PATCH 23/27] Minor grammar changes

Signed-off-by: MMotti <matthew.w.motti@gmail.com>
---
 advanced/Scripts/query.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index 5b8baa63..9134dc0f 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -145,7 +145,7 @@ if [[ -n "${str_regexlist}" ]]; then
 	# If there are matches to the domain query
 	if [[ -n "${results[*]}" ]]; then
 		# Form output strings
-		str="${matchType^} found in ${COL_BOLD}regex list${COL_NC}"
+		str="${matchType^} found in ${COL_BOLD}Regex list${COL_NC}"
 		result="${COL_BOLD}$(IFS=$'\n'; echo "${results[*]}")${COL_NC}"
 
         if [[ -z "${blockpage}" ]]; then
@@ -154,7 +154,7 @@ if [[ -n "${str_regexlist}" ]]; then
         fi
 
         case "${blockpage}" in
-            true ) echo "π regex list"; exit 0;;
+            true ) echo "π Regex list"; exit 0;;
             *    ) awk '{print "   "$0}' <<< "${result}";;
         esac
     fi

From 785f1fedd9ade421aa923b997fe873dff171f942 Mon Sep 17 00:00:00 2001
From: MMotti <matthew.w.motti@gmail.com>
Date: Mon, 10 Jun 2019 17:48:52 +0100
Subject: [PATCH 24/27] Tidy regexp queries

Signed-off-by: MMotti <matthew.w.motti@gmail.com>
---
 advanced/Scripts/query.sh | 55 +++++++++++++++++++++------------------
 1 file changed, 30 insertions(+), 25 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index 9134dc0f..e418eecf 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -133,31 +133,36 @@ scanDatabaseTable "${domainQuery}" "whitelist" "${exact}"
 scanDatabaseTable "${domainQuery}" "blacklist" "${exact}"
 
 # Scan Regex table
-mapfile -t regexlist <<< "$(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex" 2> /dev/null)"
-# Split results over new line and store in a string
-# ready for processing
-str_regexlist=$(IFS=$'\n'; echo "${regexlist[*]}")
-# If there are regexps in the DB
-if [[ -n "${str_regexlist}" ]]; then
-    # Return any regexps that match the domainQuery
-    mapfile -t results <<< "$(scanList "${domainQuery}" "${str_regexlist}" "regex")"
-
-	# If there are matches to the domain query
-	if [[ -n "${results[*]}" ]]; then
-		# Form output strings
-		str="${matchType^} found in ${COL_BOLD}Regex list${COL_NC}"
-		result="${COL_BOLD}$(IFS=$'\n'; echo "${results[*]}")${COL_NC}"
-
-        if [[ -z "${blockpage}" ]]; then
-            wcMatch=true
-            echo " $str"
-        fi
-
-        case "${blockpage}" in
-            true ) echo "π Regex list"; exit 0;;
-            *    ) awk '{print "   "$0}' <<< "${result}";;
-        esac
-    fi
+mapfile -t regexlist < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex" 2> /dev/null)
+
+# If we have regexps to process
+if [[ "${#regexlist[@]}" -ne 0 ]]; then
+	# Split regexps over a new line
+	str_regexlist=$(printf '%s\n' "${regexlist[@]}")
+	# Check domainQuery against regexps
+	mapfile -t regexMatches < <(scanList "${domainQuery}" "${str_regexlist}" "regex")
+	# If there were regex matches
+	if [[  "${#regexMatches[@]}" -ne 0 ]]; then
+		# Split matching regexps over a new line
+		str_regexMatches=$(printf '%s\n' "${regexMatches[@]}")
+		# Form a "matched" message
+		str_message="${matchType^} found in ${COL_BOLD}Regex list${COL_NC}"
+		# Form a "results" message
+		str_result="${COL_BOLD}${str_regexMatches}${COL_NC}"
+		
+		if [[ -z "${blockpage}" ]]; then
+			# Set the wildcard match flag
+			wcMatch=true
+			# Echo the "matched" message, indented by one space
+			echo " ${str_message}"
+			# Echo the "results" message, each line indented by three spaces
+			echo "${str_result}" | sed 's/^/   /'
+		else
+			echo "π Regex list"
+			exit 0
+		fi
+		
+	fi
 fi
 
 # Get version sorted *.domains filenames (without dir path)

From 10fbed50f3e3213a5a1e836cecd144cc59562107 Mon Sep 17 00:00:00 2001
From: MMotti <matthew.w.motti@gmail.com>
Date: Mon, 10 Jun 2019 18:10:25 +0100
Subject: [PATCH 25/27] Shellcheck Suggesting to use parameter expansion but we
 need to be able to identify the start of each line

Signed-off-by: MMotti <matthew.w.motti@gmail.com>
---
 advanced/Scripts/query.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index e418eecf..93d8baee 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -156,6 +156,7 @@ if [[ "${#regexlist[@]}" -ne 0 ]]; then
 			# Echo the "matched" message, indented by one space
 			echo " ${str_message}"
 			# Echo the "results" message, each line indented by three spaces
+			# shellcheck disable=SC2001
 			echo "${str_result}" | sed 's/^/   /'
 		else
 			echo "π Regex list"

From a9d0690f4dff7b081f22146cad1440eff04082a1 Mon Sep 17 00:00:00 2001
From: MMotti <matthew.w.motti@gmail.com>
Date: Mon, 10 Jun 2019 18:50:52 +0100
Subject: [PATCH 26/27] Change the case of some variables Just for consistency

Signed-off-by: MMotti <matthew.w.motti@gmail.com>
---
 advanced/Scripts/query.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index 93d8baee..aa23ab41 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -133,14 +133,14 @@ scanDatabaseTable "${domainQuery}" "whitelist" "${exact}"
 scanDatabaseTable "${domainQuery}" "blacklist" "${exact}"
 
 # Scan Regex table
-mapfile -t regexlist < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex" 2> /dev/null)
+mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex" 2> /dev/null)
 
 # If we have regexps to process
-if [[ "${#regexlist[@]}" -ne 0 ]]; then
+if [[ "${#regexList[@]}" -ne 0 ]]; then
 	# Split regexps over a new line
-	str_regexlist=$(printf '%s\n' "${regexlist[@]}")
+	str_regexList=$(printf '%s\n' "${regexList[@]}")
 	# Check domainQuery against regexps
-	mapfile -t regexMatches < <(scanList "${domainQuery}" "${str_regexlist}" "regex")
+	mapfile -t regexMatches < <(scanList "${domainQuery}" "${str_regexList}" "regex")
 	# If there were regex matches
 	if [[  "${#regexMatches[@]}" -ne 0 ]]; then
 		# Split matching regexps over a new line

From bcf03647efd8b1685bf077bfab968c327381de2a Mon Sep 17 00:00:00 2001
From: MMotti <matthew.w.motti@gmail.com>
Date: Wed, 12 Jun 2019 16:02:59 +0100
Subject: [PATCH 27/27] Add comment and remove blankspace

Signed-off-by: MMotti <matthew.w.motti@gmail.com>
---
 advanced/Scripts/query.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index aa23ab41..42ea4395 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -149,7 +149,7 @@ if [[ "${#regexList[@]}" -ne 0 ]]; then
 		str_message="${matchType^} found in ${COL_BOLD}Regex list${COL_NC}"
 		# Form a "results" message
 		str_result="${COL_BOLD}${str_regexMatches}${COL_NC}"
-		
+		# If we are displaying more than just the source of the block
 		if [[ -z "${blockpage}" ]]; then
 			# Set the wildcard match flag
 			wcMatch=true
@@ -162,7 +162,6 @@ if [[ "${#regexList[@]}" -ne 0 ]]; then
 			echo "π Regex list"
 			exit 0
 		fi
-		
 	fi
 fi