From 922ce7359c4774122c6e47f303ac8a4358afdf56 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 12 Dec 2019 10:58:41 +0000 Subject: [PATCH 1/5] pihole -q should also scan gravity table Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 78 ++++++++++++++------------------------- 1 file changed, 27 insertions(+), 51 deletions(-) mode change 100644 => 100755 advanced/Scripts/query.sh diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh old mode 100644 new mode 100755 index 1e1b159c..467fe6f4 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -53,7 +53,6 @@ Example: 'pihole -q -exact domain.com' Query the adlists for a specified domain Options: - -adlist Print the name of the block list URL -exact Search the block lists for exact domain matches -all Return all query matches within a block list -h, --help Show this help dialog" @@ -64,7 +63,6 @@ fi if [[ "${options}" == *"-bp"* ]]; then exact="exact"; blockpage=true else - [[ "${options}" == *"-adlist"* ]] && adlist=true [[ "${options}" == *"-all"* ]] && all=true if [[ "${options}" == *"-exact"* ]]; then exact="exact"; matchType="exact ${matchType}" @@ -99,10 +97,17 @@ scanDatabaseTable() { # Underscores are SQLite wildcards matching exactly one character. We obviously want to suppress this # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores. - case "${type}" in - "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";; - * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; - esac + if [[ "${table}" == "gravity" ]]; then + case "${type}" in + "exact" ) querystr="SELECT gravity.domain,adlist.address FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain = '${domain}'";; + * ) querystr="SELECT gravity.domain,adlist.address FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; + esac + else + case "${type}" in + "exact" ) querystr="SELECT domain FROM ${table} WHERE domain = '${domain}'";; + * ) querystr="SELECT domain FROM ${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; + esac + fi # Send prepared query to gravity database result="$(sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null @@ -111,6 +116,11 @@ scanDatabaseTable() { return fi + if [[ "${table}" == "gravity" ]]; then + echo "${result}" + return + fi + # Mark domain as having been white-/blacklist matched (global variable) wbMatch=true @@ -170,32 +180,15 @@ scanRegexDatabaseTable() { } # Scan Whitelist and Blacklist -scanDatabaseTable "${domainQuery}" "whitelist" "${exact}" -scanDatabaseTable "${domainQuery}" "blacklist" "${exact}" +scanDatabaseTable "${domainQuery}" "vw_whitelist" "${exact}" +scanDatabaseTable "${domainQuery}" "vw_blacklist" "${exact}" # Scan Regex table scanRegexDatabaseTable "${domainQuery}" "whitelist" scanRegexDatabaseTable "${domainQuery}" "blacklist" -# Get version sorted *.domains filenames (without dir path) -lists=("$(cd "$piholeDir" || exit 0; printf "%s\\n" -- *.domains | sort -V)") - -# Query blocklists for occurences of domain -mapfile -t results <<< "$(scanList "${domainQuery}" "${lists[*]}" "${exact}")" - -# Remove unwanted content from $results -# Each line in $results is formatted as such: [fileName]:[line] -# 1. Delete lines starting with # -# 2. Remove comments after domain -# 3. Remove hosts format IP address -# 4. Remove any lines that no longer contain the queried domain name (in case the matched domain name was in a comment) -esc_domain="${domainQuery//./\\.}" -mapfile -t results <<< "$(IFS=$'\n'; sed \ - -e "/:#/d" \ - -e "s/[ \\t]#.*//g" \ - -e "s/:.*[ \\t]/:/g" \ - -e "/${esc_domain}/!d" \ - <<< "${results[*]}")" +# Query block lists +mapfile -t results <<< "$(scanDatabaseTable "${domainQuery}" "gravity" "${exact}")" # Handle notices if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then @@ -210,12 +203,6 @@ elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 100 ]]; then exit 0 fi -# Get adlist file content as array -if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then - # Retrieve source URLs from gravity database - mapfile -t adlists <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" -fi - # Print "Exact matches for" title if [[ -n "${exact}" ]] && [[ -z "${blockpage}" ]]; then plural=""; [[ "${#results[*]}" -gt 1 ]] && plural="es" @@ -223,28 +210,17 @@ if [[ -n "${exact}" ]] && [[ -z "${blockpage}" ]]; then fi for result in "${results[@]}"; do - fileName="${result/:*/}" - - # Determine *.domains URL using filename's number - if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then - fileNum="${fileName/list./}"; fileNum="${fileNum%%.*}" - fileName="${adlists[$fileNum]}" - - # Discrepency occurs when adlists has been modified, but Gravity has not been run - if [[ -z "${fileName}" ]]; then - fileName="${COL_LIGHT_RED}(no associated adlists URL found)${COL_NC}" - fi - fi + adlistAddress="${result/*|/}" if [[ -n "${blockpage}" ]]; then - echo "${fileNum} ${fileName}" + echo "${fileNum} ${adlistAddress}" elif [[ -n "${exact}" ]]; then - echo " ${fileName}" + echo " - ${adlistAddress}" else - if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then + if [[ ! "${adlistAddress}" == "${adlistAddress_prev:-}" ]]; then count="" - echo " ${matchType^} found in ${COL_BOLD}${fileName}${COL_NC}:" - fileName_prev="${fileName}" + echo " ${matchType^} found in ${COL_BOLD}${adlistAddress}${COL_NC}:" + adlistAddress_prev="${adlistAddress}" fi : $((count++)) @@ -254,7 +230,7 @@ for result in "${results[@]}"; do [[ "${count}" -gt "${max_count}" ]] && continue echo " ${COL_GRAY}Over ${count} results found, skipping rest of file${COL_NC}" else - echo " ${result#*:}" + echo " ${result/*|//}" fi fi done From 52dd72dfa5f497d7c09d37b834aa0164edb66121 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 12 Dec 2019 11:08:19 +0000 Subject: [PATCH 2/5] Ensure output is always correct and also display if domain has been found but is disabled Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 467fe6f4..aaaf811c 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -99,8 +99,8 @@ scanDatabaseTable() { # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores. if [[ "${table}" == "gravity" ]]; then case "${type}" in - "exact" ) querystr="SELECT gravity.domain,adlist.address FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain = '${domain}'";; - * ) querystr="SELECT gravity.domain,adlist.address FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; + "exact" ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain = '${domain}'";; + * ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; esac else case "${type}" in @@ -210,12 +210,20 @@ if [[ -n "${exact}" ]] && [[ -z "${blockpage}" ]]; then fi for result in "${results[@]}"; do - adlistAddress="${result/*|/}" + match="${result/|*/}" + extra="${result#*|}" + adlistAddress="${extra/|*/}" + enabled="${extra#*|}" + if [[ "${enabled}" == "0" ]]; then + enabled="(disabled)" + else + enabled="" + fi if [[ -n "${blockpage}" ]]; then echo "${fileNum} ${adlistAddress}" elif [[ -n "${exact}" ]]; then - echo " - ${adlistAddress}" + echo " - ${adlistAddress} ${enabled}" else if [[ ! "${adlistAddress}" == "${adlistAddress_prev:-}" ]]; then count="" @@ -230,7 +238,7 @@ for result in "${results[@]}"; do [[ "${count}" -gt "${max_count}" ]] && continue echo " ${COL_GRAY}Over ${count} results found, skipping rest of file${COL_NC}" else - echo " ${result/*|//}" + echo " ${match} ${enabled}" fi fi done From 40e8657137ede303614b88c2b39a487b17eb7d6e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 12 Dec 2019 11:18:46 +0000 Subject: [PATCH 3/5] Please Mr. Stickler Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index aaaf811c..8f1c8a86 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -221,7 +221,7 @@ for result in "${results[@]}"; do fi if [[ -n "${blockpage}" ]]; then - echo "${fileNum} ${adlistAddress}" + echo "0 ${adlistAddress}" elif [[ -n "${exact}" ]]; then echo " - ${adlistAddress} ${enabled}" else From 4be7ebe61f50516d4ab5ac845700afafbe99da61 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 15 Dec 2019 11:46:14 +0000 Subject: [PATCH 4/5] Scan domainlist instead of view to also catch disabled domains. Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 47 ++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 8f1c8a86..0ba9ae40 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -88,7 +88,7 @@ if [[ -n "${str:-}" ]]; then fi scanDatabaseTable() { - local domain table type querystr result + local domain table type querystr result extra domain="$(printf "%q" "${1}")" table="${2}" type="${3:-}" @@ -98,14 +98,14 @@ scanDatabaseTable() { # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores. if [[ "${table}" == "gravity" ]]; then - case "${type}" in + case "${exact}" in "exact" ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain = '${domain}'";; * ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; esac else - case "${type}" in - "exact" ) querystr="SELECT domain FROM ${table} WHERE domain = '${domain}'";; - * ) querystr="SELECT domain FROM ${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; + case "${exact}" in + "exact" ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${type}' AND domain = '${domain}'";; + * ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${type}' AND domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; esac fi @@ -126,7 +126,7 @@ scanDatabaseTable() { # Print table name if [[ -z "${blockpage}" ]]; then - echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}" + echo " ${matchType^} found in ${COL_BOLD}exact ${table}${COL_NC}" fi # Loop over results and print them @@ -136,7 +136,13 @@ scanDatabaseTable() { echo "π ${result}" exit 0 fi - echo " ${result}" + domain="${result/|*}" + if [[ "${result#*|}" == "0" ]]; then + extra=" (disabled)" + else + extra="" + fi + echo " ${domain}${extra}" done } @@ -144,9 +150,10 @@ scanRegexDatabaseTable() { local domain list domain="${1}" list="${2}" + type="${3:-}" # Query all regex from the corresponding database tables - mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex_${list}" 2> /dev/null) + mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM domainlist WHERE type = "${type} 2> /dev/null) # If we have regexps to process if [[ "${#regexList[@]}" -ne 0 ]]; then @@ -159,7 +166,7 @@ scanRegexDatabaseTable() { # Split matching regexps over a new line str_regexMatches=$(printf '%s\n' "${regexMatches[@]}") # Form a "matched" message - str_message="${matchType^} found in ${COL_BOLD}Regex ${list}${COL_NC}" + str_message="${matchType^} found in ${COL_BOLD}regex ${list}${COL_NC}" # Form a "results" message str_result="${COL_BOLD}${str_regexMatches}${COL_NC}" # If we are displaying more than just the source of the block @@ -180,15 +187,15 @@ scanRegexDatabaseTable() { } # Scan Whitelist and Blacklist -scanDatabaseTable "${domainQuery}" "vw_whitelist" "${exact}" -scanDatabaseTable "${domainQuery}" "vw_blacklist" "${exact}" +scanDatabaseTable "${domainQuery}" "whitelist" "0" +scanDatabaseTable "${domainQuery}" "blacklist" "1" # Scan Regex table -scanRegexDatabaseTable "${domainQuery}" "whitelist" -scanRegexDatabaseTable "${domainQuery}" "blacklist" +scanRegexDatabaseTable "${domainQuery}" "whitelist" "2" +scanRegexDatabaseTable "${domainQuery}" "blacklist" "3" # Query block lists -mapfile -t results <<< "$(scanDatabaseTable "${domainQuery}" "gravity" "${exact}")" +mapfile -t results <<< "$(scanDatabaseTable "${domainQuery}" "gravity")" # Handle notices if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then @@ -213,17 +220,17 @@ for result in "${results[@]}"; do match="${result/|*/}" extra="${result#*|}" adlistAddress="${extra/|*/}" - enabled="${extra#*|}" - if [[ "${enabled}" == "0" ]]; then - enabled="(disabled)" + extra="${extra#*|}" + if [[ "${extra}" == "0" ]]; then + extra="(disabled)" else - enabled="" + extra="" fi if [[ -n "${blockpage}" ]]; then echo "0 ${adlistAddress}" elif [[ -n "${exact}" ]]; then - echo " - ${adlistAddress} ${enabled}" + echo " - ${adlistAddress} ${extra}" else if [[ ! "${adlistAddress}" == "${adlistAddress_prev:-}" ]]; then count="" @@ -238,7 +245,7 @@ for result in "${results[@]}"; do [[ "${count}" -gt "${max_count}" ]] && continue echo " ${COL_GRAY}Over ${count} results found, skipping rest of file${COL_NC}" else - echo " ${match} ${enabled}" + echo " ${match} ${extra}" fi fi done From 2444296348d6ea1b99a01160ba5f6d3e2670c013 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 15 Dec 2019 11:55:19 +0000 Subject: [PATCH 5/5] Again, Mr. Stickler Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 0ba9ae40..a96129e0 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -13,7 +13,6 @@ piholeDir="/etc/pihole" gravityDBfile="${piholeDir}/gravity.db" options="$*" -adlist="" all="" exact="" blockpage="" @@ -153,7 +152,7 @@ scanRegexDatabaseTable() { type="${3:-}" # Query all regex from the corresponding database tables - mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM domainlist WHERE type = "${type} 2> /dev/null) + mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM domainlist WHERE type = ${type}" 2> /dev/null) # If we have regexps to process if [[ "${#regexList[@]}" -ne 0 ]]; then