From d61fd01d61d95d86f79d1d041319adfc46c59c61 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Wed, 13 Jun 2018 15:47:08 +1000 Subject: [PATCH 1/9] Split queryFunc() into query.sh Signed-off-by: Rob Gill --- advanced/query.sh | 220 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 advanced/query.sh diff --git a/advanced/query.sh b/advanced/query.sh new file mode 100644 index 00000000..3bae7422 --- /dev/null +++ b/advanced/query.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +# Pi-hole: A black hole for Internet advertisements +# (c) 2018 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# Query Domain Lists +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + +# Globals +piholeDir="/etc/pihole" +adListsList="$piholeDir/adlists.list" +options="$*" +adlist="" +all="" +exact="" +blockpage="" +matchType="match" + +colfile="/opt/pihole/COL_TABLE" +source ${colfile} + +# Scan an array of files for matching strings +scanList(){ + # Escape full stops + local domain="${1//./\\.}" lists="${2}" type="${3:-}" + + # Prevent grep from printing file path + cd "$piholeDir" || exit 1 + + # Prevent grep -i matching slowly: http://bit.ly/2xFXtUX + export LC_CTYPE=C + + # /dev/null forces filename to be printed when only one list has been generated + # shellcheck disable=SC2086 + case "${type}" in + "exact" ) grep -i -E -l "(^|\\s)${domain}($|\\s|#)" ${lists} /dev/null;; + "wc" ) grep -i -o -m 1 "/${domain}/" ${lists};; + * ) grep -i "${domain}" ${lists} /dev/null;; + esac +} + +if [[ "${options}" == "-h" ]] || [[ "${options}" == "--help" ]]; then + echo "Usage: pihole -q [option] +Example: 'pihole -q -exact domain.com' +Query the adlists for a specified domain + +Options: + -adlist Print the name of the block list URL + -exact Search the block lists for exact domain matches + -all Return all query matches within a block list + -h, --help Show this help dialog" + exit 0 +fi + +if [[ ! -e "$adListsList" ]]; then + echo -e "${COL_LIGHT_RED}The file '/etc/pihole/adlists.list' was not found${COL_NC}" + exit 1 +fi + +# Handle valid options +if [[ "${options}" == *"-bp"* ]]; then + exact="exact"; blockpage=true +else + [[ "${options}" == *"-adlist"* ]] && adlist=true + [[ "${options}" == *"-all"* ]] && all=true + if [[ "${options}" == *"-exact"* ]]; then + exact="exact"; matchType="exact ${matchType}" + fi +fi + +# Strip valid options, leaving only the domain and invalid options +# This allows users to place the options before or after the domain +options=$(sed -E 's/ ?-(bp|adlists?|all|exact) ?//g' <<< "${options}") + +# Handle remaining options +# If $options contain non ASCII characters, convert to punycode +case "${options}" in + "" ) str="No domain specified";; + *" "* ) str="Unknown query option specified";; + *[![:ascii:]]* ) domainQuery=$(idn2 "${options}");; + * ) domainQuery="${options}";; +esac + +if [[ -n "${str:-}" ]]; then + echo -e "${str}${COL_NC}\\nTry 'pihole -q --help' for more information." + exit 1 +fi + +# Scan Whitelist and Blacklist +lists="whitelist.txt blacklist.txt" +mapfile -t results <<< "$(scanList "${domainQuery}" "${lists}" "${exact}")" + if [[ -n "${results[*]}" ]]; then + wbMatch=true + # Loop through each result in order to print unique file title once + for result in "${results[@]}"; do + fileName="${result%%.*}" + if [[ -n "${blockpage}" ]]; then + echo "π ${result}" + exit 0 + elif [[ -n "${exact}" ]]; then + echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" + else + # Only print filename title once per file + if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then + echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" + fileName_prev="${fileName}" + fi + echo " ${result#*:}" + fi + done +fi + +# Scan Wildcards +if [[ -e "${wildcardlist}" ]]; then + # Determine all subdomains, domain and TLDs + mapfile -t wildcards <<< "$(processWildcards "${domainQuery}")" + for match in "${wildcards[@]}"; do + # Search wildcard list for matches + mapfile -t results <<< "$(scanList "${match}" "${wildcardlist}" "wc")" + if [[ -n "${results[*]}" ]]; then + if [[ -z "${wcMatch:-}" ]] && [[ -z "${blockpage}" ]]; then + wcMatch=true + echo " ${matchType^} found in ${COL_BOLD}Wildcards${COL_NC}:" + fi + case "${blockpage}" in + true ) echo "π ${wildcardlist##*/}"; exit 0;; + * ) echo " *.${match}";; + esac + fi + done +fi + +# Get version sorted *.domains filenames (without dir path) +lists=("$(cd "$piholeDir" || exit 0; printf "%s\\n" -- *.domains | sort -V)") + +# Query blocklists for occurences of domain +mapfile -t results <<< "$(scanList "${domainQuery}" "${lists[*]}" "${exact}")" + +# Handle notices +if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then + echo -e " ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} within the block lists" + exit 0 +elif [[ -z "${results[*]}" ]]; then + # Result found in WL/BL/Wildcards + exit 0 +elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 100 ]]; then + echo -e " ${INFO} Over 100 ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} + This can be overridden using the -all option" + exit 0 +fi + +# Remove unwanted content from non-exact $results +if [[ -z "${exact}" ]]; then + # Delete lines starting with # + # Remove comments after domain + # Remove hosts format IP address + mapfile -t results <<< "$(IFS=$'\n'; sed \ + -e "/:#/d" \ + -e "s/[ \\t]#.*//g" \ + -e "s/:.*[ \\t]/:/g" \ + <<< "${results[*]}")" + # Exit if result was in a comment + [[ -z "${results[*]}" ]] && exit 0 +fi + +# Get adlist file content as array +if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then + for adlistUrl in $(< "adListsList"); do + if [[ "${adlistUrl:0:4}" =~ (http|www.) ]]; then + adlists+=("${adlistUrl}") + fi + done +fi + +# Print "Exact matches for" title +if [[ -n "${exact}" ]] && [[ -z "${blockpage}" ]]; then + plural=""; [[ "${#results[*]}" -gt 1 ]] && plural="es" + echo " ${matchType^}${plural} for ${COL_BOLD}${domainQuery}${COL_NC} found in:" +fi + +for result in "${results[@]}"; do + fileName="${result/:*/}" + + # Determine *.domains URL using filename's number + if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then + fileNum="${fileName/list./}"; fileNum="${fileNum%%.*}" + fileName="${adlists[$fileNum]}" + + # Discrepency occurs when adlists has been modified, but Gravity has not been run + if [[ -z "${fileName}" ]]; then + fileName="${COL_LIGHT_RED}(no associated adlists URL found)${COL_NC}" + fi + fi + + if [[ -n "${blockpage}" ]]; then + echo "${fileNum} ${fileName}" + elif [[ -n "${exact}" ]]; then + echo " ${fileName}" + else + if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then + count="" + echo " ${matchType^} found in ${COL_BOLD}${fileName}${COL_NC}:" + fileName_prev="${fileName}" + fi + : $((count++)) + + # Print matching domain if $max_count has not been reached + [[ -z "${all}" ]] && max_count="50" + if [[ -z "${all}" ]] && [[ "${count}" -ge "${max_count}" ]]; then + [[ "${count}" -gt "${max_count}" ]] && continue + echo " ${COL_GRAY}Over ${count} results found, skipping rest of file${COL_NC}" + else + echo " ${result#*:}" + fi + fi +done + +exit 0 From b1207949ac00ea984f57b2b4b9b233a2b33e1db8 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Wed, 13 Jun 2018 15:49:52 +1000 Subject: [PATCH 2/9] Call query.sh to replace queryFunc() Signed-off-by: Rob Gill --- pihole | 186 +-------------------------------------------------------- 1 file changed, 1 insertion(+), 185 deletions(-) diff --git a/pihole b/pihole index c125d0d9..b0cab88e 100755 --- a/pihole +++ b/pihole @@ -125,191 +125,7 @@ processWildcards() { queryFunc() { shift - local options="$*" adlist="" all="" exact="" blockpage="" matchType="match" - - if [[ "${options}" == "-h" ]] || [[ "${options}" == "--help" ]]; then - echo "Usage: pihole -q [option] -Example: 'pihole -q -exact domain.com' -Query the adlists for a specified domain - -Options: - -adlist Print the name of the block list URL - -exact Search the block lists for exact domain matches - -all Return all query matches within a block list - -h, --help Show this help dialog" - exit 0 - fi - - if [[ ! -e "/etc/pihole/adlists.list" ]]; then - echo -e "${COL_LIGHT_RED}The file '/etc/pihole/adlists.list' was not found${COL_NC}" - exit 1 - fi - - # Handle valid options - if [[ "${options}" == *"-bp"* ]]; then - exact="exact"; blockpage=true - else - [[ "${options}" == *"-adlist"* ]] && adlist=true - [[ "${options}" == *"-all"* ]] && all=true - if [[ "${options}" == *"-exact"* ]]; then - exact="exact"; matchType="exact ${matchType}" - fi - fi - - # Strip valid options, leaving only the domain and invalid options - # This allows users to place the options before or after the domain - options=$(sed -E 's/ ?-(bp|adlists?|all|exact) ?//g' <<< "${options}") - - # Handle remaining options - # If $options contain non ASCII characters, convert to punycode - case "${options}" in - "" ) str="No domain specified";; - *" "* ) str="Unknown query option specified";; - *[![:ascii:]]* ) domainQuery=$(idn2 "${options}");; - * ) domainQuery="${options}";; - esac - - if [[ -n "${str:-}" ]]; then - echo -e "${str}${COL_NC}\\nTry 'pihole -q --help' for more information." - exit 1 - fi - - # Scan Whitelist and Blacklist - lists="whitelist.txt blacklist.txt" - mapfile -t results <<< "$(scanList "${domainQuery}" "${lists}" "${exact}")" - - if [[ -n "${results[*]}" ]]; then - wbMatch=true - - # Loop through each result in order to print unique file title once - for result in "${results[@]}"; do - fileName="${result%%.*}" - - if [[ -n "${blockpage}" ]]; then - echo "π ${result}" - exit 0 - elif [[ -n "${exact}" ]]; then - echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" - else - # Only print filename title once per file - if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then - echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" - fileName_prev="${fileName}" - fi - echo " ${result#*:}" - fi - done - fi - - # Scan Wildcards - if [[ -e "${wildcardlist}" ]]; then - # Determine all subdomains, domain and TLDs - mapfile -t wildcards <<< "$(processWildcards "${domainQuery}")" - - for match in "${wildcards[@]}"; do - # Search wildcard list for matches - mapfile -t results <<< "$(scanList "${match}" "${wildcardlist}" "wc")" - - if [[ -n "${results[*]}" ]]; then - if [[ -z "${wcMatch:-}" ]] && [[ -z "${blockpage}" ]]; then - wcMatch=true - echo " ${matchType^} found in ${COL_BOLD}Wildcards${COL_NC}:" - fi - - case "${blockpage}" in - true ) echo "π ${wildcardlist##*/}"; exit 0;; - * ) echo " *.${match}";; - esac - fi - done - fi - - # Get version sorted *.domains filenames (without dir path) - lists=("$(cd "/etc/pihole" || exit 0; printf "%s\\n" -- *.domains | sort -V)") - - # Query blocklists for occurences of domain - mapfile -t results <<< "$(scanList "${domainQuery}" "${lists[*]}" "${exact}")" - - # Handle notices - if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then - echo -e " ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} within the block lists" - exit 0 - elif [[ -z "${results[*]}" ]]; then - # Result found in WL/BL/Wildcards - exit 0 - elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 100 ]]; then - echo -e " ${INFO} Over 100 ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} - This can be overridden using the -all option" - exit 0 - fi - - # Remove unwanted content from non-exact $results - if [[ -z "${exact}" ]]; then - # Delete lines starting with # - # Remove comments after domain - # Remove hosts format IP address - mapfile -t results <<< "$(IFS=$'\n'; sed \ - -e "/:#/d" \ - -e "s/[ \\t]#.*//g" \ - -e "s/:.*[ \\t]/:/g" \ - <<< "${results[*]}")" - - # Exit if result was in a comment - [[ -z "${results[*]}" ]] && exit 0 - fi - - # Get adlist file content as array - if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then - for adlistUrl in $(< "/etc/pihole/adlists.list"); do - if [[ "${adlistUrl:0:4}" =~ (http|www.) ]]; then - adlists+=("${adlistUrl}") - fi - done - fi - - # Print "Exact matches for" title - if [[ -n "${exact}" ]] && [[ -z "${blockpage}" ]]; then - plural=""; [[ "${#results[*]}" -gt 1 ]] && plural="es" - echo " ${matchType^}${plural} for ${COL_BOLD}${domainQuery}${COL_NC} found in:" - fi - - for result in "${results[@]}"; do - fileName="${result/:*/}" - - # Determine *.domains URL using filename's number - if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then - fileNum="${fileName/list./}"; fileNum="${fileNum%%.*}" - fileName="${adlists[$fileNum]}" - - # Discrepency occurs when adlists has been modified, but Gravity has not been run - if [[ -z "${fileName}" ]]; then - fileName="${COL_LIGHT_RED}(no associated adlists URL found)${COL_NC}" - fi - fi - - if [[ -n "${blockpage}" ]]; then - echo "${fileNum} ${fileName}" - elif [[ -n "${exact}" ]]; then - echo " ${fileName}" - else - if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then - count="" - echo " ${matchType^} found in ${COL_BOLD}${fileName}${COL_NC}:" - fileName_prev="${fileName}" - fi - : $((count++)) - - # Print matching domain if $max_count has not been reached - [[ -z "${all}" ]] && max_count="50" - if [[ -z "${all}" ]] && [[ "${count}" -ge "${max_count}" ]]; then - [[ "${count}" -gt "${max_count}" ]] && continue - echo " ${COL_GRAY}Over ${count} results found, skipping rest of file${COL_NC}" - else - echo " ${result#*:}" - fi - fi - done - + "${PI_HOLE_SCRIPT_DIR}"/query.sh "$@" exit 0 } From 45a8eda49b7d632ca4d6652558aa8d349fe2bbd3 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Wed, 13 Jun 2018 16:08:21 +1000 Subject: [PATCH 3/9] Stop grep leak in query Fix grep error leak from #1805 Signed-off-by: Rob Gill --- advanced/{ => Scripts}/query.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename advanced/{ => Scripts}/query.sh (97%) diff --git a/advanced/query.sh b/advanced/Scripts/query.sh similarity index 97% rename from advanced/query.sh rename to advanced/Scripts/query.sh index 3bae7422..88118ee1 100644 --- a/advanced/query.sh +++ b/advanced/Scripts/query.sh @@ -35,9 +35,9 @@ scanList(){ # /dev/null forces filename to be printed when only one list has been generated # shellcheck disable=SC2086 case "${type}" in - "exact" ) grep -i -E -l "(^|\\s)${domain}($|\\s|#)" ${lists} /dev/null;; - "wc" ) grep -i -o -m 1 "/${domain}/" ${lists};; - * ) grep -i "${domain}" ${lists} /dev/null;; + "exact" ) grep -i -E -l "(^|\\s)${domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;; + "wc" ) grep -i -o -m 1 "/${domain}/" ${lists} 2>/dev/null;; + * ) grep -i "${domain}" ${lists} /dev/null 2>/dev/null;; esac } From 2255d05664d07c581ca3a81eb37e20cff9c95a19 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Wed, 13 Jun 2018 16:09:49 +1000 Subject: [PATCH 4/9] Remove scanlist(), now in query.sh Signed-off-by: Rob Gill --- pihole | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/pihole b/pihole index b0cab88e..59ab760f 100755 --- a/pihole +++ b/pihole @@ -86,26 +86,6 @@ updateGravityFunc() { exit 0 } -# Scan an array of files for matching strings -scanList(){ - # Escape full stops - local domain="${1//./\\.}" lists="${2}" type="${3:-}" - - # Prevent grep from printing file path - cd "/etc/pihole" || exit 1 - - # Prevent grep -i matching slowly: http://bit.ly/2xFXtUX - export LC_CTYPE=C - - # /dev/null forces filename to be printed when only one list has been generated - # shellcheck disable=SC2086 - case "${type}" in - "exact" ) grep -i -E -l "(^|\\s)${domain}($|\\s|#)" ${lists} /dev/null;; - "wc" ) grep -i -o -m 1 "/${domain}/" ${lists};; - * ) grep -i "${domain}" ${lists} /dev/null;; - esac -} - # Print each subdomain # e.g: foo.bar.baz.com = "foo.bar.baz.com bar.baz.com baz.com com" processWildcards() { From b8e1849cec356d2ccfa5a9cd519636dd8858daa6 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Wed, 13 Jun 2018 16:19:07 +1000 Subject: [PATCH 5/9] wildcardlist Signed-off-by: Rob Gill --- advanced/Scripts/query.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 88118ee1..8b570ffe 100644 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -11,6 +11,7 @@ # Globals piholeDir="/etc/pihole" adListsList="$piholeDir/adlists.list" +wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf" options="$*" adlist="" all="" @@ -19,7 +20,7 @@ blockpage="" matchType="match" colfile="/opt/pihole/COL_TABLE" -source ${colfile} +source "$colfile" # Scan an array of files for matching strings scanList(){ From 8ab0b0e46073c6e06e03745113484d422bdf6afc Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Wed, 13 Jun 2018 16:25:43 +1000 Subject: [PATCH 6/9] colfile Signed-off-by: Rob Gill --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 8b570ffe..42f9c895 100644 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -20,7 +20,7 @@ blockpage="" matchType="match" colfile="/opt/pihole/COL_TABLE" -source "$colfile" +source "${colfile}" # Scan an array of files for matching strings scanList(){ From bf5566649251273f5be5a25593c40f73dfaa7c1d Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Wed, 13 Jun 2018 16:29:07 +1000 Subject: [PATCH 7/9] Appease stickler. Signed-off-by: Rob Gill --- advanced/Scripts/query.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 42f9c895..e1cd6457 100644 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck disable=SC1090 # Pi-hole: A black hole for Internet advertisements # (c) 2018 Pi-hole, LLC (https://pi-hole.net) # Network-wide ad blocking via your own hardware. From a7347238e6b22a9b78addef55809d59be7ca9e65 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Fri, 15 Jun 2018 14:42:30 +1000 Subject: [PATCH 8/9] $adListsList replace filename in text Signed-off-by: Rob Gill --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index e1cd6457..f9d25511 100644 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -57,7 +57,7 @@ Options: fi if [[ ! -e "$adListsList" ]]; then - echo -e "${COL_LIGHT_RED}The file '/etc/pihole/adlists.list' was not found${COL_NC}" + echo -e "${COL_LIGHT_RED}The file "$adListsList" was not found${COL_NC}" exit 1 fi From 23adbf9540dbe1034a87d81adc15d7b03f609a51 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Fri, 15 Jun 2018 14:45:27 +1000 Subject: [PATCH 9/9] remove quotes Signed-off-by: Rob Gill --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index f9d25511..5dc80f6b 100644 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -57,7 +57,7 @@ Options: fi if [[ ! -e "$adListsList" ]]; then - echo -e "${COL_LIGHT_RED}The file "$adListsList" was not found${COL_NC}" + echo -e "${COL_LIGHT_RED}The file $adListsList was not found${COL_NC}" exit 1 fi