From f90dffbf1a79a3da90526da2b7ed8e37946d36d1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 16 Apr 2021 11:41:26 +0200 Subject: [PATCH 1/5] Do not put hostname and pi.hole into local.list - this will be handled by FTL after FTL#1111 has been merged Signed-off-by: DL6ER --- gravity.sh | 50 ++++++-------------------------------------------- 1 file changed, 6 insertions(+), 44 deletions(-) diff --git a/gravity.sh b/gravity.sh index 24a41c48..bfe45b55 100755 --- a/gravity.sh +++ b/gravity.sh @@ -47,16 +47,6 @@ domainsExtension="domains" setupVars="${piholeDir}/setupVars.conf" if [[ -f "${setupVars}" ]];then source "${setupVars}" - - # Remove CIDR mask from IPv4/6 addresses - IPV4_ADDRESS="${IPV4_ADDRESS%/*}" - IPV6_ADDRESS="${IPV6_ADDRESS%/*}" - - # Determine if IPv4/6 addresses exist - if [[ -z "${IPV4_ADDRESS}" ]] && [[ -z "${IPV6_ADDRESS}" ]]; then - echo -e " ${COL_LIGHT_RED}No IP addresses found! Please run 'pihole -r' to reconfigure${COL_NC}" - exit 1 - fi else echo -e " ${COL_LIGHT_RED}Installation Failure: ${setupVars} does not exist! ${COL_NC} Please run 'pihole -r', and choose the 'reconfigure' option to fix." @@ -564,7 +554,7 @@ compareLists() { # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { local url="${1}" cmd_ext="${2}" agent="${3}" adlistID="${4}" saveLocation="${5}" target="${6}" compression="${7}" - local heisenbergCompensator="" patternBuffer str httpCode success="" + local heisenbergCompensator="" patternBuffer str httpCode success="" ip # Create temp file to store content on disk instead of RAM patternBuffer=$(mktemp -p "/tmp" --suffix=".phgpb") @@ -582,7 +572,10 @@ gravity_DownloadBlocklistFromUrl() { blocked=false case $BLOCKINGMODE in "IP-NODATA-AAAA"|"IP") - if [[ $(dig "${domain}" +short | grep "${IPV4_ADDRESS}" -c) -ge 1 ]]; then + # Get IP address of this domain + ip="$(dig "${domain}" +short)" + # Check if this IP matches any IP of the system + if [[ -n "${ip}" && $(grep -Ec "inet(|6) ${ip}" <<< "$(ip a)") -gt 0 ]]; then blocked=true fi;; "NXDOMAIN") @@ -785,43 +778,12 @@ gravity_ShowCount() { gravity_Table_Count "vw_regex_whitelist" "regex whitelist filters" } -# Parse list of domains into hosts format -gravity_ParseDomainsIntoHosts() { - awk -v ipv4="$IPV4_ADDRESS" -v ipv6="$IPV6_ADDRESS" '{ - # Remove windows CR line endings - sub(/\r$/, "") - # Parse each line as "ipaddr domain" - if(ipv6 && ipv4) { - print ipv4" "$0"\n"ipv6" "$0 - } else if(!ipv6) { - print ipv4" "$0 - } else { - print ipv6" "$0 - } - }' >> "${2}" < "${1}" -} - # Create "localhost" entries into hosts format gravity_generateLocalList() { - local hostname - - if [[ -s "/etc/hostname" ]]; then - hostname=$(< "/etc/hostname") - elif command -v hostname &> /dev/null; then - hostname=$(hostname -f) - else - echo -e " ${CROSS} Unable to determine fully qualified domain name of host" - return 0 - fi - - echo -e "${hostname}\\npi.hole" > "${localList}.tmp" - # Empty $localList if it already exists, otherwise, create it - : > "${localList}" + echo "### Do not modify this file, it will be overwritten by pihole -g" > "${localList}" chmod 644 "${localList}" - gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}" - # Add additional LAN hosts provided by OpenVPN (if available) if [[ -f "${VPNList}" ]]; then awk -F, '{printf $2"\t"$1".vpn\n"}' "${VPNList}" >> "${localList}" From aa88be335e602a57e5c8cc70dea6db06454ebf93 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 16 Apr 2021 11:44:54 +0200 Subject: [PATCH 2/5] Acknowledge that IPV{4,6}_ADDRESS may not exist Signed-off-by: DL6ER --- pihole | 5 +---- test/test_automated_install.py | 2 -- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/pihole b/pihole index f8085c8e..4fe80fc7 100755 --- a/pihole +++ b/pihole @@ -363,16 +363,13 @@ tailFunc() { fi echo -e " ${INFO} Press Ctrl-C to exit" - # Retrieve IPv4/6 addresses - source /etc/pihole/setupVars.conf - # Strip date from each line # Color blocklist/blacklist/wildcard entries as red # Color A/AAAA/DHCP strings as white # Color everything else as gray tail -f /var/log/pihole.log | sed -E \ -e "s,($(date +'%b %d ')| dnsmasq\[[0-9]*\]),,g" \ - -e "s,(.*(blacklisted |gravity blocked ).* is (0.0.0.0|::|NXDOMAIN|${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \ + -e "s,(.*(blacklisted |gravity blocked ).* is (0.0.0.0|::|NXDOMAIN).*),${COL_RED}&${COL_NC}," \ -e "s,.*(query\\[A|DHCP).*,${COL_NC}&${COL_NC}," \ -e "s,.*,${COL_GRAY}&${COL_NC}," exit 0 diff --git a/test/test_automated_install.py b/test/test_automated_install.py index 44624082..ba1781aa 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -45,8 +45,6 @@ def test_setupVars_are_sourced_to_global_scope(Pihole): # Currently debug test function only echo "Outputting sourced variables" echo "PIHOLE_INTERFACE=${PIHOLE_INTERFACE}" - echo "IPV4_ADDRESS=${IPV4_ADDRESS}" - echo "IPV6_ADDRESS=${IPV6_ADDRESS}" echo "PIHOLE_DNS_1=${PIHOLE_DNS_1}" echo "PIHOLE_DNS_2=${PIHOLE_DNS_2}" } From 58905a1188b9d46e97e6dec975fbb90a22101258 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 16 Apr 2021 12:14:05 +0200 Subject: [PATCH 3/5] Modify debugger to test name resolution on all available interfaces Signed-off-by: DL6ER --- advanced/Scripts/chronometer.sh | 4 -- advanced/Scripts/piholeDebug.sh | 65 ++++++++++++++++++++++----------- 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh index 4f9ea59a..3f85bdfc 100755 --- a/advanced/Scripts/chronometer.sh +++ b/advanced/Scripts/chronometer.sh @@ -498,10 +498,6 @@ chronoFunc() { printFunc " RAM usage: " "$ram_perc%" "$ram_info" printFunc " HDD usage: " "$disk_perc" "$disk_info" - if [[ "$scr_lines" -gt 17 ]] && [[ "$chrono_width" != "small" ]]; then - printFunc " LAN addr: " "${IPV4_ADDRESS/\/*/}" "$lan_info" - fi - if [[ "$DHCP_ACTIVE" == "true" ]]; then printFunc "DHCP usage: " "$ph_dhcp_percent%" "$dhcp_info" fi diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 13a886f1..c6294e6e 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -859,7 +859,6 @@ dig_at() { # Store the arguments as variables with names local protocol="${1}" - local IP="${2}" echo_current_diagnostic "Name resolution (IPv${protocol}) using a random blocked domain and a known ad-serving domain" # Set more local variables # We need to test name resolution locally, via Pi-hole, and via a public resolver @@ -874,15 +873,15 @@ dig_at() { if [[ ${protocol} == "6" ]]; then # Set the IPv6 variables and record type local local_address="::1" - local pihole_address="${IP}" local remote_address="2001:4860:4860::8888" + local sed_selector="inet6" local record_type="AAAA" # Otherwise, it should be 4 else # so use the IPv4 values local local_address="127.0.0.1" - local pihole_address="${IP}" local remote_address="8.8.8.8" + local sed_selector="inet" local record_type="A" fi @@ -895,25 +894,53 @@ dig_at() { # First, do a dig on localhost to see if Pi-hole can use itself to block a domain if local_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @${local_address} +short "${record_type}"); then # If it can, show success - log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} via ${COL_CYAN}localhost$COL_NC (${local_address})" + log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} via ${COL_CYAN}localhost$COL_NC at ${COL_CYAN}${local_address}${CON_NC}" else # Otherwise, show a failure - log_write "${CROSS} ${COL_RED}Failed to resolve${COL_NC} ${random_url} via ${COL_RED}localhost${COL_NC} (${local_address})" + log_write "${CROSS} ${COL_RED}Failed to resolve${COL_NC} ${random_url} via ${COL_RED}localhost${COL_NC} at ${COL_CYAN}${local_address}${CON_NC}" fi # Next we need to check if Pi-hole can resolve a domain when the query is sent to it's IP address # This better emulates how clients will interact with Pi-hole as opposed to above where Pi-hole is # just asing itself locally - # The default timeouts and tries are reduced in case the DNS server isn't working, so the user isn't waiting for too long - - # If Pi-hole can dig itself from it's IP (not the loopback address) - if pihole_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${pihole_address}" +short "${record_type}"); then - # show a success - log_write "${TICK} ${random_url} ${COL_GREEN}is ${pihole_dig}${COL_NC} via ${COL_CYAN}Pi-hole${COL_NC} (${pihole_address})" - else - # Otherwise, show a failure - log_write "${CROSS} ${COL_RED}Failed to resolve${COL_NC} ${random_url} via ${COL_RED}Pi-hole${COL_NC} (${pihole_address})" - fi + # The default timeouts and tries are reduced in case the DNS server isn't working, so the user isn't + # waiting for too long + # + # Turn off history expansion such that the "!" in the sed command cannot do silly things + set +H + # Get interfaces + # sed logic breakdown: + # / master /d; + # Removes all interfaces that are slaves of others (e.g. virtual docker interfaces) + # /UP/!d; + # Removes all interfaces which are not UP + # s/^[0-9]*: //g; + # Removes interface index + # s/: <.*//g; + # Removes everything after the interface name + local interfaces="$(ip link show | sed "/ master /d;/UP/!d;s/^[0-9]*: //g;s/: <.*//g;")" + + while IFS= read -r iface ; do + # Get addresses of current interface + # sed logic breakdown: + # /inet(|6) /!d; + # Removes all lines from ip a that do not contain either "inet " or "inet6 " + # s/^.*inet(|6) //g; + # Removes all leading whitespace as well as the "inet " or "inet6 " string + # s/\/.*$//g; + # Removes CIDR and everything thereafter (e.g., scope properties) + local addresses="$(ip address show dev "${iface}" | sed "/${sed_selector} /!d;s/^.*${sed_selector} //g;s/\/.*$//g;")" + while IFS= read -r local_address ; do + # Check if Pi-hole can use itself to block a domain + if local_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @${local_address} +short "${record_type}"); then + # If it can, show success + log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} on ${COL_CYAN}${iface}${COL_NC} (${COL_CYAN}${local_address}${COL_NC})" + else + # Otherwise, show a failure + log_write "${CROSS} ${COL_RED}Failed to resolve${COL_NC} ${random_url} on ${COL_RED}${iface}${COL_NC} (${COL_RED}${local_address}${COL_NC})" + fi + done <<< "${addresses}" + done <<< "${interfaces}" # Finally, we need to make sure legitimate queries can out to the Internet using an external, public DNS server # We are using the static remote_url here instead of a random one because we know it works with IPv4 and IPv6 @@ -1046,12 +1073,8 @@ parse_file() { check_name_resolution() { # Check name resolution from localhost, Pi-hole's IP, and Google's name severs # using the function we created earlier - dig_at 4 "${IPV4_ADDRESS%/*}" - # If IPv6 enabled, - if [[ "${IPV6_ADDRESS}" ]]; then - # check resolution - dig_at 6 "${IPV6_ADDRESS%/*}" - fi + dig_at 4 + dig_at 6 } # This function can check a directory exists From 3b574096b80a44b2366c8bdbb9df8cfea14091b0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 16 Apr 2021 12:24:46 +0200 Subject: [PATCH 4/5] Remove duplicated lo test Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index c6294e6e..02363b60 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -863,8 +863,9 @@ dig_at() { # Set more local variables # We need to test name resolution locally, via Pi-hole, and via a public resolver local local_dig - local pihole_dig local remote_dig + local interfaces + local addresses # Use a static domain that we know has IPv4 and IPv6 to avoid false positives # Sometimes the randomly chosen domains don't use IPv6, or something else is wrong with them local remote_url="doubleclick.com" @@ -891,15 +892,6 @@ dig_at() { local random_url random_url=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity ORDER BY RANDOM() LIMIT 1") - # First, do a dig on localhost to see if Pi-hole can use itself to block a domain - if local_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @${local_address} +short "${record_type}"); then - # If it can, show success - log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} via ${COL_CYAN}localhost$COL_NC at ${COL_CYAN}${local_address}${CON_NC}" - else - # Otherwise, show a failure - log_write "${CROSS} ${COL_RED}Failed to resolve${COL_NC} ${random_url} via ${COL_RED}localhost${COL_NC} at ${COL_CYAN}${local_address}${CON_NC}" - fi - # Next we need to check if Pi-hole can resolve a domain when the query is sent to it's IP address # This better emulates how clients will interact with Pi-hole as opposed to above where Pi-hole is # just asing itself locally @@ -918,7 +910,7 @@ dig_at() { # Removes interface index # s/: <.*//g; # Removes everything after the interface name - local interfaces="$(ip link show | sed "/ master /d;/UP/!d;s/^[0-9]*: //g;s/: <.*//g;")" + interfaces="$(ip link show | sed "/ master /d;/UP/!d;s/^[0-9]*: //g;s/: <.*//g;")" while IFS= read -r iface ; do # Get addresses of current interface @@ -929,10 +921,10 @@ dig_at() { # Removes all leading whitespace as well as the "inet " or "inet6 " string # s/\/.*$//g; # Removes CIDR and everything thereafter (e.g., scope properties) - local addresses="$(ip address show dev "${iface}" | sed "/${sed_selector} /!d;s/^.*${sed_selector} //g;s/\/.*$//g;")" + addresses="$(ip address show dev "${iface}" | sed "/${sed_selector} /!d;s/^.*${sed_selector} //g;s/\/.*$//g;")" while IFS= read -r local_address ; do # Check if Pi-hole can use itself to block a domain - if local_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @${local_address} +short "${record_type}"); then + if local_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${local_address}" +short "${record_type}"); then # If it can, show success log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} on ${COL_CYAN}${iface}${COL_NC} (${COL_CYAN}${local_address}${COL_NC})" else @@ -944,7 +936,7 @@ dig_at() { # Finally, we need to make sure legitimate queries can out to the Internet using an external, public DNS server # We are using the static remote_url here instead of a random one because we know it works with IPv4 and IPv6 - if remote_dig=$(dig +tries=1 +time=2 -"${protocol}" "${remote_url}" @${remote_address} +short "${record_type}" | head -n1); then + if remote_dig=$(dig +tries=1 +time=2 -"${protocol}" "${remote_url}" @"${remote_address}" +short "${record_type}" | head -n1); then # If successful, the real IP of the domain will be returned instead of Pi-hole's IP log_write "${TICK} ${remote_url} ${COL_GREEN}is ${remote_dig}${COL_NC} via ${COL_CYAN}a remote, public DNS server${COL_NC} (${remote_address})" else @@ -1059,7 +1051,7 @@ parse_file() { local file_lines # For each line in the file, for file_lines in "${file_info[@]}"; do - if [[ ! -z "${file_lines}" ]]; then + if [[ -n "${file_lines}" ]]; then # don't include the Web password hash [[ "${file_lines}" =~ ^\#.*$ || ! "${file_lines}" || "${file_lines}" == "WEBPASSWORD="* ]] && continue # otherwise, display the lines of the file From 60513f93a3f239bf503372972c9e3c3b312cf9be Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 18 Apr 2021 09:26:23 +0200 Subject: [PATCH 5/5] Don't set IPV{4,6}_ADDRESS in conftest.py Signed-off-by: DL6ER --- test/conftest.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 07166ec5..13731eb8 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -8,8 +8,6 @@ check_output = testinfra.get_backend( SETUPVARS = { 'PIHOLE_INTERFACE': 'eth99', - 'IPV4_ADDRESS': '1.1.1.1', - 'IPV6_ADDRESS': 'FE80::240:D0FF:FE48:4672', 'PIHOLE_DNS_1': '4.2.2.1', 'PIHOLE_DNS_2': '4.2.2.2' }