From 19d3489bcbe5aca79ec08a38008b6d5a2f1dcaf1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 20:56:23 +0100 Subject: [PATCH 1/6] gravity_DownloadBlocklistFromUrl needs ${domain} but it was declared local before and was not passed as argument, making gravity currently depend on undefined behavior. It seems to be working well in the vast majority of cases, however, it seems we have at least one report where it is not working. Signed-off-by: DL6ER --- gravity.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index edfe89a9..421ec035 100755 --- a/gravity.sh +++ b/gravity.sh @@ -464,7 +464,7 @@ gravity_DownloadBlocklists() { if [[ "${check_url}" =~ ${regex} ]]; then echo -e " ${CROSS} Invalid Target" else - gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" "${adlist_type}" + gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" "${adlist_type}" "${domain}" fi echo "" done @@ -496,7 +496,7 @@ compareLists() { # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { - local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" gravity_type="${6}" + local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" gravity_type="${6}" domain="${7}" local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext # Create temp file to store content on disk instead of RAM From f16cf7178186e33004bcfb44c3838a9fd2912007 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 21:04:46 +0100 Subject: [PATCH 2/6] ${PIHOLE_DNS_1} is gone, use the first server from pihole-FTL --config dns.upstreams instead Signed-off-by: DL6ER --- gravity.sh | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index 421ec035..90cba0f6 100755 --- a/gravity.sh +++ b/gravity.sh @@ -538,12 +538,25 @@ gravity_DownloadBlocklistFromUrl() { fi;; esac + if [[ "${blocked}" == true ]]; then - printf -v ip_addr "%s" "${PIHOLE_DNS_1%#*}" - if [[ ${PIHOLE_DNS_1} != *"#"* ]]; then + # Get first defined upstream server + local upstream + upstream="$(getFTLConfigValue dns.upstreams)" + + # Isolate first upstream server from a string like + # [ 1.2.3.4#1234, 5.6.7.8#5678, ... ] + upstream="${upstream%%,*}" + upstream="${upstream##*[}" + upstream="${upstream%%]*}" + + # Get IP address and port of this upstream server + local ip_addr port + printf -v ip_addr "%s" "${upstream%#*}" + if [[ ${upstream} != *"#"* ]]; then port=53 else - printf -v port "%s" "${PIHOLE_DNS_1#*#}" + printf -v port "%s" "${upstream#*#}" fi ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}" | tail -1) if [[ $(echo "${url}" | awk -F '://' '{print $1}') = "https" ]]; then @@ -551,7 +564,7 @@ gravity_DownloadBlocklistFromUrl() { else port=80 fi bad_list=$(pihole -q -adlist "${domain}" | head -n1 | awk -F 'Match found in ' '{print $2}') - echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${PIHOLE_DNS_1} to download ${url}"; + echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${upstream} to download ${url}"; echo -ne " ${INFO} ${str} Pending..." cmd_ext="--resolve $domain:$port:$ip" fi From 4efcdf11890ca069019c5b9cb3c1206e10a7f73e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 21:06:09 +0100 Subject: [PATCH 3/6] Add missing double quotes to prevent globbing and word splitting Signed-off-by: DL6ER --- gravity.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index 90cba0f6..7a4e52c2 100755 --- a/gravity.sh +++ b/gravity.sh @@ -99,7 +99,7 @@ gravity_swap_databases() { # Number of available blocks on disk availableBlocks=$(stat -f --format "%a" "${gravityDIR}") # Number of blocks, used by gravity.db - gravityBlocks=$(stat --format "%b" ${gravityDBfile}) + gravityBlocks=$(stat --format "%b" "${gravityDBfile}") # Only keep the old database if available disk space is at least twice the size of the existing gravity.db. # Better be safe than sorry... oldAvail=false @@ -604,7 +604,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ "${success}" == true ]]; then if [[ "${httpCode}" == "304" ]]; then # Add domains to database table file - pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" database_adlist_status "${adlistID}" "2" done="true" # Check if $listCurlBuffer is a non-zero length file @@ -614,7 +614,7 @@ gravity_DownloadBlocklistFromUrl() { # Remove curl buffer file after its use rm "${listCurlBuffer}" # Add domains to database table file - pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" # Compare lists, are they identical? compareLists "${adlistID}" "${saveLocation}" done="true" @@ -630,7 +630,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ -r "${saveLocation}" ]]; then echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" # Add domains to database table file - pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" database_adlist_status "${adlistID}" "3" else echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" From df7633bd1b2f4c8e653620ead7c00ab86604428a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 21:08:06 +0100 Subject: [PATCH 4/6] Add missing value for ${gravityDBfile_default} Signed-off-by: DL6ER --- gravity.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/gravity.sh b/gravity.sh index 7a4e52c2..64ba662e 100755 --- a/gravity.sh +++ b/gravity.sh @@ -59,6 +59,7 @@ fi # Set this only after sourcing pihole-FTL.conf as the gravity database path may # have changed gravityDBfile="${GRAVITYDB}" +gravityDBfile_default="/etc/pihole/gravity.db" gravityTEMPfile="${GRAVITYDB}_temp" gravityDIR="$(dirname -- "${gravityDBfile}")" gravityOLDfile="${gravityDIR}/gravity_old.db" From cc333f79ccba8358b87bb38d6f3694ab3bf24809 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 21:10:22 +0100 Subject: [PATCH 5/6] Check if this domain is blocked by Pi-hole but only if the domain is not a local file or empty Signed-off-by: DL6ER --- gravity.sh | 102 ++++++++++++++++++++++++++++------------------------- 1 file changed, 53 insertions(+), 49 deletions(-) diff --git a/gravity.sh b/gravity.sh index 64ba662e..4aea4d68 100755 --- a/gravity.sh +++ b/gravity.sh @@ -517,57 +517,61 @@ gravity_DownloadBlocklistFromUrl() { str="Status:" echo -ne " ${INFO} ${str} Pending..." blocked=false - case $(getFTLConfigValue dns.blocking.mode) in - "IP-NODATA-AAAA"|"IP") - # Get IP address of this domain - ip="$(dig "${domain}" +short)" - # Check if this IP matches any IP of the system - if [[ -n "${ip}" && $(grep -Ec "inet(|6) ${ip}" <<< "$(ip a)") -gt 0 ]]; then - blocked=true - fi;; - "NXDOMAIN") - if [[ $(dig "${domain}" | grep "NXDOMAIN" -c) -ge 1 ]]; then - blocked=true - fi;; - "NODATA") - if [[ $(dig "${domain}" | grep "NOERROR" -c) -ge 1 ]] && [[ -z $(dig +short "${domain}") ]]; then - blocked=true - fi;; - "NULL"|*) - if [[ $(dig "${domain}" +short | grep "0.0.0.0" -c) -ge 1 ]]; then - blocked=true - fi;; - esac - - if [[ "${blocked}" == true ]]; then - # Get first defined upstream server - local upstream - upstream="$(getFTLConfigValue dns.upstreams)" - - # Isolate first upstream server from a string like - # [ 1.2.3.4#1234, 5.6.7.8#5678, ... ] - upstream="${upstream%%,*}" - upstream="${upstream##*[}" - upstream="${upstream%%]*}" - - # Get IP address and port of this upstream server - local ip_addr port - printf -v ip_addr "%s" "${upstream%#*}" - if [[ ${upstream} != *"#"* ]]; then - port=53 - else - printf -v port "%s" "${upstream#*#}" - fi - ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}" | tail -1) - if [[ $(echo "${url}" | awk -F '://' '{print $1}') = "https" ]]; then - port=443; - else port=80 + # Check if this domain is blocked by Pi-hole but only if the domain is not a + # local file or empty + if [[ $url != "file"* ]] && [[ -n "${domain}" ]]; then + case $(getFTLConfigValue dns.blocking.mode) in + "IP-NODATA-AAAA"|"IP") + # Get IP address of this domain + ip="$(dig "${domain}" +short)" + # Check if this IP matches any IP of the system + if [[ -n "${ip}" && $(grep -Ec "inet(|6) ${ip}" <<< "$(ip a)") -gt 0 ]]; then + blocked=true + fi;; + "NXDOMAIN") + if [[ $(dig "${domain}" | grep "NXDOMAIN" -c) -ge 1 ]]; then + blocked=true + fi;; + "NODATA") + if [[ $(dig "${domain}" | grep "NOERROR" -c) -ge 1 ]] && [[ -z $(dig +short "${domain}") ]]; then + blocked=true + fi;; + "NULL"|*) + if [[ $(dig "${domain}" +short | grep "0.0.0.0" -c) -ge 1 ]]; then + blocked=true + fi;; + esac + + if [[ "${blocked}" == true ]]; then + # Get first defined upstream server + local upstream + upstream="$(getFTLConfigValue dns.upstreams)" + + # Isolate first upstream server from a string like + # [ 1.2.3.4#1234, 5.6.7.8#5678, ... ] + upstream="${upstream%%,*}" + upstream="${upstream##*[}" + upstream="${upstream%%]*}" + + # Get IP address and port of this upstream server + local ip_addr port + printf -v ip_addr "%s" "${upstream%#*}" + if [[ ${upstream} != *"#"* ]]; then + port=53 + else + printf -v port "%s" "${upstream#*#}" + fi + ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}" | tail -1) + if [[ $(echo "${url}" | awk -F '://' '{print $1}') = "https" ]]; then + port=443; + else port=80 + fi + bad_list=$(pihole -q -adlist "${domain}" | head -n1 | awk -F 'Match found in ' '{print $2}') + echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${upstream} to download ${url}"; + echo -ne " ${INFO} ${str} Pending..." + cmd_ext="--resolve $domain:$port:$ip" fi - bad_list=$(pihole -q -adlist "${domain}" | head -n1 | awk -F 'Match found in ' '{print $2}') - echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${upstream} to download ${url}"; - echo -ne " ${INFO} ${str} Pending..." - cmd_ext="--resolve $domain:$port:$ip" fi # shellcheck disable=SC2086 From c785667efea9e579db37c7d450bd3fbe9e7612a0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 23 Nov 2023 09:56:43 +0100 Subject: [PATCH 6/6] Trim leading and trailing spaces and tabs in upstream servers (if any) Signed-off-by: DL6ER --- gravity.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 4aea4d68..20ad6215 100755 --- a/gravity.sh +++ b/gravity.sh @@ -553,6 +553,9 @@ gravity_DownloadBlocklistFromUrl() { upstream="${upstream%%,*}" upstream="${upstream##*[}" upstream="${upstream%%]*}" + # Trim leading and trailing spaces and tabs + upstream="${upstream#"${upstream%%[![:space:]]*}"}" + upstream="${upstream%"${upstream##*[![:space:]]}"}" # Get IP address and port of this upstream server local ip_addr port @@ -567,8 +570,7 @@ gravity_DownloadBlocklistFromUrl() { port=443; else port=80 fi - bad_list=$(pihole -q -adlist "${domain}" | head -n1 | awk -F 'Match found in ' '{print $2}') - echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${upstream} to download ${url}"; + echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by one of your lists. Using DNS server ${upstream} instead"; echo -ne " ${INFO} ${str} Pending..." cmd_ext="--resolve $domain:$port:$ip" fi