From 164a81776eb055f98418f6eafcfe17f9333be2b2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Nov 2017 18:30:40 +0100 Subject: [PATCH 01/27] Allow to add local lists to gravity using e.g. file:///path/to/my.list in adlists.list Signed-off-by: DL6ER --- gravity.sh | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/gravity.sh b/gravity.sh index f4b5fc36..453985e6 100755 --- a/gravity.sh +++ b/gravity.sh @@ -138,8 +138,9 @@ gravity_Collapse() { # Logic: Split by folder/port awk -F '[/:]' '{ # Remove URL protocol & optional username:password@ - gsub(/(.*:\/\/|.*:.*@)/, "", $0) - print $1 + gsub(/(.*:\/\/|)/, "", $0) + if(length($1)>0){print $1} + else {print "local"} }' <<< "$(printf '%s\n' "${sources[@]}")" 2> /dev/null )" @@ -203,20 +204,27 @@ gravity_Pull() { # shellcheck disable=SC2086 httpCode=$(curl -s -L ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${patternBuffer}" 2> /dev/null) - # Determine "Status:" output based on HTTP response - case "${httpCode}" in - "200") echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true;; - "304") echo -e "${OVER} ${TICK} ${str} No changes detected"; success=true;; - "000") echo -e "${OVER} ${CROSS} ${str} Connection Refused";; - "403") echo -e "${OVER} ${CROSS} ${str} Forbidden";; - "404") echo -e "${OVER} ${CROSS} ${str} Not found";; - "408") echo -e "${OVER} ${CROSS} ${str} Time-out";; - "451") echo -e "${OVER} ${CROSS} ${str} Unavailable For Legal Reasons";; - "500") echo -e "${OVER} ${CROSS} ${str} Internal Server Error";; - "504") echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Gateway)";; - "521") echo -e "${OVER} ${CROSS} ${str} Web Server Is Down (Cloudflare)";; - "522") echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Cloudflare)";; - * ) echo -e "${OVER} ${CROSS} ${str} ${httpCode}";; + case $url in + # Did we "download" a remote file? + "http"*) + # Determine "Status:" output based on HTTP response + case "${httpCode}" in + "200") echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true;; + "304") echo -e "${OVER} ${TICK} ${str} No changes detected"; success=true;; + "000") echo -e "${OVER} ${CROSS} ${str} Connection Refused";; + "403") echo -e "${OVER} ${CROSS} ${str} Forbidden";; + "404") echo -e "${OVER} ${CROSS} ${str} Not found";; + "408") echo -e "${OVER} ${CROSS} ${str} Time-out";; + "451") echo -e "${OVER} ${CROSS} ${str} Unavailable For Legal Reasons";; + "500") echo -e "${OVER} ${CROSS} ${str} Internal Server Error";; + "504") echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Gateway)";; + "521") echo -e "${OVER} ${CROSS} ${str} Web Server Is Down (Cloudflare)";; + "522") echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Cloudflare)";; + * ) echo -e "${OVER} ${CROSS} ${str} ${httpCode}";; + esac;; + # Did we "download" a local file? + "file"*) echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true;; + * ) echo -e "${OVER} ${CROSS} ${str} ${url} ${httpCode}";; esac # Determine if the blocklist was downloaded and saved correctly @@ -229,7 +237,7 @@ gravity_Pull() { gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}" else # Fall back to previously cached list if $patternBuffer is empty - echo -e " ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" + echo -e " ${INFO} ${COL_LIGHT_GREEN}Using previously cached list${COL_NC}" fi else # Determine if cached list has read permission @@ -407,7 +415,7 @@ gravity_Filter() { # Whitelist unique blocklist domain sources gravity_WhitelistBLD() { - local uniqDomains plural="" str + local uniqDomains plural="" str echo "" From 8976930e20fcabe529c544498db442ef04f824fb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Nov 2017 18:35:58 +0100 Subject: [PATCH 02/27] Properly detect if local file was successfully downloaded Signed-off-by: DL6ER --- gravity.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index 453985e6..f563a442 100755 --- a/gravity.sh +++ b/gravity.sh @@ -193,9 +193,10 @@ gravity_Pull() { patternBuffer=$(mktemp -p "/tmp" --suffix=".phgpb") # Determine if $saveLocation has read permission - if [[ -r "${saveLocation}" ]]; then + if [[ -r "${saveLocation}" && $url != "file"* ]]; then # Have curl determine if a remote file has been modified since last retrieval # Uses "Last-Modified" header, which certain web servers do not provide (e.g: raw github urls) + # Note: Don't do this for local files, always download them heisenbergCompensator="-z ${saveLocation}" fi @@ -223,7 +224,12 @@ gravity_Pull() { * ) echo -e "${OVER} ${CROSS} ${str} ${httpCode}";; esac;; # Did we "download" a local file? - "file"*) echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true;; + "file"*) + if [[ -s "${patternBuffer}" ]]; then + echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true + else + echo -e "${OVER} ${CROSS} ${str} Not found" + fi;; * ) echo -e "${OVER} ${CROSS} ${str} ${url} ${httpCode}";; esac @@ -237,7 +243,7 @@ gravity_Pull() { gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}" else # Fall back to previously cached list if $patternBuffer is empty - echo -e " ${INFO} ${COL_LIGHT_GREEN}Using previously cached list${COL_NC}" + echo -e " ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" fi else # Determine if cached list has read permission From eb83081a5c4f8b06cbbbd893ef4d95032c8d7e90 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Nov 2017 18:36:34 +0100 Subject: [PATCH 03/27] Clarify error message: File could also have been of zero size Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index f563a442..946dd820 100755 --- a/gravity.sh +++ b/gravity.sh @@ -228,7 +228,7 @@ gravity_Pull() { if [[ -s "${patternBuffer}" ]]; then echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true else - echo -e "${OVER} ${CROSS} ${str} Not found" + echo -e "${OVER} ${CROSS} ${str} Not found / empty list" fi;; * ) echo -e "${OVER} ${CROSS} ${str} ${url} ${httpCode}";; esac From c54f04ef4cbb3b850d4f96bc0ca6576d6893c2cc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Nov 2017 20:55:47 +0100 Subject: [PATCH 04/27] Repair username:password filtering Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 946dd820..430b38f8 100755 --- a/gravity.sh +++ b/gravity.sh @@ -138,7 +138,7 @@ gravity_Collapse() { # Logic: Split by folder/port awk -F '[/:]' '{ # Remove URL protocol & optional username:password@ - gsub(/(.*:\/\/|)/, "", $0) + gsub(/(.*:\/\/|.*:.*@)/, "", $0) if(length($1)>0){print $1} else {print "local"} }' <<< "$(printf '%s\n' "${sources[@]}")" 2> /dev/null From 6f0bb30def2456c82a2b77ff97164b466c59ca1f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Nov 2017 20:58:27 +0100 Subject: [PATCH 05/27] Improve indentation Signed-off-by: DL6ER --- gravity.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 430b38f8..8e666e35 100755 --- a/gravity.sh +++ b/gravity.sh @@ -230,7 +230,8 @@ gravity_Pull() { else echo -e "${OVER} ${CROSS} ${str} Not found / empty list" fi;; - * ) echo -e "${OVER} ${CROSS} ${str} ${url} ${httpCode}";; + * ) + echo -e "${OVER} ${CROSS} ${str} ${url} ${httpCode}";; esac # Determine if the blocklist was downloaded and saved correctly From 8d721d086cbe4b49665c9e0b1d81499b284776a9 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Tue, 21 Nov 2017 18:37:38 -0500 Subject: [PATCH 06/27] Modify indentation --- gravity.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 8e666e35..eb69b49f 100755 --- a/gravity.sh +++ b/gravity.sh @@ -230,8 +230,7 @@ gravity_Pull() { else echo -e "${OVER} ${CROSS} ${str} Not found / empty list" fi;; - * ) - echo -e "${OVER} ${CROSS} ${str} ${url} ${httpCode}";; + *) echo -e "${OVER} ${CROSS} ${str} ${url} ${httpCode}";; esac # Determine if the blocklist was downloaded and saved correctly From f89d69b0819e7bbf1b4ee7a4db4692e66133078a Mon Sep 17 00:00:00 2001 From: bcambl Date: Thu, 7 Dec 2017 21:33:31 -0600 Subject: [PATCH 07/27] check NetworkManager status prior to using the cli (#1653) Closes #1653 Signed-off-by: bcambl --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 61d759ff..80ebd5fa 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -709,8 +709,8 @@ setStaticIPv4() { }> "${IFCFG_FILE}" # Use ip to immediately set the new address ip addr replace dev "${PIHOLE_INTERFACE}" "${IPV4_ADDRESS}" - # If NetworkMangler command line interface exists, - if command -v nmcli &> /dev/null;then + # If NetworkMangler command line interface exists and ready to mangle, + if command -v nmcli &> /dev/null && nmcli general status &> /dev/null; then # Tell NetworkManagler to read our new sysconfig file nmcli con load "${IFCFG_FILE}" > /dev/null fi From 80e17ab72180f15ac4d5ff0044d4e8eafd95bc29 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Thu, 7 Dec 2017 22:38:47 -0600 Subject: [PATCH 08/27] potentially fixes #1806 by falling back to dig if getent fails Signed-off-by: Jacob Salmela --- gravity.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/gravity.sh b/gravity.sh index eb69b49f..e7aed5e5 100755 --- a/gravity.sh +++ b/gravity.sh @@ -88,6 +88,19 @@ gravity_DNSLookup() { exit 1 fi + # If the /etc/resolv.conf contains resolvers other than 127.0.0.1 then the local dnsmasq will not be queried and pi.hole is NXDOMAIN. + # This means that even though name resolution is working, the getent hosts check fails and the holddown timer keeps ticking and eventualy fails + # So we check the output of the last command and if it failed, attempt to use dig +short as a fallback + if timeout 1 dig +short "${lookupDomain}" &> /dev/null; then + if [[ -n "${secs:-}" ]]; then + echo -e "${OVER} ${TICK} DNS resolution is now available\\n" + fi + return 0 + elif [[ -n "${secs:-}" ]]; then + echo -e "${OVER} ${CROSS} DNS resolution is not available" + exit 1 + fi + # Determine error output message if pidof dnsmasq &> /dev/null; then echo -e " ${CROSS} DNS resolution is currently unavailable" From 6ecd93d0c9c05cfc67576ecc36b37f980b9ef16d Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 8 Dec 2017 22:25:01 +0000 Subject: [PATCH 09/27] pihole.log permissions This change makes pihole more friendly to the non-existence of the pihole.log file. This can help with systems that are configured to mount /var/log as a tmpfs volume. It may also help with systems where the pihole.log file is accidentally/unintentionally removed. Further discussion around the details of this change are in https://github.com/pi-hole/pi-hole/issues/1798 --- advanced/pihole-FTL.service | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/pihole-FTL.service b/advanced/pihole-FTL.service index 627fad8c..b7def22c 100644 --- a/advanced/pihole-FTL.service +++ b/advanced/pihole-FTL.service @@ -25,9 +25,9 @@ start() { if is_running; then echo "pihole-FTL is already running" else - touch /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port + touch /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole.log chown pihole:pihole /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port /etc/pihole - chmod 0644 /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port + chmod 0644 /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole.log su -s /bin/sh -c "/usr/bin/pihole-FTL" "$FTLUSER" echo fi From 6f03e3015154bb018a920c4cd27c9c2663d7b76e Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Fri, 8 Dec 2017 18:23:55 -0600 Subject: [PATCH 10/27] fixes #1537 by checking if systemctl is found Signed-off-by: Jacob Salmela --- advanced/Scripts/piholeDebug.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index d69c5e4d..f8955637 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -212,7 +212,7 @@ copy_to_debug_log() { # uploaded to our server, since it can't properly display in color # This is accomplished by use sed to remove characters matching that patter # The entire file is then copied over to a sanitized version of the log - sed 's/\[[0-9;]\{1,5\}m//g' > "${PIHOLE_DEBUG_LOG_SANITIZED}" <<< cat "${PIHOLE_DEBUG_LOG}" + # sed 's/\[[0-9;]\{1,5\}m//g' > "${PIHOLE_DEBUG_LOG_SANITIZED}" <<< cat "${PIHOLE_DEBUG_LOG}" } initiate_debug() { @@ -809,8 +809,14 @@ process_status(){ local i # For each process, for i in "${PIHOLE_PROCESSES[@]}"; do - # get its status via systemctl - local status_of_process=$(systemctl is-active "${i}") + # If systemd + if command -v systemctl &> /dev/null; then + # get its status via systemctl + local status_of_process=$(systemctl is-active "${i}") + else + # Otherwise, use the service command + local status_of_process=$(service "${i}" status | awk '/Active:/ {print $2}') &> /dev/null + fi # and print it out to the user if [[ "${status_of_process}" == "active" ]]; then # If it's active, show it in green From 28bed0041ec668447d7e26de1870ed8977207b33 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Fri, 8 Dec 2017 18:29:28 -0600 Subject: [PATCH 11/27] remove comment Signed-off-by: Jacob Salmela --- advanced/Scripts/piholeDebug.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index f8955637..64334241 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -212,7 +212,7 @@ copy_to_debug_log() { # uploaded to our server, since it can't properly display in color # This is accomplished by use sed to remove characters matching that patter # The entire file is then copied over to a sanitized version of the log - # sed 's/\[[0-9;]\{1,5\}m//g' > "${PIHOLE_DEBUG_LOG_SANITIZED}" <<< cat "${PIHOLE_DEBUG_LOG}" + sed 's/\[[0-9;]\{1,5\}m//g' > "${PIHOLE_DEBUG_LOG_SANITIZED}" <<< cat "${PIHOLE_DEBUG_LOG}" } initiate_debug() { From 05d015169c19b5da163553ec9791783780b9cd4a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 9 Dec 2017 19:00:46 +0100 Subject: [PATCH 12/27] Check for local version and branches every 10 minutes, check for remote versions only once a day Signed-off-by: DL6ER --- advanced/Scripts/updatecheck.sh | 49 +++++++++++++++++---------------- advanced/pihole.cron | 7 +++-- pihole | 2 +- 3 files changed, 32 insertions(+), 26 deletions(-) diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 9b79c4cb..f8ce59dd 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -3,7 +3,7 @@ # (c) 2017 Pi-hole, LLC (https://pi-hole.net) # Network-wide ad blocking via your own hardware. # -# Checks for updates via GitHub +# Checks for local or remote versions and branches # # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. @@ -25,35 +25,38 @@ function json_extract() { fi } -GITHUB_CORE_VERSION="$(json_extract tag_name "$(curl -q 'https://api.github.com/repos/pi-hole/pi-hole/releases/latest' 2> /dev/null)")" -GITHUB_WEB_VERSION="$(json_extract tag_name "$(curl -q 'https://api.github.com/repos/pi-hole/AdminLTE/releases/latest' 2> /dev/null)")" -GITHUB_FTL_VERSION="$(json_extract tag_name "$(curl -q 'https://api.github.com/repos/pi-hole/FTL/releases/latest' 2> /dev/null)")" - -echo "${GITHUB_CORE_VERSION} ${GITHUB_WEB_VERSION} ${GITHUB_FTL_VERSION}" > "/etc/pihole/GitHubVersions" - function get_local_branch() { # Return active branch cd "${1}" 2> /dev/null || return 1 git rev-parse --abbrev-ref HEAD || return 1 } -CORE_BRANCH="$(get_local_branch /etc/.pihole)" -WEB_BRANCH="$(get_local_branch /var/www/html/admin)" -#FTL_BRANCH="$(pihole-FTL branch)" -# Don't store FTL branch until the next release of FTL which -# supports returning the branch in an easy way -FTL_BRANCH="XXX" - -echo "${CORE_BRANCH} ${WEB_BRANCH} ${FTL_BRANCH}" > "/etc/pihole/localbranches" - function get_local_version() { - # Return active branch - cd "${1}" 2> /dev/null || return 1 - git describe --long --dirty --tags || return 1 +# Return active branch +cd "${1}" 2> /dev/null || return 1 +git describe --long --dirty --tags || return 1 } -CORE_VERSION="$(get_local_version /etc/.pihole)" -WEB_VERSION="$(get_local_version /var/www/html/admin)" -FTL_VERSION="$(pihole-FTL version)" +if [[ "$2" == "remote" ]]; then + + GITHUB_CORE_VERSION="$(json_extract tag_name "$(curl -q 'https://api.github.com/repos/pi-hole/pi-hole/releases/latest' 2> /dev/null)")" + GITHUB_WEB_VERSION="$(json_extract tag_name "$(curl -q 'https://api.github.com/repos/pi-hole/AdminLTE/releases/latest' 2> /dev/null)")" + GITHUB_FTL_VERSION="$(json_extract tag_name "$(curl -q 'https://api.github.com/repos/pi-hole/FTL/releases/latest' 2> /dev/null)")" + + echo "${GITHUB_CORE_VERSION} ${GITHUB_WEB_VERSION} ${GITHUB_FTL_VERSION}" > "/etc/pihole/GitHubVersions" + +else + + CORE_BRANCH="$(get_local_branch /etc/.pihole)" + WEB_BRANCH="$(get_local_branch /var/www/html/admin)" + FTL_BRANCH="$(pihole-FTL branch)" + + echo "${CORE_BRANCH} ${WEB_BRANCH} ${FTL_BRANCH}" > "/etc/pihole/localbranches" + + CORE_VERSION="$(get_local_version /etc/.pihole)" + WEB_VERSION="$(get_local_version /var/www/html/admin)" + FTL_VERSION="$(pihole-FTL version)" + + echo "${CORE_VERSION} ${WEB_VERSION} ${FTL_VERSION}" > "/etc/pihole/localversions" -echo "${CORE_VERSION} ${WEB_VERSION} ${FTL_VERSION}" > "/etc/pihole/localversions" +fi diff --git a/advanced/pihole.cron b/advanced/pihole.cron index 2273358b..87a2bd3c 100644 --- a/advanced/pihole.cron +++ b/advanced/pihole.cron @@ -29,5 +29,8 @@ @reboot root /usr/sbin/logrotate /etc/pihole/logrotate -# Pi-hole: Grab remote version and branch every 10 minutes -*/10 * * * * root PATH="$PATH:/usr/local/bin/" pihole updatechecker +# Pi-hole: Grab local version and branch every 10 minutes +*/10 * * * * root PATH="$PATH:/usr/local/bin/" pihole updatechecker local + +# Pi-hole: Grab remote version every 24 hours +00 00 * * * root PATH="$PATH:/usr/local/bin/" pihole updatechecker remote diff --git a/pihole b/pihole index 652f4acb..e4d6215c 100755 --- a/pihole +++ b/pihole @@ -658,6 +658,6 @@ case "${1}" in "-t" | "tail" ) tailFunc;; "checkout" ) piholeCheckoutFunc "$@";; "tricorder" ) tricorderFunc;; - "updatechecker" ) updateCheckFunc;; + "updatechecker" ) updateCheckFunc "$@";; * ) helpFunc;; esac From 45ab2a3d7aadba25ee29e0f52dc943d9031c60b9 Mon Sep 17 00:00:00 2001 From: Keith Bentrup Date: Sat, 9 Dec 2017 17:03:20 -0500 Subject: [PATCH 13/27] send HTTP headers before HTML Signed-off-by: Keith Bentrup --- advanced/index.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/advanced/index.php b/advanced/index.php index 5c2f250d..999acebb 100644 --- a/advanced/index.php +++ b/advanced/index.php @@ -213,6 +213,8 @@ if (explode("-", $phVersion)[1] != "0") // Please Note: Text is added via CSS to allow an admin to provide a localised // language without the need to edit this file + +setHeader(); ?>