From 545b6605bc03dacc31e3f96d20beea5065c50bfe Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 23 Feb 2020 21:34:12 +0000 Subject: [PATCH 1/5] 4.3.3 (#3154) * Backport ee7090b8fcb11f090741b33eb68822bcc96135e6 to v4 to prevent failures in FTL download * update tests to reflect changes to FTL download URL * backport `tbd` fix Signed-off-by: Adam Warner --- advanced/Scripts/piholeCheckout.sh | 9 ++-- advanced/Scripts/update.sh | 9 +++- automated install/basic-install.sh | 60 ++++++++++++--------- test/test_automated_install.py | 87 +++++++++++------------------- 4 files changed, 81 insertions(+), 84 deletions(-) diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index c4b07a98..dd57117f 100644 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -46,6 +46,12 @@ checkout() { local corebranches local webbranches + # Check if FTL is installed - do this early on as FTL is a hard dependency for Pi-hole + local funcOutput + funcOutput=$(get_binary_name) #Store output of get_binary_name here + local binary + binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL) + # Avoid globbing set -f @@ -86,7 +92,6 @@ checkout() { fi #echo -e " ${TICK} Pi-hole Core" - get_binary_name local path path="development/${binary}" echo "development" > /etc/pihole/ftlbranch @@ -100,7 +105,6 @@ checkout() { fetch_checkout_pull_branch "${webInterfaceDir}" "master" || { echo " ${CROSS} Unable to pull Web master branch"; exit 1; } fi #echo -e " ${TICK} Web Interface" - get_binary_name local path path="master/${binary}" echo "master" > /etc/pihole/ftlbranch @@ -159,7 +163,6 @@ checkout() { fi checkout_pull_branch "${webInterfaceDir}" "${2}" elif [[ "${1}" == "ftl" ]] ; then - get_binary_name local path path="${2}/${binary}" diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index 4d352777..503d3042 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -31,7 +31,6 @@ source "/opt/pihole/COL_TABLE" # make_repo() sourced from basic-install.sh # update_repo() source from basic-install.sh # getGitFiles() sourced from basic-install.sh -# get_binary_name() sourced from basic-install.sh # FTLcheckUpdate() sourced from basic-install.sh GitCheckUpdateAvail() { @@ -194,6 +193,14 @@ main() { ${PI_HOLE_FILES_DIR}/automated\ install/basic-install.sh --reconfigure --unattended || \ echo -e "${basicError}" && exit 1 fi + + if [[ "${FTL_update}" == true || "${core_update}" == true || "${web_update}" == true ]]; then + # Force an update of the updatechecker + /opt/pihole/updatecheck.sh + /opt/pihole/updatecheck.sh x remote + echo -e " ${INFO} Local version file information updated." + fi + echo "" exit 0 } diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f99d02ab..9986df3c 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -139,9 +139,6 @@ else OVER="\\r\\033[K" fi -# Define global binary variable -binary="tbd" - # A simple function that just echoes out our logo in ASCII format # This lets users know that it is a Pi-hole, LLC product show_ascii_berry() { @@ -2154,21 +2151,15 @@ clone_or_update_repos() { } # Download FTL binary to random temp directory and install FTL binary +# Disable directive for SC2120 a value _can_ be passed to this function, but it is passed from an external script that sources this one +# shellcheck disable=SC2120 FTLinstall() { + # Local, named variables local latesttag local str="Downloading and Installing FTL" printf " %b %s..." "${INFO}" "${str}" - # Find the latest version tag for FTL - latesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep "Location" | awk -F '/' '{print $NF}') - # Tags should always start with v, check for that. - if [[ ! "${latesttag}" == v* ]]; then - printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" - printf " %bError: Unable to get latest release location from GitHub%b\\n" "${COL_LIGHT_RED}" "${COL_NC}" - return 1 - fi - # Move into the temp ftl directory pushd "$(mktemp -d)" > /dev/null || { printf "Unable to make temporary directory for FTL binary download\\n"; return 1; } @@ -2184,9 +2175,12 @@ FTLinstall() { ftlBranch="master" fi + local binary + binary="${1}" + # Determine which version of FTL to download if [[ "${ftlBranch}" == "master" ]];then - url="https://github.com/pi-hole/FTL/releases/download/${latesttag%$'\r'}" + url="https://github.com/pi-hole/ftl/releases/latest/download" else url="https://ftl.pi-hole.net/${ftlBranch}" fi @@ -2259,6 +2253,8 @@ get_binary_name() { local machine machine=$(uname -m) + local l_binary + local str="Detecting architecture" printf " %b %s..." "${INFO}" "${str}" # If the machine is arm or aarch @@ -2274,24 +2270,24 @@ get_binary_name() { if [[ "${lib}" == "/lib/ld-linux-aarch64.so.1" ]]; then printf "%b %b Detected ARM-aarch64 architecture\\n" "${OVER}" "${TICK}" # set the binary to be used - binary="pihole-FTL-aarch64-linux-gnu" + l_binary="pihole-FTL-aarch64-linux-gnu" # elif [[ "${lib}" == "/lib/ld-linux-armhf.so.3" ]]; then # if [[ "${rev}" -gt 6 ]]; then printf "%b %b Detected ARM-hf architecture (armv7+)\\n" "${OVER}" "${TICK}" # set the binary to be used - binary="pihole-FTL-arm-linux-gnueabihf" + l_binary="pihole-FTL-arm-linux-gnueabihf" # Otherwise, else printf "%b %b Detected ARM-hf architecture (armv6 or lower) Using ARM binary\\n" "${OVER}" "${TICK}" # set the binary to be used - binary="pihole-FTL-arm-linux-gnueabi" + l_binary="pihole-FTL-arm-linux-gnueabi" fi else printf "%b %b Detected ARM architecture\\n" "${OVER}" "${TICK}" # set the binary to be used - binary="pihole-FTL-arm-linux-gnueabi" + l_binary="pihole-FTL-arm-linux-gnueabi" fi elif [[ "${machine}" == "x86_64" ]]; then # This gives the architecture of packages dpkg installs (for example, "i386") @@ -2304,12 +2300,12 @@ get_binary_name() { # in the past (see https://github.com/pi-hole/pi-hole/pull/2004) if [[ "${dpkgarch}" == "i386" ]]; then printf "%b %b Detected 32bit (i686) architecture\\n" "${OVER}" "${TICK}" - binary="pihole-FTL-linux-x86_32" + l_binary="pihole-FTL-linux-x86_32" else # 64bit printf "%b %b Detected x86_64 architecture\\n" "${OVER}" "${TICK}" # set the binary to be used - binary="pihole-FTL-linux-x86_64" + l_binary="pihole-FTL-linux-x86_64" fi else # Something else - we try to use 32bit executable and warn the user @@ -2320,13 +2316,13 @@ get_binary_name() { else printf "%b %b Detected 32bit (i686) architecture\\n" "${OVER}" "${TICK}" fi - binary="pihole-FTL-linux-x86_32" + l_binary="pihole-FTL-linux-x86_32" fi + + echo ${l_binary} } FTLcheckUpdate() { - get_binary_name - #In the next section we check to see if FTL is already installed (in case of pihole -r). #If the installed version matches the latest version, then check the installed sha1sum of the binary vs the remote sha1sum. If they do not match, then download printf " %b Checking for existing FTL binary...\\n" "${INFO}" @@ -2342,6 +2338,9 @@ FTLcheckUpdate() { ftlBranch="master" fi + local binary + binary="${1}" + local remoteSha1 local localSha1 @@ -2420,8 +2419,10 @@ FTLcheckUpdate() { FTLdetect() { printf "\\n %b FTL Checks...\\n\\n" "${INFO}" - if FTLcheckUpdate ; then - FTLinstall || return 1 + printf " %b" "${2}" + + if FTLcheckUpdate "${1}"; then + FTLinstall "${1}" || return 1 fi } @@ -2581,8 +2582,17 @@ main() { else LIGHTTPD_ENABLED=false fi + # Create the pihole user + create_pihole_user + # Check if FTL is installed - do this early on as FTL is a hard dependency for Pi-hole - if ! FTLdetect; then + local funcOutput + funcOutput=$(get_binary_name) #Store output of get_binary_name here + local binary + binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL) + local theRest + theRest="${funcOutput%pihole-FTL*}" # Print the rest of get_binary_name's output to display (cut out from first instance of "pihole-FTL") + if ! FTLdetect "${binary}" "${theRest}"; then printf " %b FTL Engine not installed\\n" "${CROSS}" exit 1 fi diff --git a/test/test_automated_install.py b/test/test_automated_install.py index be8fdcfa..0c304c40 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -398,7 +398,11 @@ def test_FTL_detect_aarch64_no_errors(Pihole): ) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh - FTLdetect + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" ''') expected_stdout = info_box + ' FTL Checks...' assert expected_stdout in detectPlatform.stdout @@ -418,7 +422,11 @@ def test_FTL_detect_armv6l_no_errors(Pihole): mock_command('ldd', {'/bin/ls': ('/lib/ld-linux-armhf.so.3', '0')}, Pihole) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh - FTLdetect + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" ''') expected_stdout = info_box + ' FTL Checks...' assert expected_stdout in detectPlatform.stdout @@ -439,7 +447,11 @@ def test_FTL_detect_armv7l_no_errors(Pihole): mock_command('ldd', {'/bin/ls': ('/lib/ld-linux-armhf.so.3', '0')}, Pihole) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh - FTLdetect + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" ''') expected_stdout = info_box + ' FTL Checks...' assert expected_stdout in detectPlatform.stdout @@ -455,7 +467,11 @@ def test_FTL_detect_x86_64_no_errors(Pihole): ''' detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh - FTLdetect + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" ''') expected_stdout = info_box + ' FTL Checks...' assert expected_stdout in detectPlatform.stdout @@ -471,7 +487,11 @@ def test_FTL_detect_unknown_no_errors(Pihole): mock_command('uname', {'-m': ('mips', '0')}, Pihole) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh - FTLdetect + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" ''') expected_stdout = 'Not able to detect architecture (unknown: mips)' assert expected_stdout in detectPlatform.stdout @@ -490,68 +510,25 @@ def test_FTL_download_aarch64_no_errors(Pihole): ''') download_binary = Pihole.run(''' source /opt/pihole/basic-install.sh - binary="pihole-FTL-aarch64-linux-gnu" - FTLinstall + create_pihole_user + FTLinstall "pihole-FTL-aarch64-linux-gnu" ''') expected_stdout = tick_box + ' Downloading and Installing FTL' assert expected_stdout in download_binary.stdout assert 'error' not in download_binary.stdout.lower() -def test_FTL_download_unknown_fails_no_errors(Pihole): - ''' - confirms unknown binary is not downloaded for FTL engine - ''' - # mock whiptail answers and ensure installer dependencies - mock_command('whiptail', {'*': ('', '0')}, Pihole) - Pihole.run(''' - source /opt/pihole/basic-install.sh - distro_check - install_dependent_packages ${INSTALLER_DEPS[@]} - ''') - download_binary = Pihole.run(''' - source /opt/pihole/basic-install.sh - binary="pihole-FTL-mips" - FTLinstall - ''') - expected_stdout = cross_box + ' Downloading and Installing FTL' - assert expected_stdout in download_binary.stdout - error1 = 'Error: URL https://github.com/pi-hole/FTL/releases/download/' - assert error1 in download_binary.stdout - error2 = 'not found' - assert error2 in download_binary.stdout - - -def test_FTL_download_binary_unset_no_errors(Pihole): - ''' - confirms unset binary variable does not download FTL engine - ''' - # mock whiptail answers and ensure installer dependencies - mock_command('whiptail', {'*': ('', '0')}, Pihole) - Pihole.run(''' - source /opt/pihole/basic-install.sh - distro_check - install_dependent_packages ${INSTALLER_DEPS[@]} - ''') - download_binary = Pihole.run(''' - source /opt/pihole/basic-install.sh - FTLinstall - ''') - expected_stdout = cross_box + ' Downloading and Installing FTL' - assert expected_stdout in download_binary.stdout - error1 = 'Error: URL https://github.com/pi-hole/FTL/releases/download/' - assert error1 in download_binary.stdout - error2 = 'not found' - assert error2 in download_binary.stdout - - def test_FTL_binary_installed_and_responsive_no_errors(Pihole): ''' confirms FTL binary is copied and functional in installed location ''' installed_binary = Pihole.run(''' source /opt/pihole/basic-install.sh - FTLdetect + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" pihole-FTL version ''') expected_stdout = 'v' From 707e21b92755e9e75a5b9b1f2b19c6cc9c1b1ca9 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 4 Dec 2019 20:09:34 +0000 Subject: [PATCH 2/5] :dominik: Detect binary name before calling FTLcheckUpdate in update.sh Signed-off-by: Adam Warner --- advanced/Scripts/update.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index 503d3042..f833fc2f 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -128,7 +128,12 @@ main() { fi fi - if FTLcheckUpdate > /dev/null; then + local funcOutput + funcOutput=$(get_binary_name) #Store output of get_binary_name here + local binary + binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL) + + if FTLcheckUpdate "${binary}" > /dev/null; then FTL_update=true echo -e " ${INFO} FTL:\\t\\t${COL_YELLOW}update available${COL_NC}" else From 0fbcc6d8b5f23065d33870f069d5b919fd0cd52a Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Mon, 24 Feb 2020 09:38:37 -0800 Subject: [PATCH 3/5] Compare daemons to expected results. (#3158) Signed-off-by: Dan Schaper --- advanced/Scripts/piholeDebug.sh | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 1010f26c..d46944d6 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -643,19 +643,21 @@ ping_internet() { } compare_port_to_service_assigned() { - local service_name="${1}" - # The programs we use may change at some point, so they are in a varible here - local resolver="pihole-FTL" - local web_server="lighttpd" - local ftl="pihole-FTL" + local service_name + local expected_service + local port + + service_name="${2}" + expected_service="${1}" + port="${3}" # If the service is a Pi-hole service, highlight it in green - if [[ "${service_name}" == "${resolver}" ]] || [[ "${service_name}" == "${web_server}" ]] || [[ "${service_name}" == "${ftl}" ]]; then - log_write "[${COL_GREEN}${port_number}${COL_NC}] is in use by ${COL_GREEN}${service_name}${COL_NC}" + if [[ "${service_name}" == "${expected_service}" ]]; then + log_write "[${COL_GREEN}${port}${COL_NC}] is in use by ${COL_GREEN}${service_name}${COL_NC}" # Otherwise, else # Show the service name in red since it's non-standard - log_write "[${COL_RED}${port_number}${COL_NC}] is in use by ${COL_RED}${service_name}${COL_NC} (${FAQ_HARDWARE_REQUIREMENTS_PORTS})" + log_write "[${COL_RED}${port}${COL_NC}] is in use by ${COL_RED}${service_name}${COL_NC} (${FAQ_HARDWARE_REQUIREMENTS_PORTS})" fi } @@ -689,11 +691,11 @@ check_required_ports() { fi # Use a case statement to determine if the right services are using the right ports case "$(echo "$port_number" | rev | cut -d: -f1 | rev)" in - 53) compare_port_to_service_assigned "${resolver}" + 53) compare_port_to_service_assigned "${resolver}" "${service_name}" 53 ;; - 80) compare_port_to_service_assigned "${web_server}" + 80) compare_port_to_service_assigned "${web_server}" "${service_name}" 80 ;; - 4711) compare_port_to_service_assigned "${ftl}" + 4711) compare_port_to_service_assigned "${ftl}" "${service_name}" 4711 ;; # If it's not a default port that Pi-hole needs, just print it out for the user to see *) log_write "${port_number} ${service_name} (${protocol_type})"; From b4c2bf678f4715da45cd22dcf56ca68d1a877112 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 24 Feb 2020 20:02:48 +0000 Subject: [PATCH 4/5] Safeguard against colour output in grep commandadd -i to grep to make search for "Location" case-insensitive Signed-off-by: Adam Warner --- automated install/basic-install.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9986df3c..ee4793e7 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2382,17 +2382,14 @@ FTLcheckUpdate() { if [[ ${ftlLoc} ]]; then local FTLversion FTLversion=$(/usr/bin/pihole-FTL tag) - local FTLreleaseData local FTLlatesttag - if ! FTLreleaseData=$(curl -sI https://github.com/pi-hole/FTL/releases/latest); then + if ! FTLlatesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep --color=never -i Location | awk -F / '{print $NF}' | tr -d '[:cntrl:]'); then # There was an issue while retrieving the latest version printf " %b Failed to retrieve latest FTL release metadata" "${CROSS}" return 3 fi - FTLlatesttag=$(grep 'Location' <<< "${FTLreleaseData}" | awk -F '/' '{print $NF}' | tr -d '\r\n') - if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then return 0 else From 1c74b41869887afb87cc818e1be54d5a108db866 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 7 Sep 2019 23:11:20 +0200 Subject: [PATCH 5/5] Add use-application-dns.net = NXDOMAIN in ProcessDNSSettings rather than in the template so we can ensure that it will survive config-renewals. Signed-off-by: DL6ER --- advanced/Scripts/webpage.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 600a45a5..3b17b6b5 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -211,6 +211,11 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423 add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_DOMAIN}/${CONDITIONAL_FORWARDING_IP}" add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_REVERSE}/${CONDITIONAL_FORWARDING_IP}" fi + + # Prevent Firefox from automatically switching over to DNS-over-HTTPS + # This follows https://support.mozilla.org/en-US/kb/configuring-networks-disable-dns-over-https + # (sourced 7th September 2019) + add_dnsmasq_setting "server=/use-application-dns.net/" } SetDNSServers() {