From eb5661b553e759b926e9672842e364c7d761363b Mon Sep 17 00:00:00 2001 From: MichaIng Date: Tue, 7 Jul 2020 20:07:11 +0200 Subject: [PATCH 1/7] Further failsafe check for available APT packages - "apt-cache show package" succeeds as well if package is listed as (optional) dependency or conflict by another package, hence is not a 100% reliable measure. - There is no command which explicitly checks which package/name can be selected by apt-get for install. An install simulation/dry-run is possible as it was before Pi-hole v5.1, or the whole package cache can be scraped, which is still the less time consuming solution. - Allow to succeed if another package "provides" it, like "php7.3-apcu" provided by "php-apcu" or "awk" provided by "mawk" and "gawk", in which case the non-virtual package is selected automatically by apt-get. For reference: https://github.com/MichaIng/DietPi/pull/3657/commits/066b89fa410ff568f57ad31fa8f50ec72c97796d Signed-off-by: MichaIng --- automated install/basic-install.sh | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 45e96354..d0736bec 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -85,6 +85,8 @@ QUERY_LOGGING=true INSTALL_WEB_INTERFACE=true PRIVACY_LEVEL=0 CACHE_SIZE=10000 +# Placeholder variable for the list of available APT packages to be parsed subsequently +APT_PACKAGE_LIST="" if [ -z "${USER}" ]; then USER="$(id -un)" @@ -179,6 +181,19 @@ is_command() { command -v "${check_command}" >/dev/null 2>&1 } +is_apt_package(){ + # Checks whether a package, or one that provides it, is available in + # the installed APT repository lists. + local check_package=$1 + + # Obtain the list of available packages once + if [[ -z $APT_PACKAGE_LIST ]]; then + APT_PACKAGE_LIST=$(apt-cache dumpavail | grep -E '^P(ackage|rovides):') + fi + + grep -qE " $check_package(,|$)" <<< "$APT_PACKAGE_LIST" +} + os_check() { if [ "$PIHOLE_SKIP_OS_CHECK" != true ]; then # This function gets a list of supported OS versions from a TXT record at versions.pi-hole.net @@ -303,10 +318,10 @@ if is_command apt-get ; then # Update package cache. This is required already here to assure apt-cache calls have package lists available. update_package_cache || exit 1 # Debian 7 doesn't have iproute2 so check if it's available first - if apt-cache show iproute2 > /dev/null 2>&1; then + if is_apt_package iproute2; then iproute_pkg="iproute2" # Otherwise, check if iproute is available - elif apt-cache show iproute > /dev/null 2>&1; then + elif is_apt_package iproute; then iproute_pkg="iproute" # Else print error and exit else @@ -326,10 +341,10 @@ if is_command apt-get ; then # Check if installed php is v 7.0, or newer to determine packages to install if [[ "$phpInsNewer" != true ]]; then # Prefer the php metapackage if it's there - if apt-cache show php > /dev/null 2>&1; then + if is_apt_package php; then phpVer="php" # Else fall back on the php5 package if it's there - elif apt-cache show php5 > /dev/null 2>&1; then + elif is_apt_package php5; then phpVer="php5" # Else print error and exit else @@ -341,9 +356,9 @@ if is_command apt-get ; then phpVer="php$phpInsMajor.$phpInsMinor" fi # We also need the correct version for `php-sqlite` (which differs across distros) - if apt-cache show "${phpVer}-sqlite3" > /dev/null 2>&1; then + if is_apt_package "${phpVer}-sqlite3"; then phpSqlite="sqlite3" - elif apt-cache show "${phpVer}-sqlite" > /dev/null 2>&1; then + elif is_apt_package "${phpVer}-sqlite"; then phpSqlite="sqlite" else printf " %b Aborting installation: No SQLite PHP module was found in APT repository.\\n" "${CROSS}" From ef0bdf6470e98ac7866c4fbcee3ab06bfe095fe8 Mon Sep 17 00:00:00 2001 From: Matej Dujava Date: Mon, 7 Dec 2020 00:23:04 +0100 Subject: [PATCH 2/7] Fix validation of adlist url Already existing regex validation will be used on url after removing @ (in case its in separating userinfo and host). Signed-off-by: Matej Dujava Fixes: https://github.com/pi-hole/pi-hole/issues/3911 Fixes: 7d19ee1b: validate blocklist URL before adding to the database (#3237) --- advanced/Scripts/webpage.sh | 11 ++++++++--- gravity.sh | 9 +++++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index f26ce11d..da2afb0f 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -486,10 +486,15 @@ SetWebUITheme() { } CheckUrl(){ - local regex + local regex check_url # Check for characters NOT allowed in URLs - regex="[^a-zA-Z0-9:/?&%=~._-]" - if [[ "${1}" =~ ${regex} ]]; then + regex="[^a-zA-Z0-9:/?&%=~._()-;]" + + # this will remove first @ that is after schema and before domain + # \1 is optional schema, \2 is userinfo + check_url="$( sed -re 's#([^:/]*://)?([^/]+)@#\1\2#' <<< "$1" )" + + if [[ "${check_url}" =~ ${regex} ]]; then return 1 else return 0 diff --git a/gravity.sh b/gravity.sh index 9ac38512..c9cd615a 100755 --- a/gravity.sh +++ b/gravity.sh @@ -393,10 +393,15 @@ gravity_DownloadBlocklists() { esac echo -e " ${INFO} Target: ${url}" - local regex + local regex check_url # Check for characters NOT allowed in URLs regex="[^a-zA-Z0-9:/?&%=~._()-;]" - if [[ "${url}" =~ ${regex} ]]; then + + # this will remove first @ that is after schema and before domain + # \1 is optional schema, \2 is userinfo + check_url="$( sed -re 's#([^:/]*://)?([^/]+)@#\1\2#' <<< "$url" )" + + if [[ "${check_url}" =~ ${regex} ]]; then echo -e " ${CROSS} Invalid Target" else gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" From b4102547accaf7c69177c317f8ac4d5d50fdd63f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 21 Dec 2020 13:11:59 +0100 Subject: [PATCH 3/7] Remove deprecated malwaredomains list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 45e96354..414945f8 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1287,8 +1287,7 @@ chooseBlocklists() { # Let user select (or not) blocklists via a checklist cmd=(whiptail --separate-output --checklist "Pi-hole relies on third party lists in order to block ads.\\n\\nYou can use the suggestions below, and/or add your own after installation\\n\\nTo deselect any list, use the arrow keys and spacebar" "${r}" "${c}" 5) # In an array, show the options available (all off by default): - options=(StevenBlack "StevenBlack's Unified Hosts List" on - MalwareDom "MalwareDomains" on) + options=(StevenBlack "StevenBlack's Unified Hosts List" on) # In a variable, show the choices available; exit if Cancel is selected choices=$("${cmd[@]}" "${options[@]}" 2>&1 >/dev/tty) || { printf " %bCancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; rm "${adlistFile}" ;exit 1; } @@ -1307,7 +1306,6 @@ chooseBlocklists() { appendToListsFile() { case $1 in StevenBlack ) echo "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" >> "${adlistFile}";; - MalwareDom ) echo "https://mirror1.malwaredomains.com/files/justdomains" >> "${adlistFile}";; esac } @@ -1320,7 +1318,6 @@ installDefaultBlocklists() { return; fi appendToListsFile StevenBlack - appendToListsFile MalwareDom } # Check if /etc/dnsmasq.conf is from pi-hole. If so replace with an original and install new in .d directory From 0d710fc9e31b407977b6ab256d22bd4dfaae7baa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 24 Dec 2020 21:19:14 +0100 Subject: [PATCH 4/7] Change wording in whiptail for adlist selection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index d2c5d078..b0b7bdd8 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1285,7 +1285,7 @@ chooseBlocklists() { mv "${adlistFile}" "${adlistFile}.old" fi # Let user select (or not) blocklists via a checklist - cmd=(whiptail --separate-output --checklist "Pi-hole relies on third party lists in order to block ads.\\n\\nYou can use the suggestions below, and/or add your own after installation\\n\\nTo deselect any list, use the arrow keys and spacebar" "${r}" "${c}" 5) + cmd=(whiptail --separate-output --checklist "Pi-hole relies on third party lists in order to block ads.\\n\\nYou can use the suggestion below, and/or add your own after installation\\n\\nTo deselect the suggested list, use spacebar" "${r}" "${c}" 5) # In an array, show the options available (all off by default): options=(StevenBlack "StevenBlack's Unified Hosts List" on) From e033ee666423a89498a03da57c74b95ee49c883f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 29 Dec 2020 10:16:43 +0100 Subject: [PATCH 5/7] Fix connection mechanism between Chronometer and FTL Signed-off-by: DL6ER --- advanced/Scripts/chronometer.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh index 3e77f578..3126ed25 100755 --- a/advanced/Scripts/chronometer.sh +++ b/advanced/Scripts/chronometer.sh @@ -13,6 +13,7 @@ LC_NUMERIC=C # Retrieve stats from FTL engine pihole-FTL() { + local ftl_port LINE ftl_port=$(cat /run/pihole-FTL.port 2> /dev/null) if [[ -n "$ftl_port" ]]; then # Open connection to FTL @@ -20,12 +21,13 @@ pihole-FTL() { # Test if connection is open if { "true" >&3; } 2> /dev/null; then - # Send command to FTL - echo -e ">$1" >&3 + # Send command to FTL and ask to quit when finished + echo -e ">$1 >quit" >&3 - # Read input + # Read input until we received an empty string and the connection is + # closed read -r -t 1 LINE <&3 - until [[ ! $? ]] || [[ "$LINE" == *"EOM"* ]]; do + until [[ -z "${LINE}" ]] && [[ ! -t 3 ]]; do echo "$LINE" >&1 read -r -t 1 LINE <&3 done From a5422dbdf6bb8fe0d68095e69c9ec37796d3c838 Mon Sep 17 00:00:00 2001 From: bcambl Date: Wed, 30 Dec 2020 19:45:51 -0600 Subject: [PATCH 6/7] fix release check for centos stream fixes #3947 #3953 Signed-off-by: bcambl --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index b0b7bdd8..efa8c23d 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -408,7 +408,7 @@ elif is_command rpm ; then SUPPORTED_CENTOS_VERSION=7 SUPPORTED_CENTOS_PHP_VERSION=7 # Check current CentOS major release version - CURRENT_CENTOS_VERSION=$(grep -oP '(?<= )[0-9]+(?=\.)' /etc/redhat-release) + CURRENT_CENTOS_VERSION=$(grep -oP '(?<= )[0-9]+(?=\.?)' /etc/redhat-release) # Check if CentOS version is supported if [[ $CURRENT_CENTOS_VERSION -lt $SUPPORTED_CENTOS_VERSION ]]; then printf " %b CentOS %s is not supported.\\n" "${CROSS}" "${CURRENT_CENTOS_VERSION}" From 523f6501576f76bfcc7e5d3ddc3cf0a287089790 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Tue, 12 Jan 2021 13:42:51 -0800 Subject: [PATCH 7/7] Use the 'Location:' header only. Signed-off-by: Dan Schaper --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 1f5784c7..37cba335 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2555,7 +2555,7 @@ FTLcheckUpdate() { FTLversion=$(/usr/bin/pihole-FTL tag) local FTLlatesttag - if ! FTLlatesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep --color=never -i Location | awk -F / '{print $NF}' | tr -d '[:cntrl:]'); then + if ! FTLlatesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep --color=never -i Location: | awk -F / '{print $NF}' | tr -d '[:cntrl:]'); then # There was an issue while retrieving the latest version printf " %b Failed to retrieve latest FTL release metadata" "${CROSS}" return 3