From f713b14ba09cafa489593da07a5f3ddf66b813f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 8 Dec 2022 09:49:02 +0100 Subject: [PATCH 01/24] Cleanup if startup failed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL.service | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service index 15096972..dc7649e7 100644 --- a/advanced/Templates/pihole-FTL.service +++ b/advanced/Templates/pihole-FTL.service @@ -23,6 +23,11 @@ is_running() { return 1 } +cleanup() { + # Run post-stop script, which does cleanup among runtime files + sh "${PI_HOLE_SCRIPT_DIR}/pihole-FTL-poststop.sh" +} + # Start the service start() { @@ -33,10 +38,15 @@ start() { sh "${PI_HOLE_SCRIPT_DIR}/pihole-FTL-prestart.sh" if setcap CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_NICE,CAP_IPC_LOCK,CAP_CHOWN+eip "/usr/bin/pihole-FTL"; then - su -s /bin/sh -c "/usr/bin/pihole-FTL" pihole || exit $? + su -s /bin/sh -c "/usr/bin/pihole-FTL" pihole || ec=$? else echo "Warning: Starting pihole-FTL as root because setting capabilities is not supported on this system" - /usr/bin/pihole-FTL || exit $? + /usr/bin/pihole-FTL || ec=$? + fi + # Cleanup if startup failed + if [ -n "${ec}" ] && [ "${ec}" != 0 ]; then + cleanup + exit $ec fi echo fi @@ -65,8 +75,7 @@ stop() { else echo "Not running" fi - # Run post-stop script, which does cleanup among runtime files - sh "${PI_HOLE_SCRIPT_DIR}/pihole-FTL-poststop.sh" + cleanup echo } @@ -84,6 +93,9 @@ status() { ### main logic ### +# catch sudden termination +trap 'cleanup; exit 1' INT HUP TERM ABRT + # Get FTL's PID file path FTL_PID_FILE="$(getFTLPIDFile)" From c59e11a332358dbf8b4a6d67d844a85ca47c9bc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 24 Nov 2022 20:53:15 +0100 Subject: [PATCH 02/24] Always set lighttpd config dir permissions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index e29afad9..a81ccf0e 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1399,6 +1399,9 @@ installConfigs() { install -T -m 0755 "${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole-FTL-poststop.sh" "${PI_HOLE_INSTALL_DIR}/pihole-FTL-poststop.sh" # If the user chose to install the dashboard, + # set permissions on /etc/lighttpd/lighttpd.conf so pihole user (other) can read the file + chmod o+x /etc/lighttpd + chmod o+r "${lighttpdConfig}" if [[ "${INSTALL_WEB_SERVER}" == true ]]; then if grep -q -F "FILE AUTOMATICALLY OVERWRITTEN BY PI-HOLE" "${lighttpdConfig}"; then # Attempt to preserve backwards compatibility with older versions From b9ebb0524679e6ad1d318023c9f63ee5de8410f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 28 Dec 2022 13:42:57 +0100 Subject: [PATCH 03/24] Set permission after we know the user wants to install the web server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a81ccf0e..fcfb2405 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1399,10 +1399,10 @@ installConfigs() { install -T -m 0755 "${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole-FTL-poststop.sh" "${PI_HOLE_INSTALL_DIR}/pihole-FTL-poststop.sh" # If the user chose to install the dashboard, + if [[ "${INSTALL_WEB_SERVER}" == true ]]; then # set permissions on /etc/lighttpd/lighttpd.conf so pihole user (other) can read the file chmod o+x /etc/lighttpd chmod o+r "${lighttpdConfig}" - if [[ "${INSTALL_WEB_SERVER}" == true ]]; then if grep -q -F "FILE AUTOMATICALLY OVERWRITTEN BY PI-HOLE" "${lighttpdConfig}"; then # Attempt to preserve backwards compatibility with older versions install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} "${lighttpdConfig}" From 9e47b61c8f6e0cf6c478a208b81542ee7ed0633d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 3 Jan 2023 21:20:22 +0100 Subject: [PATCH 04/24] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL.service | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service index dc7649e7..460339ae 100644 --- a/advanced/Templates/pihole-FTL.service +++ b/advanced/Templates/pihole-FTL.service @@ -38,15 +38,16 @@ start() { sh "${PI_HOLE_SCRIPT_DIR}/pihole-FTL-prestart.sh" if setcap CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_NICE,CAP_IPC_LOCK,CAP_CHOWN+eip "/usr/bin/pihole-FTL"; then - su -s /bin/sh -c "/usr/bin/pihole-FTL" pihole || ec=$? + su -s /bin/sh -c "/usr/bin/pihole-FTL" pihole else echo "Warning: Starting pihole-FTL as root because setting capabilities is not supported on this system" - /usr/bin/pihole-FTL || ec=$? + /usr/bin/pihole-FTL fi + rc=$? # Cleanup if startup failed - if [ -n "${ec}" ] && [ "${ec}" != 0 ]; then + if [ "${rc}" != 0 ]; then cleanup - exit $ec + exit $rc fi echo fi From 7bb86e41189682d4c2fb502e672bd1dce99fb3c3 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 18 Jan 2023 22:12:29 +0000 Subject: [PATCH 05/24] Add a check for the version number of lighttpd. Only remove the server.modules += section of the new pihole-admin config if the version number is not greater than or equal to 1.4.56 Signed-off-by: Adam Warner --- advanced/lighttpd.conf.debian | 1 - automated install/basic-install.sh | 19 ++++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/advanced/lighttpd.conf.debian b/advanced/lighttpd.conf.debian index 06c284fe..61d20821 100644 --- a/advanced/lighttpd.conf.debian +++ b/advanced/lighttpd.conf.debian @@ -17,7 +17,6 @@ server.modules = ( "mod_access", - "mod_accesslog", "mod_auth", "mod_expire", "mod_redirect", diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 3b37bf28..44b4b313 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1432,14 +1432,23 @@ installConfigs() { elif [[ -d "/etc/lighttpd/conf-available" ]]; then conf=/etc/lighttpd/conf-available/15-pihole-admin.conf install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/pihole-admin.conf $conf - # disable server.modules += ( ... ) in $conf to avoid module dups - # (needed until Debian 10 no longer supported by pi-hole) - # (server.modules duplication is ignored in lighttpd 1.4.56+) - if awk '!/^server\.modules/{print}' $conf > $conf.$$ && mv $conf.$$ $conf; then + + # Get the version number of lighttpd + version=$(lighttpd -v | grep -o -E '[0-9]+\.[0-9]+\.[0-9]+') + # Test if that version is greater than or euqal to 1.4.56 + if dpkg --compare-versions "$version" "ge" "1.4.56"; then + # If it is, then we don't need to disable the modules : else - rm $conf.$$ + # disable server.modules += ( ... ) in $conf to avoid module dups + # (server.modules duplication is ignored in lighttpd 1.4.56+) + if awk '!/^server\.modules/{print}' $conf > $conf.$$ && mv $conf.$$ $conf; then + : + else + rm $conf.$$ + fi fi + chmod 644 $conf if is_command lighty-enable-mod ; then lighty-enable-mod pihole-admin access accesslog redirect fastcgi setenv > /dev/null || true From 771b7cfcc7aafb3c9d902f2d22a039a0ba6e145c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 19 Jan 2023 21:41:32 +0100 Subject: [PATCH 06/24] Run updatechecker after gravity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 3b37bf28..be906ea1 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2717,12 +2717,12 @@ main() { restart_service pihole-FTL - # Update local and remote versions via updatechecker - /opt/pihole/updatecheck.sh - # Download and compile the aggregated block list runGravity + # Update local and remote versions via updatechecker + /opt/pihole/updatecheck.sh + if [[ "${useUpdateVars}" == false ]]; then displayFinalMessage "${pw}" fi From 79f4a7cef094ae7661ddfe2816f84b0585957523 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 20 Jan 2023 18:52:14 +0000 Subject: [PATCH 07/24] Maintain a list of valid hostnames (taken from the top of StevenBlack's host file), and discount them when calculating the number of "invalid" domains in a given list Soften the output message when reporting on unsuable domains Signed-off-by: Adam Warner --- gravity.sh | 67 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 48 insertions(+), 19 deletions(-) diff --git a/gravity.sh b/gravity.sh index a5c944ce..e8dfb34d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -244,7 +244,7 @@ database_adlist_number() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${num_source_lines}" "${num_invalid}" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${num_source_lines}" "${num_unusable}" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -522,9 +522,9 @@ gravity_DownloadBlocklists() { # num_target_lines does increase for every correctly added domain in pareseList() num_target_lines=0 num_source_lines=0 -num_invalid=0 +num_unusable=0 parseList() { - local adlistID="${1}" src="${2}" target="${3}" incorrect_lines sample_incorrect_lines + local adlistID="${1}" src="${2}" target="${3}" unusable_lines sample_unusable_lines tmp_unusuable_lines_str false_positive # This sed does the following things: # 1. Remove all lines containing no domains # 2. Remove all domains containing invalid characters. Valid are: a-z, A-Z, 0-9, dot (.), minus (-), underscore (_) @@ -534,10 +534,41 @@ parseList() { sed -r "/([^\.]+\.)+[^\.]{2,}/!d;/[^a-zA-Z0-9.\_-]/d;s/\.$//;s/$/,${adlistID}/;/.$/a\\" "${src}" >> "${target}" # Find lines containing no domains or with invalid characters (see above) - # Remove duplicates and limit to 5 domains - mapfile -t incorrect_lines <<< "$(sed -r "/([^\.]+\.)+[^\.]{2,}/d" < "${src}")" - mapfile -t -O "${#incorrect_lines[@]}" incorrect_lines <<< "$(sed -r "/[^a-zA-Z0-9.\_-]/!d" < "${src}")" - IFS=" " read -r -a sample_incorrect_lines <<< "$(tr ' ' '\n' <<< "${incorrect_lines[@]}" | sort -u | head -n 5| tr '\n' ' ')" + # Remove duplicates from the list + mapfile -t unusable_lines <<< "$(sed -r "/([^\.]+\.)+[^\.]{2,}/d" < "${src}")" + mapfile -t -O "${#unusable_lines[@]}" unusable_lines <<< "$(sed -r "/[^a-zA-Z0-9.\_-]/!d" < "${src}")" + IFS=" " read -r -a unusable_lines <<< "$(tr ' ' '\n' <<< "${unusable_lines[@]}" | sort -u | tr '\n' ' ')" + + # A list of items of common local hostnames not to report as unusable + # Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files + # but flagging them as unusable causes more confusion than it's worth - so we suppress them from the output + false_positives=( + "localhost" + "localhost.localdomain" + "local" + "broadcasthost" + "localhost" + "ip6-localhost" + "ip6-loopback" + "lo0 localhost" + "ip6-localnet" + "ip6-mcastprefix" + "ip6-allnodes" + "ip6-allrouters" + "ip6-allhosts" + ) + + # Read the unusable lines into a string + tmp_unusuable_lines_str=" ${unusable_lines[*]} " + for false_positive in "${false_positives[@]}"; do + # Remove false positives from tmp_unusuable_lines_str + tmp_unusuable_lines_str="${tmp_unusuable_lines_str/ ${false_positive} / }" + done + # Read the string back into an array + IFS=" " read -r -a unusable_lines <<< "${tmp_unusuable_lines_str}" + + # Get a sample of the incorrect lines, limited to 5 (the list should already have been de-duplicated) + IFS=" " read -r -a sample_unusable_lines <<< "$(tr ' ' '\n' <<< "${unusable_lines[@]}" | head -n 5 | tr '\n' ' ')" local num_target_lines_new num_correct_lines # Get number of lines in source file @@ -548,22 +579,20 @@ parseList() { num_correct_lines="$(( num_target_lines_new-num_target_lines ))" # Update number of lines in target file num_target_lines="$num_target_lines_new" - num_invalid="$(( num_source_lines-num_correct_lines ))" - if [[ "${num_invalid}" -eq 0 ]]; then - echo " ${INFO} Analyzed ${num_source_lines} domains" - else - echo " ${INFO} Analyzed ${num_source_lines} domains, ${num_invalid} domains invalid!" - fi + num_unusable="${#unusable_lines[@]}" - # Display sample of invalid lines if we found some - if [ ${#sample_incorrect_lines[@]} -ne 0 ]; then - echo " Sample of invalid domains:" - for each in "${sample_incorrect_lines[@]}" + if [[ "${num_unusable}" -ne 0 ]]; then + echo " ${INFO} Imported ${num_correct_lines} domains, ignoring ${num_unusable} non-domain entries" + echo " Sample of non-domain entries:" + for each in "${sample_unusable_lines[@]}" do - echo " - ${each}" + echo " - ${each}" done + else + echo " ${INFO} Imported ${num_correct_lines} domains" fi } + compareLists() { local adlistID="${1}" target="${2}" @@ -717,7 +746,7 @@ gravity_DownloadBlocklistFromUrl() { echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" # Manually reset these two numbers because we do not call parseList here num_source_lines=0 - num_invalid=0 + num_unusable=0 database_adlist_number "${adlistID}" database_adlist_status "${adlistID}" "4" fi From d6e25403eeb348271dce571642170ebcf5b54098 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 21 Jan 2023 10:01:41 +0000 Subject: [PATCH 08/24] Bump tox from 4.2.8 to 4.3.5 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.2.8 to 4.3.5. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.2.8...4.3.5) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index e891242c..686f2a58 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ docker-compose == 1.29.2 pytest == 7.2.1 pytest-xdist == 3.1.0 pytest-testinfra == 7.0.0 -tox == 4.2.8 +tox == 4.3.5 From d30a5f1b950a5a1b8efcf93ccc4f80bf90e1e706 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Thu, 19 Jan 2023 12:24:57 +0000 Subject: [PATCH 09/24] Get the lighttpd version from `dpkg-query` instead Signed-off-by: Adam Warner --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 44b4b313..20fd997d 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1434,14 +1434,14 @@ installConfigs() { install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/pihole-admin.conf $conf # Get the version number of lighttpd - version=$(lighttpd -v | grep -o -E '[0-9]+\.[0-9]+\.[0-9]+') + version=$(dpkg-query -f='${Version}\n' --show lighttpd) # Test if that version is greater than or euqal to 1.4.56 if dpkg --compare-versions "$version" "ge" "1.4.56"; then # If it is, then we don't need to disable the modules + # (server.modules duplication is ignored in lighttpd 1.4.56+) : else # disable server.modules += ( ... ) in $conf to avoid module dups - # (server.modules duplication is ignored in lighttpd 1.4.56+) if awk '!/^server\.modules/{print}' $conf > $conf.$$ && mv $conf.$$ $conf; then : else From 9331cbff4b186f91542a5981b54355076cb08a3b Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 21 Jan 2023 14:25:06 +0000 Subject: [PATCH 10/24] remove the access log configuration from lighttpd.conf.debian and .conf.fedora to prevent issues on upgrades (this is defined in pihole-admin.conf) Signed-off-by: Adam Warner --- advanced/lighttpd.conf.debian | 2 -- advanced/lighttpd.conf.fedora | 2 -- 2 files changed, 4 deletions(-) diff --git a/advanced/lighttpd.conf.debian b/advanced/lighttpd.conf.debian index 61d20821..5a96c6cd 100644 --- a/advanced/lighttpd.conf.debian +++ b/advanced/lighttpd.conf.debian @@ -33,8 +33,6 @@ server.groupname = "www-data" # For lighttpd version 1.4.46 or above, the port can be overwritten in `/etc/lighttpd/external.conf` using the := operator # e.g. server.port := 8000 server.port = 80 -accesslog.filename = "/var/log/lighttpd/access-pihole.log" -accesslog.format = "%{%s}t|%V|%r|%s|%b" # Allow streaming response # reference: https://redmine.lighttpd.net/projects/lighttpd/wiki/Server_stream-response-bodyDetails diff --git a/advanced/lighttpd.conf.fedora b/advanced/lighttpd.conf.fedora index 04f3ee01..6276bfcb 100644 --- a/advanced/lighttpd.conf.fedora +++ b/advanced/lighttpd.conf.fedora @@ -35,8 +35,6 @@ server.groupname = "lighttpd" # For lighttpd version 1.4.46 or above, the port can be overwritten in `/etc/lighttpd/external.conf` using the := operator # e.g. server.port := 8000 server.port = 80 -accesslog.filename = "/var/log/lighttpd/access-pihole.log" -accesslog.format = "%{%s}t|%V|%r|%s|%b" # Allow streaming response # reference: https://redmine.lighttpd.net/projects/lighttpd/wiki/Server_stream-response-bodyDetails From 6b4f77bdfe0b747b5c9ff8f80505322a30b0676f Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 21 Jan 2023 17:42:32 +0000 Subject: [PATCH 11/24] change socket path back to /run/lighttpd/* - possibly causing issues https://github.com/pi-hole/pi-hole/issues/5131 Signed-off-by: Adam Warner --- advanced/pihole-admin.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/pihole-admin.conf b/advanced/pihole-admin.conf index 60eca564..ef15c8fe 100644 --- a/advanced/pihole-admin.conf +++ b/advanced/pihole-admin.conf @@ -23,7 +23,7 @@ $HTTP["url"] =~ "^/admin/" { fastcgi.server = ( ".php" => ( "localhost" => ( - "socket" => "/tmp/pihole-php-fastcgi.socket", + "socket" => "/run/lighttpd/pihole-php-fastcgi.socket", "bin-path" => "/usr/bin/php-cgi", "min-procs" => 0, "max-procs" => 1, From 9939cf1d774ced34b528c37ea6a90137419315b5 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 21 Jan 2023 23:47:19 +0000 Subject: [PATCH 12/24] Rename some of the variables to hopefully make the process a little clearer Signed-off-by: Adam Warner --- gravity.sh | 68 +++++++++++++++++++++++++++--------------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/gravity.sh b/gravity.sh index e8dfb34d..c2795442 100755 --- a/gravity.sh +++ b/gravity.sh @@ -244,7 +244,7 @@ database_adlist_number() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${num_source_lines}" "${num_unusable}" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${num_domains}" "${num_non_domains}" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -519,12 +519,12 @@ gravity_DownloadBlocklists() { gravity_Blackbody=true } -# num_target_lines does increase for every correctly added domain in pareseList() -num_target_lines=0 -num_source_lines=0 -num_unusable=0 +# num_total_imported_domains increases for each list processed +num_total_imported_domains=0 +num_domains=0 +num_non_domains=0 parseList() { - local adlistID="${1}" src="${2}" target="${3}" unusable_lines sample_unusable_lines tmp_unusuable_lines_str false_positive + local adlistID="${1}" src="${2}" target="${3}" non_domains sample_non_domains tmp_non_domains_str false_positive # This sed does the following things: # 1. Remove all lines containing no domains # 2. Remove all domains containing invalid characters. Valid are: a-z, A-Z, 0-9, dot (.), minus (-), underscore (_) @@ -535,9 +535,9 @@ parseList() { # Find lines containing no domains or with invalid characters (see above) # Remove duplicates from the list - mapfile -t unusable_lines <<< "$(sed -r "/([^\.]+\.)+[^\.]{2,}/d" < "${src}")" - mapfile -t -O "${#unusable_lines[@]}" unusable_lines <<< "$(sed -r "/[^a-zA-Z0-9.\_-]/!d" < "${src}")" - IFS=" " read -r -a unusable_lines <<< "$(tr ' ' '\n' <<< "${unusable_lines[@]}" | sort -u | tr '\n' ' ')" + mapfile -t non_domains <<< "$(sed -r "/([^\.]+\.)+[^\.]{2,}/d" < "${src}")" + mapfile -t -O "${#non_domains[@]}" non_domains <<< "$(sed -r "/[^a-zA-Z0-9.\_-]/!d" < "${src}")" + IFS=" " read -r -a non_domains <<< "$(tr ' ' '\n' <<< "${non_domains[@]}" | sort -u | tr '\n' ' ')" # A list of items of common local hostnames not to report as unusable # Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files @@ -559,37 +559,37 @@ parseList() { ) # Read the unusable lines into a string - tmp_unusuable_lines_str=" ${unusable_lines[*]} " + tmp_non_domains_str=" ${non_domains[*]} " for false_positive in "${false_positives[@]}"; do - # Remove false positives from tmp_unusuable_lines_str - tmp_unusuable_lines_str="${tmp_unusuable_lines_str/ ${false_positive} / }" + # Remove false positives from tmp_non_domains_str + tmp_non_domains_str="${tmp_non_domains_str/ ${false_positive} / }" done # Read the string back into an array - IFS=" " read -r -a unusable_lines <<< "${tmp_unusuable_lines_str}" - - # Get a sample of the incorrect lines, limited to 5 (the list should already have been de-duplicated) - IFS=" " read -r -a sample_unusable_lines <<< "$(tr ' ' '\n' <<< "${unusable_lines[@]}" | head -n 5 | tr '\n' ' ')" - - local num_target_lines_new num_correct_lines - # Get number of lines in source file - num_source_lines="$(grep -c "^" "${src}")" - # Get the new number of lines in destination file - num_target_lines_new="$(grep -c "^" "${target}")" - # Number of new correctly added lines - num_correct_lines="$(( num_target_lines_new-num_target_lines ))" - # Update number of lines in target file - num_target_lines="$num_target_lines_new" - num_unusable="${#unusable_lines[@]}" - - if [[ "${num_unusable}" -ne 0 ]]; then - echo " ${INFO} Imported ${num_correct_lines} domains, ignoring ${num_unusable} non-domain entries" + IFS=" " read -r -a non_domains <<< "${tmp_non_domains_str}" + + # Get a sample of non-domain entries, limited to 5 (the list should already have been de-duplicated) + IFS=" " read -r -a sample_non_domains <<< "$(tr ' ' '\n' <<< "${non_domains[@]}" | head -n 5 | tr '\n' ' ')" + + local tmp_new_imported_total + # Get the new number of domains in destination file + tmp_new_imported_total="$(grep -c "^" "${target}")" + # Number of imported lines for this file is the difference between the new total and the old total. (Or, the number of domains we just added.) + num_domains="$(( tmp_new_imported_total-num_total_imported_domains ))" + # Replace the running total with the new total. + num_total_imported_domains="$tmp_new_imported_total" + # Get the number of non_domains (this is the number of entries left after stripping the source of comments/duplicates/false positives/domains) + num_non_domains="${#non_domains[@]}" + + # If there are unusable lines, we display some information about them. This is not error or major cause for concern. + if [[ "${num_non_domains}" -ne 0 ]]; then + echo " ${INFO} Imported ${num_domains} domains, ignoring ${num_non_domains} non-domain entries" echo " Sample of non-domain entries:" - for each in "${sample_unusable_lines[@]}" + for each in "${sample_non_domains[@]}" do echo " - ${each}" done else - echo " ${INFO} Imported ${num_correct_lines} domains" + echo " ${INFO} Imported ${num_domains} domains" fi } @@ -745,8 +745,8 @@ gravity_DownloadBlocklistFromUrl() { else echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" # Manually reset these two numbers because we do not call parseList here - num_source_lines=0 - num_unusable=0 + num_domains=0 + num_non_domains=0 database_adlist_number "${adlistID}" database_adlist_status "${adlistID}" "4" fi From 68a03cc877294c39f9be4ff99988c1f23361bcbf Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 22 Jan 2023 18:05:05 +0000 Subject: [PATCH 13/24] Ensure that /run/lighttpd exists and is owned by lighttpd user. It is likely that new installs will fail currently Signed-off-by: Adam Warner --- automated install/basic-install.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index b1bd773e..6bb8e363 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1403,12 +1403,16 @@ installConfigs() { # set permissions on /etc/lighttpd/lighttpd.conf so pihole user (other) can read the file chmod o+x /etc/lighttpd chmod o+r "${lighttpdConfig}" + + # Ensure /run/lighttpd exists and is owned by lighttpd user + # Needed for the php socket + mkdir -p /run/lighttpd + chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} /run/lighttpd + if grep -q -F "FILE AUTOMATICALLY OVERWRITTEN BY PI-HOLE" "${lighttpdConfig}"; then # Attempt to preserve backwards compatibility with older versions install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} "${lighttpdConfig}" # Make the directories if they do not exist and set the owners - mkdir -p /run/lighttpd - chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} /run/lighttpd mkdir -p /var/cache/lighttpd/compress chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} /var/cache/lighttpd/compress mkdir -p /var/cache/lighttpd/uploads From e59f5db145e274fa86d9d3dbc3ea0afd6292fbc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 22 Jan 2023 22:37:19 +0100 Subject: [PATCH 14/24] Add pihole-admin.conf to debug log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index c0264d1a..23767e00 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -77,6 +77,8 @@ PIHOLE_CRON_FILE="${CRON_D_DIRECTORY}/pihole" WEB_SERVER_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/lighttpd.conf" WEB_SERVER_CUSTOM_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/external.conf" +WEB_SERVER_PIHOLE_CONFIG_FILE_DEBIAN="${WEB_SERVER_CONFIG_DIRECTORY}/conf-available/15-pihole-admin.conf" +WEB_SERVER_PIHOLE_CONFIG_FILE_FEDORA="${WEB_SERVER_CONFIG_DIRECTORY}/conf.d/pihole-admin.conf" PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" @@ -140,6 +142,8 @@ PIHOLE_PROCESSES=( "lighttpd" "pihole-FTL" ) REQUIRED_FILES=("${PIHOLE_CRON_FILE}" "${WEB_SERVER_CONFIG_FILE}" "${WEB_SERVER_CUSTOM_CONFIG_FILE}" +"${WEB_SERVER_PIHOLE_CONFIG_FILE_DEBIAN}" +"${WEB_SERVER_PIHOLE_CONFIG_FILE_FEDORA}" "${PIHOLE_INSTALL_LOG_FILE}" "${PIHOLE_RAW_BLOCKLIST_FILES}" "${PIHOLE_LOCAL_HOSTS_FILE}" @@ -1069,10 +1073,13 @@ dir_check() { # check if exists first; if it does, if ls "${filename}" 1> /dev/null 2>&1; then # do nothing - : + true + return else # Otherwise, show an error log_write "${COL_RED}${directory} does not exist.${COL_NC}" + false + return fi done } @@ -1132,9 +1139,10 @@ show_content_of_files_in_dir() { # Set a local variable for better readability local directory="${1}" # Check if the directory exists - dir_check "${directory}" - # if it does, list the files in it - list_files_in_dir "${directory}" + if dir_check "${directory}"; then + # if it does, list the files in it + list_files_in_dir "${directory}" + fi } show_content_of_pihole_files() { @@ -1142,6 +1150,8 @@ show_content_of_pihole_files() { show_content_of_files_in_dir "${PIHOLE_DIRECTORY}" show_content_of_files_in_dir "${DNSMASQ_D_DIRECTORY}" show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY}" + show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY}/conf.d" + show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY}/conf-available" show_content_of_files_in_dir "${CRON_D_DIRECTORY}" show_content_of_files_in_dir "${WEB_SERVER_LOG_DIRECTORY}" show_content_of_files_in_dir "${LOG_DIRECTORY}" From 9bde5de601197a477f3c195571585cf1cdcc3501 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 24 Jan 2023 21:59:13 +0100 Subject: [PATCH 15/24] Use conf-enabled instead of conf-available MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 23767e00..614e00b1 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -77,7 +77,7 @@ PIHOLE_CRON_FILE="${CRON_D_DIRECTORY}/pihole" WEB_SERVER_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/lighttpd.conf" WEB_SERVER_CUSTOM_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/external.conf" -WEB_SERVER_PIHOLE_CONFIG_FILE_DEBIAN="${WEB_SERVER_CONFIG_DIRECTORY}/conf-available/15-pihole-admin.conf" +WEB_SERVER_PIHOLE_CONFIG_FILE_DEBIAN="${WEB_SERVER_CONFIG_DIRECTORY}/conf-enabled/15-pihole-admin.conf" WEB_SERVER_PIHOLE_CONFIG_FILE_FEDORA="${WEB_SERVER_CONFIG_DIRECTORY}/conf.d/pihole-admin.conf" PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" @@ -1151,7 +1151,7 @@ show_content_of_pihole_files() { show_content_of_files_in_dir "${DNSMASQ_D_DIRECTORY}" show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY}" show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY}/conf.d" - show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY}/conf-available" + show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY}/conf-enabled" show_content_of_files_in_dir "${CRON_D_DIRECTORY}" show_content_of_files_in_dir "${WEB_SERVER_LOG_DIRECTORY}" show_content_of_files_in_dir "${LOG_DIRECTORY}" From be0efa233293f82e524a918dafdbeefa69477b4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 24 Jan 2023 22:19:24 +0100 Subject: [PATCH 16/24] Add lighttpd selftest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 614e00b1..aa197512 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -981,6 +981,20 @@ ftl_full_status(){ fi } +lighttpd_test_configuration(){ + # let lighttpd test it's own configuration + local lighttpd_conf_test + echo_current_diagnostic "Lighttpd configuration test" + lighttpd_conf_test=$(lighttpd -tt -f /etc/lighttpd/lighttpd.conf) + if [ -z "${lighttpd_conf_test}" ]; then + # empty output + log_write "${TICK} ${COL_GREEN}No error in lighttpd configuration${COL_NC}" + else + log_write "${CROSS} ${COL_RED}Error in lighttpd configuration${COL_NC}" + log_write " ${lighttpd_conf_test}" + fi +} + make_array_from_file() { local filename="${1}" # The second argument can put a limit on how many line should be read from the file @@ -1506,6 +1520,7 @@ check_name_resolution check_dhcp_servers process_status ftl_full_status +lighttpd_test_configuration parse_setup_vars check_x_headers analyze_ftl_db From 18ab94135ffc4aee8fe59c19a88a3fadaeb4f6c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 24 Jan 2023 23:02:38 +0100 Subject: [PATCH 17/24] Show all files (not their content) of the lighttpd config dirs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index aa197512..ad25e866 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -66,6 +66,8 @@ RUN_DIRECTORY="/run" LOG_DIRECTORY="/var/log/pihole" WEB_SERVER_LOG_DIRECTORY="/var/log/lighttpd" WEB_SERVER_CONFIG_DIRECTORY="/etc/lighttpd" +WEB_SERVER_CONFIG_DIRECTORY_FEDORA="${WEB_SERVER_CONFIG_DIRECTORY}/conf.d" +WEB_SERVER_CONFIG_DIRECTORY_DEBIAN="${WEB_SERVER_CONFIG_DIRECTORY}/conf-enabled" HTML_DIRECTORY="/var/www/html" WEB_GIT_DIRECTORY="${HTML_DIRECTORY}/admin" SHM_DIRECTORY="/dev/shm" @@ -77,8 +79,8 @@ PIHOLE_CRON_FILE="${CRON_D_DIRECTORY}/pihole" WEB_SERVER_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/lighttpd.conf" WEB_SERVER_CUSTOM_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/external.conf" -WEB_SERVER_PIHOLE_CONFIG_FILE_DEBIAN="${WEB_SERVER_CONFIG_DIRECTORY}/conf-enabled/15-pihole-admin.conf" -WEB_SERVER_PIHOLE_CONFIG_FILE_FEDORA="${WEB_SERVER_CONFIG_DIRECTORY}/conf.d/pihole-admin.conf" +WEB_SERVER_PIHOLE_CONFIG_FILE_DEBIAN="${WEB_SERVER_CONFIG_DIRECTORY_DEBIAN}/15-pihole-admin.conf" +WEB_SERVER_PIHOLE_CONFIG_FILE_FEDORA="${WEB_SERVER_CONFIG_DIRECTORY_FEDORA}/pihole-admin.conf" PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" @@ -1101,6 +1103,19 @@ dir_check() { list_files_in_dir() { # Set the first argument passed to this function as a named variable for better readability local dir_to_parse="${1}" + + # show files and sizes of some directories, don't print the file content (yet) + if [[ "${dir_to_parse}" == "${SHM_DIRECTORY}" ]]; then + # SHM file - we do not want to see the content, but we want to see the files and their sizes + log_write "$(ls -lh "${dir_to_parse}/")" + elif [[ "${dir_to_parse}" == "${WEB_SERVER_CONFIG_DIRECTORY_FEDORA}" ]]; then + # we want to see all files files in /etc/lighttpd/conf.d + log_write "$(ls -lh "${dir_to_parse}/" 2> /dev/null )" + elif [[ "${dir_to_parse}" == "${WEB_SERVER_CONFIG_DIRECTORY_DEBIAN}" ]]; then + # we want to see all files files in /etc/lighttpd/conf.d + log_write "$(ls -lh "${dir_to_parse}/"/ 2> /dev/null )" + fi + # Store the files found in an array mapfile -t files_found < <(ls "${dir_to_parse}") # For each file in the array, @@ -1116,11 +1131,8 @@ list_files_in_dir() { [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_WEB_SERVER_ACCESS_LOG_FILE}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_LOG_GZIPS}" ]]; then : - elif [[ "${dir_to_parse}" == "${SHM_DIRECTORY}" ]]; then - # SHM file - we do not want to see the content, but we want to see the files and their sizes - log_write "$(ls -lhd "${dir_to_parse}"/"${each_file}")" elif [[ "${dir_to_parse}" == "${DNSMASQ_D_DIRECTORY}" ]]; then - # in case of the dnsmasq directory inlcuede all files in the debug output + # in case of the dnsmasq directory include all files in the debug output log_write "\\n${COL_GREEN}$(ls -lhd "${dir_to_parse}"/"${each_file}")${COL_NC}" make_array_from_file "${dir_to_parse}/${each_file}" else @@ -1164,8 +1176,8 @@ show_content_of_pihole_files() { show_content_of_files_in_dir "${PIHOLE_DIRECTORY}" show_content_of_files_in_dir "${DNSMASQ_D_DIRECTORY}" show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY}" - show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY}/conf.d" - show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY}/conf-enabled" + show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY_FEDORA}" + show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY_DEBIAN}" show_content_of_files_in_dir "${CRON_D_DIRECTORY}" show_content_of_files_in_dir "${WEB_SERVER_LOG_DIRECTORY}" show_content_of_files_in_dir "${LOG_DIRECTORY}" From b9e401aaa3e3770c615cb27c7b7d0b0b17f853a5 Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Wed, 25 Jan 2023 14:35:57 -0300 Subject: [PATCH 18/24] Change `min_procs` value to `1` The original configuration used the default `min_procs=`. A recent change set this value to zero, but a lot of systems started to receive error messages about the socket file and in some systems lighttpd wasn't able to restart the PHP process, returning HTTP error 503. Setting this to 1 fixed those errors. Signed-off-by: RD WebDesign --- advanced/pihole-admin.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/pihole-admin.conf b/advanced/pihole-admin.conf index ef15c8fe..0bb6eac9 100644 --- a/advanced/pihole-admin.conf +++ b/advanced/pihole-admin.conf @@ -25,7 +25,7 @@ $HTTP["url"] =~ "^/admin/" { "localhost" => ( "socket" => "/run/lighttpd/pihole-php-fastcgi.socket", "bin-path" => "/usr/bin/php-cgi", - "min-procs" => 0, + "min-procs" => 1, "max-procs" => 1, "bin-environment" => ( "PHP_FCGI_CHILDREN" => "4", From 3ad896595953b98647e6e01587fd06c5c9d88f5e Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 1 Feb 2023 17:38:21 +0000 Subject: [PATCH 19/24] Tweak old pihole lighttpd config warning message to better reflect the consequences of making changes to the file Signed-off-by: Adam Warner --- advanced/lighttpd.conf.debian | 16 +++++++++------- advanced/lighttpd.conf.fedora | 16 +++++++++------- automated install/basic-install.sh | 2 +- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/advanced/lighttpd.conf.debian b/advanced/lighttpd.conf.debian index 5a96c6cd..f31f7bcd 100644 --- a/advanced/lighttpd.conf.debian +++ b/advanced/lighttpd.conf.debian @@ -7,13 +7,15 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. -############################################################################### -# FILE AUTOMATICALLY OVERWRITTEN BY PI-HOLE INSTALL/UPDATE PROCEDURE. # -# ANY CHANGES MADE TO THIS FILE AFTER INSTALL WILL BE LOST ON THE NEXT UPDATE # -# # -# CHANGES SHOULD BE MADE IN A SEPARATE CONFIG FILE: # -# /etc/lighttpd/external.conf # -############################################################################### +################################################################################################### +# IF THIS HEADER EXISTS, THE FILE WILL BE OVERWRITTEN BY PI-HOLE'S UPDATE PROCEDURE. # +# ANY CHANGES MADE TO THIS FILE WILL BE LOST ON THE NEXT UPDATE UNLESS YOU REMOVE THIS HEADER # +# # +# ENSURE THAT YOU DO NOT REMOVE THE REQUIRED LINE: # +# # +# include "/etc/lighttpd/conf-enabled/*.conf" # +# # +################################################################################################### server.modules = ( "mod_access", diff --git a/advanced/lighttpd.conf.fedora b/advanced/lighttpd.conf.fedora index 6276bfcb..e09d7760 100644 --- a/advanced/lighttpd.conf.fedora +++ b/advanced/lighttpd.conf.fedora @@ -7,13 +7,15 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. -############################################################################### -# FILE AUTOMATICALLY OVERWRITTEN BY PI-HOLE INSTALL/UPDATE PROCEDURE. # -# ANY CHANGES MADE TO THIS FILE AFTER INSTALL WILL BE LOST ON THE NEXT UPDATE # -# # -# CHANGES SHOULD BE MADE IN A SEPARATE CONFIG FILE: # -# /etc/lighttpd/external.conf # -############################################################################### +################################################################################################### +# IF THIS HEADER EXISTS, THE FILE WILL BE OVERWRITTEN BY PI-HOLE'S UPDATE PROCEDURE. # +# ANY CHANGES MADE TO THIS FILE WILL BE LOST ON THE NEXT UPDATE UNLESS YOU REMOVE THIS HEADER # +# # +# ENSURE THAT YOU DO NOT REMOVE THE REQUIRED LINE: # +# # +# include "/etc/lighttpd/conf.d/pihole-admin.conf" # +# # +################################################################################################### server.modules = ( "mod_access", diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 6bb8e363..05bc0e4e 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1409,7 +1409,7 @@ installConfigs() { mkdir -p /run/lighttpd chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} /run/lighttpd - if grep -q -F "FILE AUTOMATICALLY OVERWRITTEN BY PI-HOLE" "${lighttpdConfig}"; then + if grep -q -F "FILE WILL BE OVERWRITTEN BY PI-HOLE" "${lighttpdConfig}"; then # Attempt to preserve backwards compatibility with older versions install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} "${lighttpdConfig}" # Make the directories if they do not exist and set the owners From ca00ffa101d919b0fc493201a734646a620b0de0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 2 Feb 2023 11:58:58 +0100 Subject: [PATCH 20/24] Only source versions file if the file exits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- pihole | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pihole b/pihole index a99a37e7..1d9ad82c 100755 --- a/pihole +++ b/pihole @@ -24,7 +24,12 @@ utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" source "${utilsfile}" versionsfile="/etc/pihole/versions" -source "${versionsfile}" +if [ -f "${versionsfile}" ]; then + # Only source versionsfile if the file exits + # fixes a warning during installation where versionsfile does not exist yet + # but gravity calls `pihole -status` and thereby sourcing the file + source "${versionsfile}" +fi webpageFunc() { source "${PI_HOLE_SCRIPT_DIR}/webpage.sh" From e5ea361b53d53a49e53de341c6b26f8f63a7f4ee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 4 Feb 2023 10:01:16 +0000 Subject: [PATCH 21/24] Bump tox from 4.3.5 to 4.4.4 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.3.5 to 4.4.4. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.3.5...4.4.4) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 686f2a58..08e7027a 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ docker-compose == 1.29.2 pytest == 7.2.1 pytest-xdist == 3.1.0 pytest-testinfra == 7.0.0 -tox == 4.3.5 +tox == 4.4.4 From 5ecdfb53c23712a03f22a2d0bbbc78843cf4f68e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 11 Feb 2023 10:57:20 +0000 Subject: [PATCH 22/24] Bump tox from 4.4.4 to 4.4.5 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.4.4 to 4.4.5. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.4.4...4.4.5) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 08e7027a..1670e765 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ docker-compose == 1.29.2 pytest == 7.2.1 pytest-xdist == 3.1.0 pytest-testinfra == 7.0.0 -tox == 4.4.4 +tox == 4.4.5 From a4bdf2454bf6b8676fc199db7d81a699536c9b8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 16 Feb 2023 10:34:19 +0100 Subject: [PATCH 23/24] Don't use intermediate strings to filter false positives in gravity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/gravity.sh b/gravity.sh index c2795442..28c28a9e 100755 --- a/gravity.sh +++ b/gravity.sh @@ -524,7 +524,7 @@ num_total_imported_domains=0 num_domains=0 num_non_domains=0 parseList() { - local adlistID="${1}" src="${2}" target="${3}" non_domains sample_non_domains tmp_non_domains_str false_positive + local adlistID="${1}" src="${2}" target="${3}" non_domains sample_non_domains # This sed does the following things: # 1. Remove all lines containing no domains # 2. Remove all domains containing invalid characters. Valid are: a-z, A-Z, 0-9, dot (.), minus (-), underscore (_) @@ -542,30 +542,13 @@ parseList() { # A list of items of common local hostnames not to report as unusable # Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files # but flagging them as unusable causes more confusion than it's worth - so we suppress them from the output - false_positives=( - "localhost" - "localhost.localdomain" - "local" - "broadcasthost" - "localhost" - "ip6-localhost" - "ip6-loopback" - "lo0 localhost" - "ip6-localnet" - "ip6-mcastprefix" - "ip6-allnodes" - "ip6-allrouters" - "ip6-allhosts" - ) - - # Read the unusable lines into a string - tmp_non_domains_str=" ${non_domains[*]} " - for false_positive in "${false_positives[@]}"; do - # Remove false positives from tmp_non_domains_str - tmp_non_domains_str="${tmp_non_domains_str/ ${false_positive} / }" - done - # Read the string back into an array - IFS=" " read -r -a non_domains <<< "${tmp_non_domains_str}" + false_positives="localhost|localhost.localdomain|local|broadcasthost|localhost|ip6-localhost|ip6-loopback|lo0 localhost|ip6-localnet|ip6-mcastprefix|ip6-allnodes|ip6-allrouters|ip6-allhosts" + + # if there are any non-domains, filter the array for false-positives + # Credit: https://stackoverflow.com/a/40264051 + if [[ "${#non_domains[@]}" -gt 0 ]]; then + mapfile -d $'\0' -t non_domains < <(printf '%s\0' "${non_domains[@]}" | grep -Ezv "^${false_positives}") + fi # Get a sample of non-domain entries, limited to 5 (the list should already have been de-duplicated) IFS=" " read -r -a sample_non_domains <<< "$(tr ' ' '\n' <<< "${non_domains[@]}" | head -n 5 | tr '\n' ' ')" From 90da155053dcebb205f33f438af8fdd4f9bcb3a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 16 Feb 2023 21:02:49 +0100 Subject: [PATCH 24/24] Use buildx to create docker test images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/tox.centos_8.ini | 2 +- test/tox.centos_9.ini | 2 +- test/tox.debian_10.ini | 2 +- test/tox.debian_11.ini | 2 +- test/tox.fedora_36.ini | 2 +- test/tox.fedora_37.ini | 2 +- test/tox.ubuntu_20.ini | 2 +- test/tox.ubuntu_22.ini | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/tox.centos_8.ini b/test/tox.centos_8.ini index dac10e97..dca77c93 100644 --- a/test/tox.centos_8.ini +++ b/test/tox.centos_8.ini @@ -4,5 +4,5 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt -commands = docker build -f _centos_8.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _centos_8.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py diff --git a/test/tox.centos_9.ini b/test/tox.centos_9.ini index aa7009e1..a69c336a 100644 --- a/test/tox.centos_9.ini +++ b/test/tox.centos_9.ini @@ -4,5 +4,5 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt -commands = docker build -f _centos_9.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _centos_9.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py diff --git a/test/tox.debian_10.ini b/test/tox.debian_10.ini index a012bda4..f107300f 100644 --- a/test/tox.debian_10.ini +++ b/test/tox.debian_10.ini @@ -4,5 +4,5 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt -commands = docker build -f _debian_10.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _debian_10.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.debian_11.ini b/test/tox.debian_11.ini index 48dc9df1..c38a15fb 100644 --- a/test/tox.debian_11.ini +++ b/test/tox.debian_11.ini @@ -4,5 +4,5 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt -commands = docker build -f _debian_11.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _debian_11.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.fedora_36.ini b/test/tox.fedora_36.ini index 0cc6f29c..515487ed 100644 --- a/test/tox.fedora_36.ini +++ b/test/tox.fedora_36.ini @@ -4,5 +4,5 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt -commands = docker build -f _fedora_36.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _fedora_36.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py diff --git a/test/tox.fedora_37.ini b/test/tox.fedora_37.ini index d6f44533..2a8ef398 100644 --- a/test/tox.fedora_37.ini +++ b/test/tox.fedora_37.ini @@ -4,5 +4,5 @@ envlist = py3 [testenv] allowlist_externals = docker deps = -rrequirements.txt -commands = docker build -f _fedora_37.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _fedora_37.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py diff --git a/test/tox.ubuntu_20.ini b/test/tox.ubuntu_20.ini index 88ee0b54..49a6153e 100644 --- a/test/tox.ubuntu_20.ini +++ b/test/tox.ubuntu_20.ini @@ -4,5 +4,5 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt -commands = docker build -f _ubuntu_20.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _ubuntu_20.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.ubuntu_22.ini b/test/tox.ubuntu_22.ini index cb5527ab..8014d6d6 100644 --- a/test/tox.ubuntu_22.ini +++ b/test/tox.ubuntu_22.ini @@ -4,5 +4,5 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt -commands = docker build -f _ubuntu_22.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _ubuntu_22.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py