From f5c3dc5d9d4f00caed58eddfc40d712d565ab038 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 7 Aug 2018 16:51:58 +0200 Subject: [PATCH 01/15] Improve download error message that is shown when downloading or verifying the binary failed Signed-off-by: DL6ER --- automated install/basic-install.sh | 4 ++-- test/test_automated_install.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 61dfe831..b12028a5 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2099,7 +2099,7 @@ FTLinstall() { # the download failed, so just go back to the original directory popd > /dev/null || { echo "Unable to return to original directory after FTL binary download."; return 1; } echo -e "${OVER} ${CROSS} ${str}" - echo -e " ${COL_LIGHT_RED}Error: Download of binary from Github failed${COL_NC}" + echo -e " ${COL_LIGHT_RED}Error: Download of ${url}/${binary} failed (checksum error)${COL_NC}" return 1 fi # Otherwise, @@ -2107,7 +2107,7 @@ FTLinstall() { popd > /dev/null || { echo "Unable to return to original directory after FTL binary download."; return 1; } echo -e "${OVER} ${CROSS} ${str}" # The URL could not be found - echo -e " ${COL_LIGHT_RED}Error: URL not found${COL_NC}" + echo -e " ${COL_LIGHT_RED}Error: URL ${url}/${binary} not found${COL_NC}" return 1 fi } diff --git a/test/test_automated_install.py b/test/test_automated_install.py index 876b06eb..2cded451 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -501,8 +501,10 @@ def test_FTL_download_unknown_fails_no_errors(Pihole): ''') expected_stdout = cross_box + ' Downloading and Installing FTL' assert expected_stdout in download_binary.stdout - error = 'Error: URL not found' - assert error in download_binary.stdout + error1 = 'Error: URL https://github.com/pi-hole/FTL/releases/download/' + assert error1 in download_binary.stdout + error2 = 'not found' + assert error2 in download_binary.stdout def test_FTL_binary_installed_and_responsive_no_errors(Pihole): From a1bf9fad98f3ac97fe4330337197b7b5a4d62512 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Sat, 11 Aug 2018 12:55:18 +1000 Subject: [PATCH 02/15] Fix colour tail for use with new blocking methods Signed-off-by: WaLLy3K --- pihole | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pihole b/pihole index 8be03f79..b444efe7 100755 --- a/pihole +++ b/pihole @@ -303,7 +303,7 @@ tailFunc() { # Colour everything else as gray tail -f /var/log/pihole.log | sed -E \ -e "s,($(date +'%b %d ')| dnsmasq[.*[0-9]]),,g" \ - -e "s,(.*(gravity.list|black.list| config ).* is (${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \ + -e "s,(.*(gravity.list|black.list| config ).* is (0.0.0.0|::|NXDOMAIN|${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \ -e "s,.*(query\\[A|DHCP).*,${COL_NC}&${COL_NC}," \ -e "s,.*,${COL_GRAY}&${COL_NC}," exit 0 From d9195970b7da98e685945abe8a7dcd7417ccf44c Mon Sep 17 00:00:00 2001 From: Realtbo Date: Sun, 12 Aug 2018 23:19:16 +0200 Subject: [PATCH 03/15] Create adlist file while running unattended setup Signed-off-by: Realtebo --- automated install/basic-install.sh | 45 +++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a7f590ab..8f54e440 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1087,19 +1087,42 @@ chooseBlocklists() { # For each choice available, for choice in ${choices} do - # Set the values to true - case ${choice} in - StevenBlack ) echo "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" >> "${adlistFile}";; - MalwareDom ) echo "https://mirror1.malwaredomains.com/files/justdomains" >> "${adlistFile}";; - Cameleon ) echo "http://sysctl.org/cameleon/hosts" >> "${adlistFile}";; - ZeusTracker ) echo "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist" >> "${adlistFile}";; - DisconTrack ) echo "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt" >> "${adlistFile}";; - DisconAd ) echo "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt" >> "${adlistFile}";; - HostsFile ) echo "https://hosts-file.net/ad_servers.txt" >> "${adlistFile}";; - esac + appendToListsFile choice done } +# Accept a string parameter, it must be one of the default lists +# This function allow to not duplicate code in chooseBlocklists and +# in installDefaultBlocklists +appendToListsFile() { + case $1 in + StevenBlack ) echo "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" >> "${adlistFile}";; + MalwareDom ) echo "https://mirror1.malwaredomains.com/files/justdomains" >> "${adlistFile}";; + Cameleon ) echo "http://sysctl.org/cameleon/hosts" >> "${adlistFile}";; + ZeusTracker ) echo "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist" >> "${adlistFile}";; + DisconTrack ) echo "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt" >> "${adlistFile}";; + DisconAd ) echo "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt" >> "${adlistFile}";; + HostsFile ) echo "https://hosts-file.net/ad_servers.txt" >> "${adlistFile}";; + esac +} + +# Used only in unattended setup +# If there is already the adListFile, we keep it, else we create it using all default lists +installDefaultBlocklists() { + # In unattended setup, could be usefull to use userdefined blocklist. + # If this file exists, we avoid to override it. + if [[ -f "${adlistFile}" ]]; then + return; + fi + appendToListsFile StevenBlack + appendToListsFile MalwareDom + appendToListsFile Cameleon + appendToListsFile ZeusTracker + appendToListsFile DisconTrack + appendToListsFile DisconAd + appendToListsFile HostsFile +} + # Check if /etc/dnsmasq.conf is from pi-hole. If so replace with an original and install new in .d directory version_check_dnsmasq() { # Local, named variables @@ -2391,6 +2414,8 @@ main() { # Let the user decide if they want query logging enabled... setLogging else + # Setup adlist file if not exists + installDefaultBlocklists # Source ${setupVars} to use predefined user variables in the functions source ${setupVars} fi From d163e3838070a9d427f1d1240d4059b254da2e14 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Mon, 13 Aug 2018 14:56:38 +1000 Subject: [PATCH 04/15] Ensure regex-blocked lines are red * Add `regex.list` to seds list Signed-off-by: WaLLy3K --- pihole | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pihole b/pihole index b444efe7..0f584b15 100755 --- a/pihole +++ b/pihole @@ -303,7 +303,7 @@ tailFunc() { # Colour everything else as gray tail -f /var/log/pihole.log | sed -E \ -e "s,($(date +'%b %d ')| dnsmasq[.*[0-9]]),,g" \ - -e "s,(.*(gravity.list|black.list| config ).* is (0.0.0.0|::|NXDOMAIN|${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \ + -e "s,(.*(gravity.list|black.list|regex.list| config ).* is (0.0.0.0|::|NXDOMAIN|${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \ -e "s,.*(query\\[A|DHCP).*,${COL_NC}&${COL_NC}," \ -e "s,.*,${COL_GRAY}&${COL_NC}," exit 0 From ca309d3b282b4cf83051b925046db7239be6ab58 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 14 Aug 2018 13:14:39 +0200 Subject: [PATCH 05/15] Restart pihole-FTL after log flushing to force reloading history. Fixes #2339 Signed-off-by: DL6ER --- advanced/Scripts/piholeLogFlush.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 4847282f..561fbce7 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -58,6 +58,8 @@ else # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history) deleted=$(sqlite3 "${DBFILE}" "DELETE FROM queries WHERE timestamp >= strftime('%s','now')-86400; select changes() from queries limit 1") + # Restart pihole-FTL to force reloading history + sudo pihole restartdns fi if [[ "$@" != *"quiet"* ]]; then From 476fecbcf5b28634a7e3a963697a79f01aa7e159 Mon Sep 17 00:00:00 2001 From: Mirko Tebaldi Date: Tue, 14 Aug 2018 14:59:41 +0200 Subject: [PATCH 06/15] Fixed typos as suggestes Signed-off-by: Realtebo --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 23d8a896..f91d64a7 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1109,8 +1109,8 @@ appendToListsFile() { # Used only in unattended setup # If there is already the adListFile, we keep it, else we create it using all default lists installDefaultBlocklists() { - # In unattended setup, could be usefull to use userdefined blocklist. - # If this file exists, we avoid to override it. + # In unattended setup, could be useful to use userdefined blocklist. + # If this file exists, we avoid overriding it. if [[ -f "${adlistFile}" ]]; then return; fi From 0d1b3cee4f96290db52df19e563c3ecad3991850 Mon Sep 17 00:00:00 2001 From: Realtebo Date: Thu, 16 Aug 2018 17:52:47 +0200 Subject: [PATCH 07/15] Enable FTL service before starting it Signed-off-by: Realtebo --- automated install/basic-install.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f91d64a7..9f596b45 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2483,9 +2483,12 @@ main() { echo -e " ${INFO} Restarting services..." # Start services - # Enable FTL - start_service pihole-FTL + # Enable FTL + # Ensure the service is enabled before trying to start it + # Fixes a problem reported on Ubuntu 18.04 where trying to start + # the service before enabling causes installer to exit enable_service pihole-FTL + start_service pihole-FTL # Download and compile the aggregated block list runGravity From d30b565d98149dfa68b84cf8b2756cf548a296e2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 Aug 2018 13:53:34 +0200 Subject: [PATCH 08/15] Touch dhcp.leases to ensure it exists. There are some systems out there where the installer didn't finish on the first run and some users don't have this file being created. Although /etc/pihole should be owned by pihole:pihole, pihole-FTL sometimes fails to open this file and - if this is the case and DHCP is enabled - refuses to start altogether. Signed-off-by: DL6ER --- advanced/Templates/pihole-FTL.service | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service index ecc7a52a..04fae69a 100644 --- a/advanced/Templates/pihole-FTL.service +++ b/advanced/Templates/pihole-FTL.service @@ -26,13 +26,16 @@ start() { if is_running; then echo "pihole-FTL is already running" else - touch /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole.log + touch /var/log/pihole-FTL.log /var/log/pihole.log + touch /run/pihole-FTL.pid /run/pihole-FTL.port + touch /etc/pihole/dhcp.leases mkdir -p /var/run/pihole mkdir -p /var/log/pihole chown pihole:pihole /var/run/pihole /var/log/pihole rm /var/run/pihole/FTL.sock 2> /dev/null - chown pihole:pihole /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port - chown pihole:pihole /etc/pihole /etc/pihole/dhcp.leases /var/log/pihole.log + chown pihole:pihole /run/pihole-FTL.pid /run/pihole-FTL.port + chown pihole:pihole /etc/pihole /etc/pihole/dhcp.leases + chown pihole:pihole /var/log/pihole-FTL.log /var/log/pihole.log chmod 0644 /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole.log setcap CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN+eip "$(which pihole-FTL)" echo "nameserver 127.0.0.1" | /sbin/resolvconf -a lo.piholeFTL @@ -78,7 +81,7 @@ status() { echo "[ ] pihole-FTL is not running" exit 1 fi -} +} ### main logic ### From 5cd3b11391b719f18313f614a062d29a5e702752 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 Aug 2018 14:32:19 +0200 Subject: [PATCH 09/15] Start pihole-FTL as root if capabilities are not supported by the system Signed-off-by: DL6ER --- advanced/Templates/pihole-FTL.service | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service index ecc7a52a..266ecd3e 100644 --- a/advanced/Templates/pihole-FTL.service +++ b/advanced/Templates/pihole-FTL.service @@ -34,9 +34,13 @@ start() { chown pihole:pihole /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port chown pihole:pihole /etc/pihole /etc/pihole/dhcp.leases /var/log/pihole.log chmod 0644 /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole.log - setcap CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN+eip "$(which pihole-FTL)" echo "nameserver 127.0.0.1" | /sbin/resolvconf -a lo.piholeFTL - su -s /bin/sh -c "/usr/bin/pihole-FTL" "$FTLUSER" + if setcap CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN+eip "$(which pihole-FTL)"; then + su -s /bin/sh -c "/usr/bin/pihole-FTL" "$FTLUSER" + else + echo "Warning: Starting pihole-FTL as root because setting capabilities is not supported on this system" + pihole-FTL + fi echo fi } @@ -78,7 +82,7 @@ status() { echo "[ ] pihole-FTL is not running" exit 1 fi -} +} ### main logic ### From ec2426b24d9e82a3f507b624b232f867f096e3f8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 19 Aug 2018 19:55:47 +0200 Subject: [PATCH 10/15] Add clarifying comments Signed-off-by: DL6ER --- advanced/Templates/pihole-FTL.service | 2 ++ 1 file changed, 2 insertions(+) diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service index 04fae69a..14205fe7 100644 --- a/advanced/Templates/pihole-FTL.service +++ b/advanced/Templates/pihole-FTL.service @@ -26,6 +26,7 @@ start() { if is_running; then echo "pihole-FTL is already running" else + # Touch files to ensure they exist (create if non-existing, preserve if existing) touch /var/log/pihole-FTL.log /var/log/pihole.log touch /run/pihole-FTL.pid /run/pihole-FTL.port touch /etc/pihole/dhcp.leases @@ -33,6 +34,7 @@ start() { mkdir -p /var/log/pihole chown pihole:pihole /var/run/pihole /var/log/pihole rm /var/run/pihole/FTL.sock 2> /dev/null + # Ensure that permissions are set so that pihole-FTL can edit all necessary files chown pihole:pihole /run/pihole-FTL.pid /run/pihole-FTL.port chown pihole:pihole /etc/pihole /etc/pihole/dhcp.leases chown pihole:pihole /var/log/pihole-FTL.log /var/log/pihole.log From 28c6b1393fbabf5cd375a7a01c33523d00f48d17 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Aug 2018 22:52:22 +0200 Subject: [PATCH 11/15] Backup existing /etc/dnsmasq.conf if present and ensure that /etc/dnsmasq.conf contains only "conf-dir=/etc/dnsmasq.d" Signed-off-by: DL6ER --- automated install/basic-install.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9f596b45..37077a36 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1092,7 +1092,7 @@ chooseBlocklists() { } # Accept a string parameter, it must be one of the default lists -# This function allow to not duplicate code in chooseBlocklists and +# This function allow to not duplicate code in chooseBlocklists and # in installDefaultBlocklists appendToListsFile() { case $1 in @@ -1113,7 +1113,7 @@ installDefaultBlocklists() { # If this file exists, we avoid overriding it. if [[ -f "${adlistFile}" ]]; then return; - fi + fi appendToListsFile StevenBlack appendToListsFile MalwareDom appendToListsFile Cameleon @@ -2109,12 +2109,14 @@ FTLinstall() { fi fi - #ensure /etc/dnsmasq.conf contains `conf-dir=/etc/dnsmasq.d` - confdir="conf-dir=/etc/dnsmasq.d" - conffile="/etc/dnsmasq.conf" - if ! grep -q "$confdir" "$conffile"; then - echo "$confdir" >> "$conffile" + # Backup existing /etc/dnsmasq.conf if present and ensure that + # /etc/dnsmasq.conf contains only "conf-dir=/etc/dnsmasq.d" + local conffile="/etc/dnsmasq.conf" + if [[ -f "${conffile}" ]]; then + mv "${conffile}" "${conffile}.old" fi + # Create /etc/dnsmasq.conf + echo "conf-dir=/etc/dnsmasq.d" > "${conffile}" return 0 # Otherwise, @@ -2483,7 +2485,7 @@ main() { echo -e " ${INFO} Restarting services..." # Start services - # Enable FTL + # Enable FTL # Ensure the service is enabled before trying to start it # Fixes a problem reported on Ubuntu 18.04 where trying to start # the service before enabling causes installer to exit From 1dca1efbc6df3d31ebf4e9ba13beedade074a8b9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 20 Aug 2018 23:11:23 +0200 Subject: [PATCH 12/15] Don't use extra query logging Signed-off-by: DL6ER --- advanced/01-pihole.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/01-pihole.conf b/advanced/01-pihole.conf index 85d260b2..f7b78ab0 100644 --- a/advanced/01-pihole.conf +++ b/advanced/01-pihole.conf @@ -39,7 +39,7 @@ interface=@INT@ cache-size=10000 -log-queries=extra +log-queries log-facility=/var/log/pihole.log local-ttl=2 From f141b9f778065a60cac0e313987b19dca934c81d Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 20 Aug 2018 17:36:16 -0400 Subject: [PATCH 13/15] Add support for privacy level 4 Signed-off-by: Mcat12 --- advanced/Scripts/webpage.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 0d4cab90..cba7af00 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -541,8 +541,8 @@ clearAudit() } SetPrivacyLevel() { - # Set privacy level. Minimum is 0, maximum is 3 - if [ "${args[2]}" -ge 0 ] && [ "${args[2]}" -le 3 ]; then + # Set privacy level. Minimum is 0, maximum is 4 + if [ "${args[2]}" -ge 0 ] && [ "${args[2]}" -le 4 ]; then changeFTLsetting "PRIVACYLEVEL" "${args[2]}" fi } From 1813c25117c828e36b1c3398c41e2546718a2b6f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Aug 2018 13:07:23 +0200 Subject: [PATCH 14/15] If config file was backed up: Print info message Signed-off-by: DL6ER --- automated install/basic-install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 37077a36..a08bb3e3 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2113,6 +2113,7 @@ FTLinstall() { # /etc/dnsmasq.conf contains only "conf-dir=/etc/dnsmasq.d" local conffile="/etc/dnsmasq.conf" if [[ -f "${conffile}" ]]; then + echo " ${INFO} Backing up ${conffile} to ${conffile}.old" mv "${conffile}" "${conffile}.old" fi # Create /etc/dnsmasq.conf From 45bf4b02e7e96708e7d4cf5a5cd97b9a0ca8c294 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 27 Aug 2018 16:51:33 +0200 Subject: [PATCH 15/15] Ensure that when pihole-FTL is launched as user root, it will drop permissions as soon as this is possible. We tell pihole-FTL to change UID/GID to pihole:pihole instead of the default nobody:nogroup. This default causes conflicts with our SQLite3 database. See that corresponding pull request for further details. Signed-off-by: DL6ER --- advanced/01-pihole.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/advanced/01-pihole.conf b/advanced/01-pihole.conf index f7b78ab0..d86b660f 100644 --- a/advanced/01-pihole.conf +++ b/advanced/01-pihole.conf @@ -24,6 +24,9 @@ addn-hosts=/etc/pihole/gravity.list addn-hosts=/etc/pihole/black.list addn-hosts=/etc/pihole/local.list +user=pihole +group=pihole + domain-needed localise-queries