From 15a9d662ac5e1aab12a7c025a48d75971b3687f0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 14 Mar 2020 11:18:43 +0000 Subject: [PATCH 01/28] Add option --comment "whatever" for adding comments for new domains through the CLI interface. Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 4f2e046f..77a5dece 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -22,6 +22,9 @@ web=false domList=() typeId="" +comment="" +declare -i domaincount +domaincount=0 colfile="/opt/pihole/COL_TABLE" source ${colfile} @@ -97,10 +100,12 @@ ValidateDomain() { fi if [[ -n "${validDomain}" ]]; then - domList=("${domList[@]}" ${validDomain}) + domList=("${domList[@]}" "${validDomain}") else echo -e " ${CROSS} ${domain} is not a valid argument or domain name!" fi + + domaincount=$((domaincount+1)) } ProcessDomainList() { @@ -151,7 +156,12 @@ AddDomain() { reload=true # Insert only the domain here. The enabled and date_added fields will be filled # with their default values (enabled = true, date_added = current timestamp) - sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});" + if [[ -z "${comment}" ]]; then + sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});" + else + # also add comment when variable has been set through the "--comment" option + sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');" + fi } RemoveDomain() { @@ -224,8 +234,16 @@ NukeList() { sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};" } -for var in "$@"; do - case "${var}" in +GetComment() { + comment="$1" + if [[ "${comment}" =~ [^a-zA-Z0-9_\#:/\.,\ -] ]]; then + echo " ${CROSS} Found invalid characters in domain comment!" + exit + fi +} + +while (( "$#" )); do + case "${1}" in "-w" | "whitelist" ) typeId=0;; "-b" | "blacklist" ) typeId=1;; "--white-regex" | "white-regex" ) typeId=2;; @@ -239,13 +257,15 @@ for var in "$@"; do "-l" | "--list" ) Displaylist;; "--nuke" ) NukeList;; "--web" ) web=true;; - * ) ValidateDomain "${var}";; + "--comment" ) GetComment "${2}"; shift;; + * ) ValidateDomain "${1}";; esac + shift done shift -if [[ $# = 0 ]]; then +if [[ ${domaincount} == 0 ]]; then helpFunc fi From 7b15a88dc45e4bda8e52a9ed6c64c1c7c44a9882 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 31 Mar 2020 18:36:40 +0000 Subject: [PATCH 02/28] Strip comments from downloaded lists instead of discarding lines with comments altogether Signed-off-by: DL6ER --- gravity.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index c421e832..2afbb17b 100755 --- a/gravity.sh +++ b/gravity.sh @@ -566,12 +566,14 @@ gravity_ParseFileIntoDomains() { # It also helps with debugging so each stage of the script can be researched more in depth # 1) Remove carriage returns # 2) Convert all characters to lowercase - # 3) Remove lines containing "#" or "/" - # 4) Remove leading tabs, spaces, etc. - # 5) Delete lines not matching domain names + # 3) Remove comments (text starting with "#", include possible spaces before the hash sign) + # 4) Remove lines containing "/" + # 5) Remove leading tabs, spaces, etc. + # 6) Delete lines not matching domain names < "${source}" tr -d '\r' | \ tr '[:upper:]' '[:lower:]' | \ - sed -r '/(\/|#).*$/d' | \ + sed 's/\s*#.*//g' | \ + sed -r '/(\/).*$/d' | \ sed -r 's/^.*\s+//g' | \ sed -r '/([^\.]+\.)+[^\.]{2,}/!d' > "${destination}" chmod 644 "${destination}" From 16f664cdb431070f716cf68833a715901e8c3677 Mon Sep 17 00:00:00 2001 From: Antoine Tenart Date: Wed, 1 Apr 2020 18:24:15 +0200 Subject: [PATCH 03/28] basic_install: remove remaining references to hosts-file.net Commit dc35709a1b3a ("Remove hosts-file.net from default lists") left a few references to hosts-file.net. Removes them. Signed-off-by: Antoine Tenart --- automated install/basic-install.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a8ac91f3..e25f7389 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1206,7 +1206,7 @@ chooseBlocklists() { mv "${adlistFile}" "${adlistFile}.old" fi # Let user select (or not) blocklists via a checklist - cmd=(whiptail --separate-output --checklist "Pi-hole relies on third party lists in order to block ads.\\n\\nYou can use the suggestions below, and/or add your own after installation\\n\\nTo deselect any list, use the arrow keys and spacebar" "${r}" "${c}" 6) + cmd=(whiptail --separate-output --checklist "Pi-hole relies on third party lists in order to block ads.\\n\\nYou can use the suggestions below, and/or add your own after installation\\n\\nTo deselect any list, use the arrow keys and spacebar" "${r}" "${c}" 5) # In an array, show the options available (all off by default): options=(StevenBlack "StevenBlack's Unified Hosts List" on MalwareDom "MalwareDomains" on @@ -1250,7 +1250,6 @@ installDefaultBlocklists() { appendToListsFile Cameleon appendToListsFile DisconTrack appendToListsFile DisconAd - appendToListsFile HostsFile } # Check if /etc/dnsmasq.conf is from pi-hole. If so replace with an original and install new in .d directory From ebbb7168a4c288e61eac30da1c0fe3df71386b22 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Sat, 4 Apr 2020 21:45:09 +0200 Subject: [PATCH 04/28] add [options] for pihole restartdns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- manpages/pihole.8 | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/manpages/pihole.8 b/manpages/pihole.8 index ed012092..9ee50110 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -1,4 +1,4 @@ -.TH "Pi-hole" "8" "Pi-hole" "Pi-hole" "May 2018" +.TH "Pi-hole" "8" "Pi-hole" "Pi-hole" "April 2020" .SH "NAME" Pi-hole : A black-hole for internet advertisements @@ -43,7 +43,7 @@ pihole -g\fR .br pihole status .br -pihole restartdns\fR +pihole restartdns\fR [options] .br \fBpihole\fR (\fBenable\fR|\fBdisable\fR [time]) .br @@ -260,9 +260,16 @@ Available commands and options: #m Disable Pi-hole functionality for # minute(s) .br -\fBrestartdns\fR +\fBrestartdns\fR [options] .br - Restart Pi-hole subsystems + Full restart Pi-hole subsystems +.br + + (restart options): +.br + reload Updates the lists, flushes dnsmasq 's DNS cache +.br + reload-lists Only update the lists WITHOUT flushing dnsmasq 's DNS cache .br \fBcheckout\fR [repo] [branch] From 3095fd4dd68a02945b32ee1bd2910349216baa3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 5 Apr 2020 08:49:35 +0200 Subject: [PATCH 05/28] add restart [options] to cli help MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- pihole | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pihole b/pihole index 6e72b4a3..90227e46 100755 --- a/pihole +++ b/pihole @@ -413,7 +413,9 @@ Options: enable Enable Pi-hole subsystems disable Disable Pi-hole subsystems Add '-h' for more info on disable usage - restartdns Restart Pi-hole subsystems + restartdns Full restart Pi-hole subsystems + Add '-- reload' to only updates the lists and flushes dnsmasq 's DNS cache + Add '--reload-lists' to only update the lists WITHOUT flushing dnsmasq 's DNS cache checkout Switch Pi-hole subsystems to a different Github branch Add '-h' for more info on checkout usage arpflush Flush information stored in Pi-hole's network tables"; From de42669bb7fe5a1ef31033a90a778292dee99cba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 5 Apr 2020 08:56:10 +0200 Subject: [PATCH 06/28] fix typo in pihole help MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- pihole | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pihole b/pihole index 90227e46..e03e29c0 100755 --- a/pihole +++ b/pihole @@ -414,7 +414,7 @@ Options: disable Disable Pi-hole subsystems Add '-h' for more info on disable usage restartdns Full restart Pi-hole subsystems - Add '-- reload' to only updates the lists and flushes dnsmasq 's DNS cache + Add '--reload' to only updates the lists and flushes dnsmasq 's DNS cache Add '--reload-lists' to only update the lists WITHOUT flushing dnsmasq 's DNS cache checkout Switch Pi-hole subsystems to a different Github branch Add '-h' for more info on checkout usage From d27a565d3950bf0fdb8011b1be7f98e3c8940ee0 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Sun, 5 Apr 2020 11:44:44 +0200 Subject: [PATCH 07/28] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: DL6ER Signed-off-by: Christian König --- manpages/pihole.8 | 4 ++-- pihole | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 9ee50110..84449429 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -267,9 +267,9 @@ Available commands and options: (restart options): .br - reload Updates the lists, flushes dnsmasq 's DNS cache + reload Updates the lists and flushes DNS cache .br - reload-lists Only update the lists WITHOUT flushing dnsmasq 's DNS cache + reload-lists Updates the lists WITHOUT flushing the DNS cache .br \fBcheckout\fR [repo] [branch] diff --git a/pihole b/pihole index e03e29c0..c0f916b2 100755 --- a/pihole +++ b/pihole @@ -414,8 +414,8 @@ Options: disable Disable Pi-hole subsystems Add '-h' for more info on disable usage restartdns Full restart Pi-hole subsystems - Add '--reload' to only updates the lists and flushes dnsmasq 's DNS cache - Add '--reload-lists' to only update the lists WITHOUT flushing dnsmasq 's DNS cache + Add 'reload' to update the lists and flush the cache without restarting the DNS server + Add 'reload-lists' to only update the lists WITHOUT flushing the cache or restarting the DNS server checkout Switch Pi-hole subsystems to a different Github branch Add '-h' for more info on checkout usage arpflush Flush information stored in Pi-hole's network tables"; From a9b19df4ec2b99442704cf2b7656908e3e10bc14 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Apr 2020 12:28:33 +0100 Subject: [PATCH 08/28] expand email validation regex to catch more valid emails see comments on PR #3254 Signed-off-by: Adam Warner --- advanced/Scripts/webpage.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index f0f8bc31..b60428f6 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -519,7 +519,9 @@ Options: if [[ -n "${args[2]}" ]]; then # Sanitize email address in case of security issues - if [[ ! "${args[2]}" =~ ^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,4}$ ]]; then + local regex + regex="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$" + if [[ ! "${args[2]}" =~ ${regex} ]]; then echo -e " ${CROSS} Invalid email address" exit 0 fi From b6ac1585ec3c093c471b64485b1da36346a9a58a Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Apr 2020 12:29:45 +0100 Subject: [PATCH 09/28] add regex attribution Signed-off-by: Adam Warner --- advanced/Scripts/webpage.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index b60428f6..5279cab3 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -519,6 +519,7 @@ Options: if [[ -n "${args[2]}" ]]; then # Sanitize email address in case of security issues + # Regex from https://stackoverflow.com/a/2138832/4065967 local regex regex="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$" if [[ ! "${args[2]}" =~ ${regex} ]]; then From 26f71e4dbe08517ebd35cc972b70462bb1823e5e Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Apr 2020 12:34:14 +0100 Subject: [PATCH 10/28] accidentally a space Signed-off-by: Adam Warner --- advanced/Scripts/webpage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 5279cab3..3a04bbd7 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -522,7 +522,7 @@ Options: # Regex from https://stackoverflow.com/a/2138832/4065967 local regex regex="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$" - if [[ ! "${args[2]}" =~ ${regex} ]]; then + if [[ ! "${args[2]}" =~ ${regex} ]]; then echo -e " ${CROSS} Invalid email address" exit 0 fi From 308eb5eda5c32fd262c09a81c5df7ac2433886d9 Mon Sep 17 00:00:00 2001 From: Matthias Schoettle Date: Tue, 10 Mar 2020 14:31:05 -0400 Subject: [PATCH 11/28] Fixes broken blocking page and landing page when changing server port and/or hostname. See issues #2195 and #2720. Signed-off-by: Matthias Schoettle --- advanced/index.php | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/advanced/index.php b/advanced/index.php index b0c4a7c3..4f2a17f7 100644 --- a/advanced/index.php +++ b/advanced/index.php @@ -6,8 +6,8 @@ * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -// Sanitise HTTP_HOST output -$serverName = htmlspecialchars($_SERVER["HTTP_HOST"]); +// Sanitize SERVER_NAME output +$serverName = htmlspecialchars($_SERVER["SERVER_NAME"]); // Remove external ipv6 brackets if any $serverName = preg_replace('/^\[(.*)\]$/', '${1}', $serverName); @@ -50,16 +50,24 @@ function setHeader($type = "x") { } // Determine block page type -if ($serverName === "pi.hole") { +if ($serverName === "pi.hole" + || (!empty($_SERVER["VIRTUAL_HOST"]) && $serverName === $_SERVER["VIRTUAL_HOST"])) { // Redirect to Web Interface exit(header("Location: /admin")); } elseif (filter_var($serverName, FILTER_VALIDATE_IP) || in_array($serverName, $authorizedHosts)) { // Set Splash Page output $splashPage = " - + + $viewPort - -
Pi-hole: Your black hole for Internet advertisements
Did you mean to go to the admin panel? + + + +
+ Pi-hole: Your black hole for Internet advertisements
+ Did you mean to go to the admin panel? + + "; // Set splash/landing page based off presence of $landPage @@ -68,7 +76,7 @@ if ($serverName === "pi.hole") { // Unset variables so as to not be included in $landPage unset($serverName, $svPasswd, $svEmail, $authorizedHosts, $validExtTypes, $currentUrlExt, $viewPort); - // Render splash/landing page when directly browsing via IP or authorised hostname + // Render splash/landing page when directly browsing via IP or authorized hostname exit($renderPage); } elseif ($currentUrlExt === "js") { // Serve Pi-hole Javascript for blocked domains requesting JS @@ -131,7 +139,12 @@ ini_set("default_socket_timeout", 3); function queryAds($serverName) { // Determine the time it takes while querying adlists $preQueryTime = microtime(true)-$_SERVER["REQUEST_TIME_FLOAT"]; - $queryAds = file("http://127.0.0.1/admin/scripts/pi-hole/php/queryads.php?domain=$serverName&bp", FILE_IGNORE_NEW_LINES); + $queryAdsURL = sprintf( + "http://127.0.0.1:%s/admin/scripts/pi-hole/php/queryads.php?domain=%s&bp", + $_SERVER["SERVER_PORT"], + $serverName + ); + $queryAds = file($queryAdsURL, FILE_IGNORE_NEW_LINES); $queryAds = array_values(array_filter(preg_replace("/data:\s+/", "", $queryAds))); $queryTime = sprintf("%.0f", (microtime(true)-$_SERVER["REQUEST_TIME_FLOAT"]) - $preQueryTime); @@ -209,7 +222,7 @@ $phVersion = exec("cd /etc/.pihole/ && git describe --long --tags"); if (explode("-", $phVersion)[1] != "0") $execTime = microtime(true)-$_SERVER["REQUEST_TIME_FLOAT"]; -// Please Note: Text is added via CSS to allow an admin to provide a localised +// Please Note: Text is added via CSS to allow an admin to provide a localized // language without the need to edit this file setHeader(); @@ -226,10 +239,10 @@ setHeader(); - - + + ● <?=$serverName ?> - +