From d254d6075afa7b2c71003c4fa3e103894d0be01d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 6 Jun 2018 19:26:56 +0200 Subject: [PATCH 01/22] First step from wildcards to regex lists for blocking Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 46 ++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 1d96ea3c..a54d7dc8 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -13,7 +13,7 @@ basename=pihole piholeDir=/etc/"${basename}" whitelist="${piholeDir}"/whitelist.txt blacklist="${piholeDir}"/blacklist.txt -readonly wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf" +readonly regexlist="/etc/pihole/regex.list" reload=false addmode=true verbose=true @@ -31,7 +31,7 @@ helpFunc() { if [[ "${listMain}" == "${whitelist}" ]]; then param="w" type="white" - elif [[ "${listMain}" == "${wildcardlist}" ]]; then + elif [[ "${listMain}" == "${regexlist}" ]]; then param="wild" type="wildcard black" else @@ -57,7 +57,8 @@ Options: EscapeRegexp() { # This way we may safely insert an arbitrary # string in our regular expressions - # Also remove leading "." if present + # This sed is intentionally executed in three steps to ease maintainability + # The first sed removes any amount of leading dots echo $* | sed 's/^\.*//' | sed "s/[]\.|$(){}?+*^]/\\\\&/g" | sed "s/\\//\\\\\//g" } @@ -94,9 +95,6 @@ PoplistFile() { if ${addmode}; then AddDomain "${dom}" "${listMain}" RemoveDomain "${dom}" "${listAlt}" - if [[ "${listMain}" == "${whitelist}" || "${listMain}" == "${blacklist}" ]]; then - RemoveDomain "${dom}" "${wildcardlist}" - fi else RemoveDomain "${dom}" "${listMain}" fi @@ -109,7 +107,6 @@ AddDomain() { [[ "${list}" == "${whitelist}" ]] && listname="whitelist" [[ "${list}" == "${blacklist}" ]] && listname="blacklist" - [[ "${list}" == "${wildcardlist}" ]] && listname="wildcard blacklist" if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only" @@ -121,7 +118,7 @@ AddDomain() { if [[ "${bool}" == false ]]; then # Domain not found in the whitelist file, add it! if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding $1 to $listname..." + echo -e " ${INFO} Adding ${1} to ${listname}..." fi reload=true # Add it to the list we want to add it to @@ -131,28 +128,22 @@ AddDomain() { echo -e " ${INFO} ${1} already exists in ${listname}, no need to add!" fi fi - elif [[ "${list}" == "${wildcardlist}" ]]; then - source "${piholeDir}/setupVars.conf" - # Remove the /* from the end of the IP addresses - IPV4_ADDRESS=${IPV4_ADDRESS%/*} - IPV6_ADDRESS=${IPV6_ADDRESS%/*} + elif [[ "${list}" == "${regexlist}" ]]; then [[ -z "${type}" ]] && type="--wildcard-only" bool=true # Is the domain in the list? - grep -e "address=\/${domain}\/" "${wildcardlist}" > /dev/null 2>&1 || bool=false + # Search only for exactly matching lines + grep -E "^${domain}$" "${regexlist}" > /dev/null 2>&1 || bool=false if [[ "${bool}" == false ]]; then if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding $1 to wildcard blacklist..." + echo -e " ${INFO} Adding ${1} to regex list..." fi reload="restart" - echo "address=/$1/${IPV4_ADDRESS}" >> "${wildcardlist}" - if [[ "${#IPV6_ADDRESS}" > 0 ]]; then - echo "address=/$1/${IPV6_ADDRESS}" >> "${wildcardlist}" - fi + echo "$1" >> "${regexlist}" else if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} already exists in wildcard blacklist, no need to add!" + echo -e " ${INFO} ${1} already exists in regex list, no need to add!" fi fi fi @@ -164,7 +155,6 @@ RemoveDomain() { [[ "${list}" == "${whitelist}" ]] && listname="whitelist" [[ "${list}" == "${blacklist}" ]] && listname="blacklist" - [[ "${list}" == "${wildcardlist}" ]] && listname="wildcard blacklist" if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then bool=true @@ -174,7 +164,7 @@ RemoveDomain() { grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false if [[ "${bool}" == true ]]; then # Remove it from the other one - echo -e " ${INFO} Removing $1 from $listname..." + echo -e " ${INFO} Removing $1 from ${listname}..." # /I flag: search case-insensitive sed -i "/${domain}/Id" "${list}" reload=true @@ -183,20 +173,20 @@ RemoveDomain() { echo -e " ${INFO} ${1} does not exist in ${listname}, no need to remove!" fi fi - elif [[ "${list}" == "${wildcardlist}" ]]; then + elif [[ "${list}" == "${regexlist}" ]]; then [[ -z "${type}" ]] && type="--wildcard-only" bool=true # Is it in the list? - grep -e "address=\/${domain}\/" "${wildcardlist}" > /dev/null 2>&1 || bool=false + grep -E "^${domain}$" "${regexlist}" > /dev/null 2>&1 || bool=false if [[ "${bool}" == true ]]; then # Remove it from the other one - echo -e " ${INFO} Removing $1 from $listname..." + echo -e " ${INFO} Removing $1 from regex list..." # /I flag: search case-insensitive - sed -i "/address=\/${domain}/Id" "${list}" + sed -i "/${domain}/Id" "${list}" reload=true else if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} does not exist in ${listname}, no need to remove!" + echo -e " ${INFO} ${1} does not exist in regex list, no need to remove!" fi fi fi @@ -241,7 +231,7 @@ for var in "$@"; do case "${var}" in "-w" | "whitelist" ) listMain="${whitelist}"; listAlt="${blacklist}";; "-b" | "blacklist" ) listMain="${blacklist}"; listAlt="${whitelist}";; - "-wild" | "wildcard" ) listMain="${wildcardlist}";; + "-wild" | "wildcard" ) listMain="${regexlist}";; "-nr"| "--noreload" ) reload=false;; "-d" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; From bc705aac039ba7c3fbfcd0aa73b0d991db2b8a14 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 17 Jun 2018 14:26:57 +0200 Subject: [PATCH 02/22] Add automated wildcard list -> regex filter conversion Signed-off-by: DL6ER --- advanced/Scripts/wildcard_regex_converter.sh | 18 ++++++++++++++++++ gravity.sh | 17 ++++++++--------- 2 files changed, 26 insertions(+), 9 deletions(-) create mode 100644 advanced/Scripts/wildcard_regex_converter.sh diff --git a/advanced/Scripts/wildcard_regex_converter.sh b/advanced/Scripts/wildcard_regex_converter.sh new file mode 100644 index 00000000..e862f391 --- /dev/null +++ b/advanced/Scripts/wildcard_regex_converter.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +wildcardFile="/etc/dnsmasq.d/03-pihole-wildcard.conf" + +convert_wildcard_to_regex() { + if [ ! -f "${wildcardFile}" ]; then + return + fi + local addrlines domains uniquedomains + # Obtain wildcard domains from old file + addrlines="$(grep -oE "/.*/" ${wildcardFile})" + # Strip "/" from domain names + domains="$(sed 's/\///g;' <<< "${addrlines}")" + # Remove repeated domains (may have been inserted two times due to A and AAAA blocking) + uniquedomains="$(uniq <<< "${domains}")" + # Automatically generate regex filters and remove old wildcards file + awk '{print "(^)|(\\.)"$0"$"}' <<< "${uniquedomains}" >> "${regexFile}" && rm "${wildcardFile}" +} diff --git a/gravity.sh b/gravity.sh index 58419029..0f64be68 100755 --- a/gravity.sh +++ b/gravity.sh @@ -15,6 +15,8 @@ export LC_ALL=C coltable="/opt/pihole/COL_TABLE" source "${coltable}" +regexconverter="/opt/pihole/wildcard_regex_converter.sh" +source "${regexconverter}" basename="pihole" PIHOLE_COMMAND="/usr/local/bin/${basename}" @@ -26,7 +28,7 @@ adListDefault="${piholeDir}/adlists.default" whitelistFile="${piholeDir}/whitelist.txt" blacklistFile="${piholeDir}/blacklist.txt" -wildcardFile="/etc/dnsmasq.d/03-pihole-wildcard.conf" +regexFile="${piholeDir}/regex.list" adList="${piholeDir}/gravity.list" blackList="${piholeDir}/black.list" @@ -453,7 +455,7 @@ gravity_Whitelist() { echo -e "${OVER} ${INFO} ${str}" } -# Output count of blacklisted domains and wildcards +# Output count of blacklisted domains and regex filters gravity_ShowBlockCount() { local num @@ -462,13 +464,9 @@ gravity_ShowBlockCount() { echo -e " ${INFO} Number of blacklisted domains: ${num}" fi - if [[ -f "${wildcardFile}" ]]; then - num=$(grep -c "^" "${wildcardFile}") - # If IPv4 and IPv6 is used, divide total wildcard count by 2 - if [[ -n "${IPV4_ADDRESS}" ]] && [[ -n "${IPV6_ADDRESS}" ]];then - num=$(( num/2 )) - fi - echo -e " ${INFO} Number of wildcard blocked domains: ${num}" + if [[ -f "${regexFile}" ]]; then + num=$(grep -c "^" "${regexFile}") + echo -e " ${INFO} Number of regex filters: ${num}" fi } @@ -646,6 +644,7 @@ if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "whitelist" ]]; then gravity_Whitelist fi +convert_wildcard_to_regex gravity_ShowBlockCount # Perform when downloading blocklists, or modifying the white/blacklist (not wildcards) From cd026cd865603bac7ca0e1b88b7e1bce5556c556 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 17 Jun 2018 14:43:33 +0200 Subject: [PATCH 03/22] Add comment where regexList is defined Signed-off-by: DL6ER --- advanced/Scripts/wildcard_regex_converter.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/advanced/Scripts/wildcard_regex_converter.sh b/advanced/Scripts/wildcard_regex_converter.sh index e862f391..7c80d07e 100644 --- a/advanced/Scripts/wildcard_regex_converter.sh +++ b/advanced/Scripts/wildcard_regex_converter.sh @@ -1,4 +1,14 @@ #!/bin/bash +# Pi-hole: A black hole for Internet advertisements +# (c) 2017 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# Provides an automated migration subroutine to convert Pi-hole v3.x wildcard domains to Pi-hole v4.x regex filters +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + +# regexFile set in gravity.sh wildcardFile="/etc/dnsmasq.d/03-pihole-wildcard.conf" From c7afe3e9a4a5728faf3c61722609b66d885be8b8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 17 Jun 2018 14:46:26 +0200 Subject: [PATCH 04/22] Please stickler by making clear that the variable regexFile is declared outside of this routine Signed-off-by: DL6ER --- advanced/Scripts/wildcard_regex_converter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/wildcard_regex_converter.sh b/advanced/Scripts/wildcard_regex_converter.sh index 7c80d07e..46c5e952 100644 --- a/advanced/Scripts/wildcard_regex_converter.sh +++ b/advanced/Scripts/wildcard_regex_converter.sh @@ -24,5 +24,5 @@ convert_wildcard_to_regex() { # Remove repeated domains (may have been inserted two times due to A and AAAA blocking) uniquedomains="$(uniq <<< "${domains}")" # Automatically generate regex filters and remove old wildcards file - awk '{print "(^)|(\\.)"$0"$"}' <<< "${uniquedomains}" >> "${regexFile}" && rm "${wildcardFile}" + awk '{print "(^)|(\\.)"$0"$"}' <<< "${uniquedomains}" >> "${regexFile:?}" && rm "${wildcardFile}" } From 2809579dd736dfe5f5a5273c55777baf59e3eb8e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 17 Jun 2018 14:59:21 +0200 Subject: [PATCH 05/22] Refine regex filter generation Signed-off-by: DL6ER --- advanced/Scripts/wildcard_regex_converter.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/wildcard_regex_converter.sh b/advanced/Scripts/wildcard_regex_converter.sh index 46c5e952..d8ebeeb8 100644 --- a/advanced/Scripts/wildcard_regex_converter.sh +++ b/advanced/Scripts/wildcard_regex_converter.sh @@ -19,10 +19,10 @@ convert_wildcard_to_regex() { local addrlines domains uniquedomains # Obtain wildcard domains from old file addrlines="$(grep -oE "/.*/" ${wildcardFile})" - # Strip "/" from domain names - domains="$(sed 's/\///g;' <<< "${addrlines}")" + # Strip "/" from domain names and convert "." to regex-compatible "\." + domains="$(sed 's/\///g;s/\./\\./g' <<< "${addrlines}")" # Remove repeated domains (may have been inserted two times due to A and AAAA blocking) uniquedomains="$(uniq <<< "${domains}")" # Automatically generate regex filters and remove old wildcards file - awk '{print "(^)|(\\.)"$0"$"}' <<< "${uniquedomains}" >> "${regexFile:?}" && rm "${wildcardFile}" + awk '{print "((^)|(\\.))"$0"$"}' <<< "${uniquedomains}" >> "${regexFile:?}" && rm "${wildcardFile}" } From 55175087c4738063b2c778befbc6517bd74c6c14 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Thu, 28 Jun 2018 22:52:44 -0400 Subject: [PATCH 06/22] Set regex file permissions for web access Signed-off-by: Mcat12 --- gravity.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gravity.sh b/gravity.sh index 0f64be68..aaab6237 100755 --- a/gravity.sh +++ b/gravity.sh @@ -644,6 +644,11 @@ if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "whitelist" ]]; then gravity_Whitelist fi +# Set proper permissions on the regex file +touch "${regexFile}" +chown pihole:www-data "${regexFile}" +chmod 664 "${regexFile}" + convert_wildcard_to_regex gravity_ShowBlockCount From cffb4de83b99361a46ced3ba33253fe69c987075 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Thu, 28 Jun 2018 22:59:15 -0400 Subject: [PATCH 07/22] Use strict grep instead of regex for duplicates Signed-off-by: Mcat12 --- advanced/Scripts/list.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index a54d7dc8..3a9c8ece 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -133,7 +133,7 @@ AddDomain() { bool=true # Is the domain in the list? # Search only for exactly matching lines - grep -E "^${domain}$" "${regexlist}" > /dev/null 2>&1 || bool=false + grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false if [[ "${bool}" == false ]]; then if [[ "${verbose}" == true ]]; then @@ -177,7 +177,7 @@ RemoveDomain() { [[ -z "${type}" ]] && type="--wildcard-only" bool=true # Is it in the list? - grep -E "^${domain}$" "${regexlist}" > /dev/null 2>&1 || bool=false + grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false if [[ "${bool}" == true ]]; then # Remove it from the other one echo -e " ${INFO} Removing $1 from regex list..." From 8435eeed4dc5b7bf62dd56a8db3c0760e78afd90 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Thu, 28 Jun 2018 23:21:01 -0400 Subject: [PATCH 08/22] Don't validate regex domains Signed-off-by: Mcat12 --- advanced/Scripts/list.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 3a9c8ece..04eed7db 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -66,10 +66,14 @@ HandleOther() { # Convert to lowercase domain="${1,,}" - # Check validity of domain + # Check validity of domain (don't check for regex entries) if [[ "${#domain}" -le 253 ]]; then - validDomain=$(grep -P "^((-|_)*[a-z\d]((-|_)*[a-z\d])*(-|_)*)(\.(-|_)*([a-z\d]((-|_)*[a-z\d])*))*$" <<< "${domain}") # Valid chars check - validDomain=$(grep -P "^[^\.]{1,63}(\.[^\.]{1,63})*$" <<< "${validDomain}") # Length of each label + if [[ "${listMain}" == "${regexlist}" ]]; then + validDomain="" + else + validDomain=$(grep -P "^((-|_)*[a-z\d]((-|_)*[a-z\d])*(-|_)*)(\.(-|_)*([a-z\d]((-|_)*[a-z\d])*))*$" <<< "${domain}") # Valid chars check + validDomain=$(grep -P "^[^\.]{1,63}(\.[^\.]{1,63})*$" <<< "${validDomain}") # Length of each label + fi fi if [[ -n "${validDomain}" ]]; then From 4a953b66e0e989307dcfd1114cd06b8f1f242ffa Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 29 Jun 2018 22:51:37 -0400 Subject: [PATCH 09/22] Make removing regex domains work correctly Signed-off-by: Mcat12 --- advanced/Scripts/list.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 04eed7db..38eea69f 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -69,7 +69,7 @@ HandleOther() { # Check validity of domain (don't check for regex entries) if [[ "${#domain}" -le 253 ]]; then if [[ "${listMain}" == "${regexlist}" ]]; then - validDomain="" + validDomain="${domain}" else validDomain=$(grep -P "^((-|_)*[a-z\d]((-|_)*[a-z\d])*(-|_)*)(\.(-|_)*([a-z\d]((-|_)*[a-z\d])*))*$" <<< "${domain}") # Valid chars check validDomain=$(grep -P "^[^\.]{1,63}(\.[^\.]{1,63})*$" <<< "${validDomain}") # Length of each label @@ -185,8 +185,9 @@ RemoveDomain() { if [[ "${bool}" == true ]]; then # Remove it from the other one echo -e " ${INFO} Removing $1 from regex list..." - # /I flag: search case-insensitive - sed -i "/${domain}/Id" "${list}" + local lineNumber + lineNumber=$(grep -Fnx "$1" "${list}" | cut -f1 -d:) + sed -i "${lineNumber}d" "${list}" reload=true else if [[ "${verbose}" == true ]]; then From 0decc37b5ad7371f5faa0c4cd1012e8ddab3b9f9 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 29 Jun 2018 22:55:00 -0400 Subject: [PATCH 10/22] Appease stickler Signed-off-by: Mcat12 --- advanced/Scripts/list.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 38eea69f..b2a15434 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -71,8 +71,8 @@ HandleOther() { if [[ "${listMain}" == "${regexlist}" ]]; then validDomain="${domain}" else - validDomain=$(grep -P "^((-|_)*[a-z\d]((-|_)*[a-z\d])*(-|_)*)(\.(-|_)*([a-z\d]((-|_)*[a-z\d])*))*$" <<< "${domain}") # Valid chars check - validDomain=$(grep -P "^[^\.]{1,63}(\.[^\.]{1,63})*$" <<< "${validDomain}") # Length of each label + validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check + validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label fi fi From 3077c22e4fa9cd7ebf69f759df3f70e03d53e67e Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 29 Jun 2018 23:10:49 -0400 Subject: [PATCH 11/22] Change help strings from wildcard to regex blacklist Signed-off-by: Mcat12 --- advanced/Scripts/list.sh | 2 +- pihole | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index b2a15434..bd80c3d8 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -33,7 +33,7 @@ helpFunc() { type="white" elif [[ "${listMain}" == "${regexlist}" ]]; then param="wild" - type="wildcard black" + type="regex black" else param="b" type="black" diff --git a/pihole b/pihole index b3f532af..8cda6938 100755 --- a/pihole +++ b/pihole @@ -599,7 +599,7 @@ Add '-h' after specific commands for more information on usage Whitelist/Blacklist Options: -w, whitelist Whitelist domain(s) -b, blacklist Blacklist domain(s) - -wild, wildcard Blacklist domain(s), and all its subdomains + -wild, wildcard Regex blacklist domain(s) Add '-h' for more info on whitelist/blacklist usage Debugging Options: From 61f0cbe10a1bd9f61b4f9307a2d4d41c765bc278 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 29 Jun 2018 23:34:15 -0400 Subject: [PATCH 12/22] Update pihole man page for regex Signed-off-by: Mcat12 --- manpages/pihole.8 | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 54bf4a31..76ca0c7c 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -68,7 +68,7 @@ Available commands and options: \fB-wild, wildcard\fR [options] [ ] .br - Add or removes specified domain, and all subdomains to the blacklist + Add or removes specified regex pattern to the regex blacklist .br (Whitelist/Blacklist manipulation options): @@ -167,9 +167,9 @@ Available commands and options: Show a help dialog .br -\fB-l, logging\fR [on|off|off noflush] +\fB-l, logging\fR [on|off|off noflush] .br - Specify whether the Pi-hole log should be used + Specify whether the Pi-hole log should be used .br (Logging options): @@ -193,7 +193,7 @@ Available commands and options: .br Show installed versions of Pi-hole, Web Interface & FTL .br - + .br (repo options): .br @@ -232,7 +232,7 @@ Available commands and options: Disable Pi-hole subsystems, optionally for a set duration .br - (time options): + (time options): .br #s Disable Pi-hole functionality for # second(s) .br @@ -275,9 +275,9 @@ Some usage examples \fBpihole -w iloveads.example.com\fR Add "iloveads.example.com" to whitelist .br - \fBpihole -b -d noads.example.com\fR Remove "noads.example.com" from blacklist + \fBpihole -b -d noads.example.com\fR Remove "noads.example.com" from blacklist .br - \fBpihole -wild example.com\fR Add "example.com" as wildcard - would block ads.example.com, www.example.com etc. + \fBpihole -wild ^example.*$\fR Add "^example.*$" as a regex pattern - would block all domains starting with "example" .br Changing the Web Interface password From c5bb404d6a69a3c6a2bb6a329fe36118d19f4dcd Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sun, 8 Jul 2018 14:37:33 -0400 Subject: [PATCH 13/22] Emulate wildcard blacklisting via -wild and move regex to -regex Signed-off-by: Mcat12 --- advanced/Scripts/list.sh | 11 +++++++++-- manpages/pihole.8 | 19 ++++++++++++++----- pihole | 22 +++++++--------------- 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index bd80c3d8..def65f96 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -17,6 +17,7 @@ readonly regexlist="/etc/pihole/regex.list" reload=false addmode=true verbose=true +wildcard=false domList=() @@ -68,7 +69,7 @@ HandleOther() { # Check validity of domain (don't check for regex entries) if [[ "${#domain}" -le 253 ]]; then - if [[ "${listMain}" == "${regexlist}" ]]; then + if [[ "${listMain}" == "${regexlist}" && "${wildcard}" == false ]]; then validDomain="${domain}" else validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check @@ -135,6 +136,9 @@ AddDomain() { elif [[ "${list}" == "${regexlist}" ]]; then [[ -z "${type}" ]] && type="--wildcard-only" bool=true + + [[ "${wildcard}" == true ]] && domain="((^)|(\\.))${domain}$" + # Is the domain in the list? # Search only for exactly matching lines grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false @@ -179,6 +183,8 @@ RemoveDomain() { fi elif [[ "${list}" == "${regexlist}" ]]; then [[ -z "${type}" ]] && type="--wildcard-only" + [[ "${wildcard}" == true ]] && domain="((^)|(\\.))${domain}$" + bool=true # Is it in the list? grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false @@ -236,7 +242,8 @@ for var in "$@"; do case "${var}" in "-w" | "whitelist" ) listMain="${whitelist}"; listAlt="${blacklist}";; "-b" | "blacklist" ) listMain="${blacklist}"; listAlt="${whitelist}";; - "-wild" | "wildcard" ) listMain="${regexlist}";; + "-wild" | "wildcard" ) listMain="${regexlist}"; wildcard=true;; + "-regex" | "regex" ) listMain="${regexlist}";; "-nr"| "--noreload" ) reload=false;; "-d" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 76ca0c7c..68cd2864 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -5,7 +5,7 @@ Pi-hole : A black-hole for internet advertisements .br .SH "SYNOPSIS" -\fBpihole\fR (\fB-w\fR|\fB-b\fR|\fB-wild\fR) [options] domain(s) +\fBpihole\fR (\fB-w\fR|\fB-b\fR|\fB-wild\fR|\fB-regex\fR) [options] domain(s) .br \fBpihole -a\fR \fB-p\fR password .br @@ -68,7 +68,12 @@ Available commands and options: \fB-wild, wildcard\fR [options] [ ] .br - Add or removes specified regex pattern to the regex blacklist + Add or removes specified domain to the wildcard blacklist +.br + +\fB-regex, regex\fR [options] [ ] +.br + Add or removes specified regex filter to the regex blacklist .br (Whitelist/Blacklist manipulation options): @@ -273,11 +278,15 @@ Some usage examples Whitelist/blacklist manipulation .br - \fBpihole -w iloveads.example.com\fR Add "iloveads.example.com" to whitelist + \fBpihole -w iloveads.example.com\fR Add "iloveads.example.com" to whitelist +.br + \fBpihole -b -d noads.example.com\fR Remove "noads.example.com" from blacklist .br - \fBpihole -b -d noads.example.com\fR Remove "noads.example.com" from blacklist + \fBpihole -wild "^example.*$"\fR Add "^example.*$" as a regex pattern - would + block all domains starting with "example" .br - \fBpihole -wild ^example.*$\fR Add "^example.*$" as a regex pattern - would block all domains starting with "example" + \fBpihole -regex "ad.*\.example\.com$"\fR Add "ad.*\.example\.com$" to the regex + blacklist - would block all subdomains of example.com which start with "ad" .br Changing the Web Interface password diff --git a/pihole b/pihole index ac558687..ce8ea82b 100755 --- a/pihole +++ b/pihole @@ -33,17 +33,7 @@ webpageFunc() { exit 0 } -whitelistFunc() { - "${PI_HOLE_SCRIPT_DIR}"/list.sh "$@" - exit 0 -} - -blacklistFunc() { - "${PI_HOLE_SCRIPT_DIR}"/list.sh "$@" - exit 0 -} - -wildcardFunc() { +listFunc() { "${PI_HOLE_SCRIPT_DIR}"/list.sh "$@" exit 0 } @@ -403,7 +393,8 @@ Add '-h' after specific commands for more information on usage Whitelist/Blacklist Options: -w, whitelist Whitelist domain(s) -b, blacklist Blacklist domain(s) - -wild, wildcard Regex blacklist domain(s) + -wild, wildcard Wildcard blacklist domain(s) + -regex, regex Regex blacklist domains(s) Add '-h' for more info on whitelist/blacklist usage Debugging Options: @@ -445,9 +436,10 @@ fi # Handle redirecting to specific functions based on arguments case "${1}" in - "-w" | "whitelist" ) whitelistFunc "$@";; - "-b" | "blacklist" ) blacklistFunc "$@";; - "-wild" | "wildcard" ) wildcardFunc "$@";; + "-w" | "whitelist" ) listFunc "$@";; + "-b" | "blacklist" ) listFunc "$@";; + "-wild" | "wildcard" ) listFunc "$@";; + "-regex" | "regex" ) listFunc "$@";; "-d" | "debug" ) debugFunc "$@";; "-f" | "flush" ) flushFunc "$@";; "-up" | "updatePihole" ) updatePiholeFunc "$@";; From cfd95dbe87260dad85884288b5440e2d1d67d6cc Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sun, 8 Jul 2018 15:38:44 -0400 Subject: [PATCH 14/22] Fix incorrect variable usage Signed-off-by: Mcat12 --- advanced/Scripts/list.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index def65f96..3b7b0254 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -136,6 +136,7 @@ AddDomain() { elif [[ "${list}" == "${regexlist}" ]]; then [[ -z "${type}" ]] && type="--wildcard-only" bool=true + domain="${1}" [[ "${wildcard}" == true ]] && domain="((^)|(\\.))${domain}$" @@ -145,13 +146,13 @@ AddDomain() { if [[ "${bool}" == false ]]; then if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding ${1} to regex list..." + echo -e " ${INFO} Adding ${domain} to regex list..." fi reload="restart" - echo "$1" >> "${regexlist}" + echo "$domain" >> "${regexlist}" else if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} already exists in regex list, no need to add!" + echo -e " ${INFO} ${domain} already exists in regex list, no need to add!" fi fi fi @@ -183,6 +184,8 @@ RemoveDomain() { fi elif [[ "${list}" == "${regexlist}" ]]; then [[ -z "${type}" ]] && type="--wildcard-only" + domain="${1}" + [[ "${wildcard}" == true ]] && domain="((^)|(\\.))${domain}$" bool=true @@ -190,14 +193,14 @@ RemoveDomain() { grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false if [[ "${bool}" == true ]]; then # Remove it from the other one - echo -e " ${INFO} Removing $1 from regex list..." + echo -e " ${INFO} Removing $domain from regex list..." local lineNumber - lineNumber=$(grep -Fnx "$1" "${list}" | cut -f1 -d:) + lineNumber=$(grep -Fnx "$domain" "${list}" | cut -f1 -d:) sed -i "${lineNumber}d" "${list}" reload=true else if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} does not exist in regex list, no need to remove!" + echo -e " ${INFO} ${domain} does not exist in regex list, no need to remove!" fi fi fi From 70f2b26c86d52f88d29d08131480a5df63468bfa Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sun, 8 Jul 2018 15:45:06 -0400 Subject: [PATCH 15/22] Escape periods in wildcard converted regex filters. Signed-off-by: Mcat12 --- advanced/Scripts/list.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 3b7b0254..0c370759 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -138,7 +138,7 @@ AddDomain() { bool=true domain="${1}" - [[ "${wildcard}" == true ]] && domain="((^)|(\\.))${domain}$" + [[ "${wildcard}" == true ]] && domain="((^)|(\\.))$(echo domain | sed "s/\./\\\./g")$" # Is the domain in the list? # Search only for exactly matching lines @@ -186,7 +186,7 @@ RemoveDomain() { [[ -z "${type}" ]] && type="--wildcard-only" domain="${1}" - [[ "${wildcard}" == true ]] && domain="((^)|(\\.))${domain}$" + [[ "${wildcard}" == true ]] && domain="((^)|(\\.))$(echo domain | sed "s/\./\\\./g")$" bool=true # Is it in the list? From af5fc6547ac7633d1d6e687187a5fdea6ca81627 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sun, 8 Jul 2018 15:47:18 -0400 Subject: [PATCH 16/22] Fix echoing wrong domain when converting wildcard Signed-off-by: Mcat12 --- advanced/Scripts/list.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 0c370759..98a371eb 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -138,7 +138,7 @@ AddDomain() { bool=true domain="${1}" - [[ "${wildcard}" == true ]] && domain="((^)|(\\.))$(echo domain | sed "s/\./\\\./g")$" + [[ "${wildcard}" == true ]] && domain="((^)|(\\.))$(echo "${domain}" | sed "s/\./\\\./g")$" # Is the domain in the list? # Search only for exactly matching lines @@ -186,7 +186,7 @@ RemoveDomain() { [[ -z "${type}" ]] && type="--wildcard-only" domain="${1}" - [[ "${wildcard}" == true ]] && domain="((^)|(\\.))$(echo domain | sed "s/\./\\\./g")$" + [[ "${wildcard}" == true ]] && domain="((^)|(\\.))$(echo "${domain}" | sed "s/\./\\\./g")$" bool=true # Is it in the list? From 1ca852191f547e608c9eeede90cfb6771c3b866e Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sun, 8 Jul 2018 15:50:05 -0400 Subject: [PATCH 17/22] Fix stickler complaints Signed-off-by: Mcat12 --- advanced/Scripts/list.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 98a371eb..ba710445 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -138,7 +138,7 @@ AddDomain() { bool=true domain="${1}" - [[ "${wildcard}" == true ]] && domain="((^)|(\\.))$(echo "${domain}" | sed "s/\./\\\./g")$" + [[ "${wildcard}" == true ]] && domain="((^)|(\\.))${domain//\\./\\\.}$" # Is the domain in the list? # Search only for exactly matching lines @@ -186,7 +186,7 @@ RemoveDomain() { [[ -z "${type}" ]] && type="--wildcard-only" domain="${1}" - [[ "${wildcard}" == true ]] && domain="((^)|(\\.))$(echo "${domain}" | sed "s/\./\\\./g")$" + [[ "${wildcard}" == true ]] && domain="((^)|(\\.))${domain//\\./\\\.}$" bool=true # Is it in the list? From 8084c4b2a346762fbcf5b249184ff417550e6ad5 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sun, 8 Jul 2018 15:56:15 -0400 Subject: [PATCH 18/22] Fix regex period substitution due to switching to bashism Signed-off-by: Mcat12 --- advanced/Scripts/list.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index ba710445..d9036f42 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -138,7 +138,7 @@ AddDomain() { bool=true domain="${1}" - [[ "${wildcard}" == true ]] && domain="((^)|(\\.))${domain//\\./\\\.}$" + [[ "${wildcard}" == true ]] && domain="((^)|(\\.))${domain//\./\\.}$" # Is the domain in the list? # Search only for exactly matching lines @@ -186,7 +186,7 @@ RemoveDomain() { [[ -z "${type}" ]] && type="--wildcard-only" domain="${1}" - [[ "${wildcard}" == true ]] && domain="((^)|(\\.))${domain//\\./\\\.}$" + [[ "${wildcard}" == true ]] && domain="((^)|(\\.))${domain//\./\\.}$" bool=true # Is it in the list? From 1b276a74fe81beca8bcee4d896c27cb9db65f005 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sun, 8 Jul 2018 16:03:16 -0400 Subject: [PATCH 19/22] Display all lines of a list, even if there is no newline at the end Signed-off-by: Mcat12 --- advanced/Scripts/list.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index d9036f42..0088e0c1 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -222,7 +222,7 @@ Displaylist() { verbose=false echo -e "Displaying $string:\n" count=1 - while IFS= read -r RD; do + while IFS= read -r RD || [ -n "${RD}" ]; do echo " ${count}: ${RD}" count=$((count+1)) done < "${listMain}" From b2a36f71a0531d4ebffe3cd2d3c10df53d244d92 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Thu, 12 Jul 2018 21:17:32 -0400 Subject: [PATCH 20/22] Fix wildcard description in pihole man page Signed-off-by: Mcat12 --- manpages/pihole.8 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 68cd2864..1017aaea 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -282,8 +282,8 @@ Some usage examples .br \fBpihole -b -d noads.example.com\fR Remove "noads.example.com" from blacklist .br - \fBpihole -wild "^example.*$"\fR Add "^example.*$" as a regex pattern - would - block all domains starting with "example" + \fBpihole -wild example.com\fR Add example.com as a wildcard - would + block all subdomains of example.com, including example.com itself. .br \fBpihole -regex "ad.*\.example\.com$"\fR Add "ad.*\.example\.com$" to the regex blacklist - would block all subdomains of example.com which start with "ad" From 72be7ad5f0fde78a0bde92ad1d98502ffa6c9afd Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Thu, 12 Jul 2018 21:21:06 -0400 Subject: [PATCH 21/22] Don't count commented out regex lines Signed-off-by: Mcat12 --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 9a5a5fd7..e5795cb6 100755 --- a/gravity.sh +++ b/gravity.sh @@ -465,7 +465,7 @@ gravity_ShowBlockCount() { fi if [[ -f "${regexFile}" ]]; then - num=$(grep -c "^" "${regexFile}") + num=$(grep -c "^(?!#)" "${regexFile}") echo -e " ${INFO} Number of regex filters: ${num}" fi } From 941a766aa3a3c2311df9a8676d556ea5df3ba1ed Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Thu, 12 Jul 2018 21:38:43 -0400 Subject: [PATCH 22/22] Replace -wild and -regex with --wild and --regex Signed-off-by: Mcat12 --- advanced/Scripts/list.sh | 4 ++-- manpages/pihole.8 | 10 +++++----- pihole | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 0088e0c1..97de1604 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -245,8 +245,8 @@ for var in "$@"; do case "${var}" in "-w" | "whitelist" ) listMain="${whitelist}"; listAlt="${blacklist}";; "-b" | "blacklist" ) listMain="${blacklist}"; listAlt="${whitelist}";; - "-wild" | "wildcard" ) listMain="${regexlist}"; wildcard=true;; - "-regex" | "regex" ) listMain="${regexlist}";; + "--wild" | "wildcard" ) listMain="${regexlist}"; wildcard=true;; + "--regex" | "regex" ) listMain="${regexlist}";; "-nr"| "--noreload" ) reload=false;; "-d" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 1017aaea..c30570df 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -5,7 +5,7 @@ Pi-hole : A black-hole for internet advertisements .br .SH "SYNOPSIS" -\fBpihole\fR (\fB-w\fR|\fB-b\fR|\fB-wild\fR|\fB-regex\fR) [options] domain(s) +\fBpihole\fR (\fB-w\fR|\fB-b\fR|\fB--wild\fR|\fB--regex\fR) [options] domain(s) .br \fBpihole -a\fR \fB-p\fR password .br @@ -66,12 +66,12 @@ Available commands and options: Adds or removes specified domain or domains to the blacklist .br -\fB-wild, wildcard\fR [options] [ ] +\fB--wild, wildcard\fR [options] [ ] .br Add or removes specified domain to the wildcard blacklist .br -\fB-regex, regex\fR [options] [ ] +\fB--regex, regex\fR [options] [ ] .br Add or removes specified regex filter to the regex blacklist .br @@ -282,10 +282,10 @@ Some usage examples .br \fBpihole -b -d noads.example.com\fR Remove "noads.example.com" from blacklist .br - \fBpihole -wild example.com\fR Add example.com as a wildcard - would + \fBpihole --wild example.com\fR Add example.com as a wildcard - would block all subdomains of example.com, including example.com itself. .br - \fBpihole -regex "ad.*\.example\.com$"\fR Add "ad.*\.example\.com$" to the regex + \fBpihole --regex "ad.*\.example\.com$"\fR Add "ad.*\.example\.com$" to the regex blacklist - would block all subdomains of example.com which start with "ad" .br diff --git a/pihole b/pihole index ce8ea82b..fea9b339 100755 --- a/pihole +++ b/pihole @@ -393,8 +393,8 @@ Add '-h' after specific commands for more information on usage Whitelist/Blacklist Options: -w, whitelist Whitelist domain(s) -b, blacklist Blacklist domain(s) - -wild, wildcard Wildcard blacklist domain(s) - -regex, regex Regex blacklist domains(s) + --wild, wildcard Wildcard blacklist domain(s) + --regex, regex Regex blacklist domains(s) Add '-h' for more info on whitelist/blacklist usage Debugging Options: @@ -438,8 +438,8 @@ fi case "${1}" in "-w" | "whitelist" ) listFunc "$@";; "-b" | "blacklist" ) listFunc "$@";; - "-wild" | "wildcard" ) listFunc "$@";; - "-regex" | "regex" ) listFunc "$@";; + "--wild" | "wildcard" ) listFunc "$@";; + "--regex" | "regex" ) listFunc "$@";; "-d" | "debug" ) debugFunc "$@";; "-f" | "flush" ) flushFunc "$@";; "-up" | "updatePihole" ) updatePiholeFunc "$@";;