diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 1d96ea3c..97de1604 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -13,10 +13,11 @@ basename=pihole piholeDir=/etc/"${basename}" whitelist="${piholeDir}"/whitelist.txt blacklist="${piholeDir}"/blacklist.txt -readonly wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf" +readonly regexlist="/etc/pihole/regex.list" reload=false addmode=true verbose=true +wildcard=false domList=() @@ -31,9 +32,9 @@ helpFunc() { if [[ "${listMain}" == "${whitelist}" ]]; then param="w" type="white" - elif [[ "${listMain}" == "${wildcardlist}" ]]; then + elif [[ "${listMain}" == "${regexlist}" ]]; then param="wild" - type="wildcard black" + type="regex black" else param="b" type="black" @@ -57,7 +58,8 @@ Options: EscapeRegexp() { # This way we may safely insert an arbitrary # string in our regular expressions - # Also remove leading "." if present + # This sed is intentionally executed in three steps to ease maintainability + # The first sed removes any amount of leading dots echo $* | sed 's/^\.*//' | sed "s/[]\.|$(){}?+*^]/\\\\&/g" | sed "s/\\//\\\\\//g" } @@ -65,10 +67,14 @@ HandleOther() { # Convert to lowercase domain="${1,,}" - # Check validity of domain + # Check validity of domain (don't check for regex entries) if [[ "${#domain}" -le 253 ]]; then - validDomain=$(grep -P "^((-|_)*[a-z\d]((-|_)*[a-z\d])*(-|_)*)(\.(-|_)*([a-z\d]((-|_)*[a-z\d])*))*$" <<< "${domain}") # Valid chars check - validDomain=$(grep -P "^[^\.]{1,63}(\.[^\.]{1,63})*$" <<< "${validDomain}") # Length of each label + if [[ "${listMain}" == "${regexlist}" && "${wildcard}" == false ]]; then + validDomain="${domain}" + else + validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check + validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label + fi fi if [[ -n "${validDomain}" ]]; then @@ -94,9 +100,6 @@ PoplistFile() { if ${addmode}; then AddDomain "${dom}" "${listMain}" RemoveDomain "${dom}" "${listAlt}" - if [[ "${listMain}" == "${whitelist}" || "${listMain}" == "${blacklist}" ]]; then - RemoveDomain "${dom}" "${wildcardlist}" - fi else RemoveDomain "${dom}" "${listMain}" fi @@ -109,7 +112,6 @@ AddDomain() { [[ "${list}" == "${whitelist}" ]] && listname="whitelist" [[ "${list}" == "${blacklist}" ]] && listname="blacklist" - [[ "${list}" == "${wildcardlist}" ]] && listname="wildcard blacklist" if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only" @@ -121,7 +123,7 @@ AddDomain() { if [[ "${bool}" == false ]]; then # Domain not found in the whitelist file, add it! if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding $1 to $listname..." + echo -e " ${INFO} Adding ${1} to ${listname}..." fi reload=true # Add it to the list we want to add it to @@ -131,28 +133,26 @@ AddDomain() { echo -e " ${INFO} ${1} already exists in ${listname}, no need to add!" fi fi - elif [[ "${list}" == "${wildcardlist}" ]]; then - source "${piholeDir}/setupVars.conf" - # Remove the /* from the end of the IP addresses - IPV4_ADDRESS=${IPV4_ADDRESS%/*} - IPV6_ADDRESS=${IPV6_ADDRESS%/*} + elif [[ "${list}" == "${regexlist}" ]]; then [[ -z "${type}" ]] && type="--wildcard-only" bool=true + domain="${1}" + + [[ "${wildcard}" == true ]] && domain="((^)|(\\.))${domain//\./\\.}$" + # Is the domain in the list? - grep -e "address=\/${domain}\/" "${wildcardlist}" > /dev/null 2>&1 || bool=false + # Search only for exactly matching lines + grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false if [[ "${bool}" == false ]]; then if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding $1 to wildcard blacklist..." + echo -e " ${INFO} Adding ${domain} to regex list..." fi reload="restart" - echo "address=/$1/${IPV4_ADDRESS}" >> "${wildcardlist}" - if [[ "${#IPV6_ADDRESS}" > 0 ]]; then - echo "address=/$1/${IPV6_ADDRESS}" >> "${wildcardlist}" - fi + echo "$domain" >> "${regexlist}" else if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} already exists in wildcard blacklist, no need to add!" + echo -e " ${INFO} ${domain} already exists in regex list, no need to add!" fi fi fi @@ -164,7 +164,6 @@ RemoveDomain() { [[ "${list}" == "${whitelist}" ]] && listname="whitelist" [[ "${list}" == "${blacklist}" ]] && listname="blacklist" - [[ "${list}" == "${wildcardlist}" ]] && listname="wildcard blacklist" if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then bool=true @@ -174,7 +173,7 @@ RemoveDomain() { grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false if [[ "${bool}" == true ]]; then # Remove it from the other one - echo -e " ${INFO} Removing $1 from $listname..." + echo -e " ${INFO} Removing $1 from ${listname}..." # /I flag: search case-insensitive sed -i "/${domain}/Id" "${list}" reload=true @@ -183,20 +182,25 @@ RemoveDomain() { echo -e " ${INFO} ${1} does not exist in ${listname}, no need to remove!" fi fi - elif [[ "${list}" == "${wildcardlist}" ]]; then + elif [[ "${list}" == "${regexlist}" ]]; then [[ -z "${type}" ]] && type="--wildcard-only" + domain="${1}" + + [[ "${wildcard}" == true ]] && domain="((^)|(\\.))${domain//\./\\.}$" + bool=true # Is it in the list? - grep -e "address=\/${domain}\/" "${wildcardlist}" > /dev/null 2>&1 || bool=false + grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false if [[ "${bool}" == true ]]; then # Remove it from the other one - echo -e " ${INFO} Removing $1 from $listname..." - # /I flag: search case-insensitive - sed -i "/address=\/${domain}/Id" "${list}" + echo -e " ${INFO} Removing $domain from regex list..." + local lineNumber + lineNumber=$(grep -Fnx "$domain" "${list}" | cut -f1 -d:) + sed -i "${lineNumber}d" "${list}" reload=true else if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} does not exist in ${listname}, no need to remove!" + echo -e " ${INFO} ${domain} does not exist in regex list, no need to remove!" fi fi fi @@ -218,7 +222,7 @@ Displaylist() { verbose=false echo -e "Displaying $string:\n" count=1 - while IFS= read -r RD; do + while IFS= read -r RD || [ -n "${RD}" ]; do echo " ${count}: ${RD}" count=$((count+1)) done < "${listMain}" @@ -241,7 +245,8 @@ for var in "$@"; do case "${var}" in "-w" | "whitelist" ) listMain="${whitelist}"; listAlt="${blacklist}";; "-b" | "blacklist" ) listMain="${blacklist}"; listAlt="${whitelist}";; - "-wild" | "wildcard" ) listMain="${wildcardlist}";; + "--wild" | "wildcard" ) listMain="${regexlist}"; wildcard=true;; + "--regex" | "regex" ) listMain="${regexlist}";; "-nr"| "--noreload" ) reload=false;; "-d" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; diff --git a/advanced/Scripts/wildcard_regex_converter.sh b/advanced/Scripts/wildcard_regex_converter.sh new file mode 100644 index 00000000..d8ebeeb8 --- /dev/null +++ b/advanced/Scripts/wildcard_regex_converter.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Pi-hole: A black hole for Internet advertisements +# (c) 2017 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# Provides an automated migration subroutine to convert Pi-hole v3.x wildcard domains to Pi-hole v4.x regex filters +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + +# regexFile set in gravity.sh + +wildcardFile="/etc/dnsmasq.d/03-pihole-wildcard.conf" + +convert_wildcard_to_regex() { + if [ ! -f "${wildcardFile}" ]; then + return + fi + local addrlines domains uniquedomains + # Obtain wildcard domains from old file + addrlines="$(grep -oE "/.*/" ${wildcardFile})" + # Strip "/" from domain names and convert "." to regex-compatible "\." + domains="$(sed 's/\///g;s/\./\\./g' <<< "${addrlines}")" + # Remove repeated domains (may have been inserted two times due to A and AAAA blocking) + uniquedomains="$(uniq <<< "${domains}")" + # Automatically generate regex filters and remove old wildcards file + awk '{print "((^)|(\\.))"$0"$"}' <<< "${uniquedomains}" >> "${regexFile:?}" && rm "${wildcardFile}" +} diff --git a/gravity.sh b/gravity.sh index b7089ed1..b018e2ea 100755 --- a/gravity.sh +++ b/gravity.sh @@ -15,6 +15,8 @@ export LC_ALL=C coltable="/opt/pihole/COL_TABLE" source "${coltable}" +regexconverter="/opt/pihole/wildcard_regex_converter.sh" +source "${regexconverter}" basename="pihole" PIHOLE_COMMAND="/usr/local/bin/${basename}" @@ -26,7 +28,7 @@ adListDefault="${piholeDir}/adlists.default" whitelistFile="${piholeDir}/whitelist.txt" blacklistFile="${piholeDir}/blacklist.txt" -wildcardFile="/etc/dnsmasq.d/03-pihole-wildcard.conf" +regexFile="${piholeDir}/regex.list" adList="${piholeDir}/gravity.list" blackList="${piholeDir}/black.list" @@ -452,7 +454,7 @@ gravity_Whitelist() { echo -e "${OVER} ${INFO} ${str}" } -# Output count of blacklisted domains and wildcards +# Output count of blacklisted domains and regex filters gravity_ShowBlockCount() { local num @@ -461,13 +463,9 @@ gravity_ShowBlockCount() { echo -e " ${INFO} Number of blacklisted domains: ${num}" fi - if [[ -f "${wildcardFile}" ]]; then - num=$(grep -c "^" "${wildcardFile}") - # If IPv4 and IPv6 is used, divide total wildcard count by 2 - if [[ -n "${IPV4_ADDRESS}" ]] && [[ -n "${IPV6_ADDRESS}" ]];then - num=$(( num/2 )) - fi - echo -e " ${INFO} Number of wildcard blocked domains: ${num}" + if [[ -f "${regexFile}" ]]; then + num=$(grep -c "^(?!#)" "${regexFile}") + echo -e " ${INFO} Number of regex filters: ${num}" fi } @@ -645,6 +643,12 @@ if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "whitelist" ]]; then gravity_Whitelist fi +# Set proper permissions on the regex file +touch "${regexFile}" +chown pihole:www-data "${regexFile}" +chmod 664 "${regexFile}" + +convert_wildcard_to_regex gravity_ShowBlockCount # Perform when downloading blocklists, or modifying the white/blacklist (not wildcards) diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 54bf4a31..c30570df 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -5,7 +5,7 @@ Pi-hole : A black-hole for internet advertisements .br .SH "SYNOPSIS" -\fBpihole\fR (\fB-w\fR|\fB-b\fR|\fB-wild\fR) [options] domain(s) +\fBpihole\fR (\fB-w\fR|\fB-b\fR|\fB--wild\fR|\fB--regex\fR) [options] domain(s) .br \fBpihole -a\fR \fB-p\fR password .br @@ -66,9 +66,14 @@ Available commands and options: Adds or removes specified domain or domains to the blacklist .br -\fB-wild, wildcard\fR [options] [ ] +\fB--wild, wildcard\fR [options] [ ] .br - Add or removes specified domain, and all subdomains to the blacklist + Add or removes specified domain to the wildcard blacklist +.br + +\fB--regex, regex\fR [options] [ ] +.br + Add or removes specified regex filter to the regex blacklist .br (Whitelist/Blacklist manipulation options): @@ -167,9 +172,9 @@ Available commands and options: Show a help dialog .br -\fB-l, logging\fR [on|off|off noflush] +\fB-l, logging\fR [on|off|off noflush] .br - Specify whether the Pi-hole log should be used + Specify whether the Pi-hole log should be used .br (Logging options): @@ -193,7 +198,7 @@ Available commands and options: .br Show installed versions of Pi-hole, Web Interface & FTL .br - + .br (repo options): .br @@ -232,7 +237,7 @@ Available commands and options: Disable Pi-hole subsystems, optionally for a set duration .br - (time options): + (time options): .br #s Disable Pi-hole functionality for # second(s) .br @@ -273,11 +278,15 @@ Some usage examples Whitelist/blacklist manipulation .br - \fBpihole -w iloveads.example.com\fR Add "iloveads.example.com" to whitelist + \fBpihole -w iloveads.example.com\fR Add "iloveads.example.com" to whitelist +.br + \fBpihole -b -d noads.example.com\fR Remove "noads.example.com" from blacklist .br - \fBpihole -b -d noads.example.com\fR Remove "noads.example.com" from blacklist + \fBpihole --wild example.com\fR Add example.com as a wildcard - would + block all subdomains of example.com, including example.com itself. .br - \fBpihole -wild example.com\fR Add "example.com" as wildcard - would block ads.example.com, www.example.com etc. + \fBpihole --regex "ad.*\.example\.com$"\fR Add "ad.*\.example\.com$" to the regex + blacklist - would block all subdomains of example.com which start with "ad" .br Changing the Web Interface password diff --git a/pihole b/pihole index 0ee0c22f..8be03f79 100755 --- a/pihole +++ b/pihole @@ -33,17 +33,7 @@ webpageFunc() { exit 0 } -whitelistFunc() { - "${PI_HOLE_SCRIPT_DIR}"/list.sh "$@" - exit 0 -} - -blacklistFunc() { - "${PI_HOLE_SCRIPT_DIR}"/list.sh "$@" - exit 0 -} - -wildcardFunc() { +listFunc() { "${PI_HOLE_SCRIPT_DIR}"/list.sh "$@" exit 0 } @@ -386,7 +376,8 @@ Add '-h' after specific commands for more information on usage Whitelist/Blacklist Options: -w, whitelist Whitelist domain(s) -b, blacklist Blacklist domain(s) - -wild, wildcard Blacklist domain(s), and all its subdomains + --wild, wildcard Wildcard blacklist domain(s) + --regex, regex Regex blacklist domains(s) Add '-h' for more info on whitelist/blacklist usage Debugging Options: @@ -428,9 +419,10 @@ fi # Handle redirecting to specific functions based on arguments case "${1}" in - "-w" | "whitelist" ) whitelistFunc "$@";; - "-b" | "blacklist" ) blacklistFunc "$@";; - "-wild" | "wildcard" ) wildcardFunc "$@";; + "-w" | "whitelist" ) listFunc "$@";; + "-b" | "blacklist" ) listFunc "$@";; + "--wild" | "wildcard" ) listFunc "$@";; + "--regex" | "regex" ) listFunc "$@";; "-d" | "debug" ) debugFunc "$@";; "-f" | "flush" ) flushFunc "$@";; "-up" | "updatePihole" ) updatePiholeFunc "$@";;