Merge pull request #2838 from pi-hole/new/whitelist-regex-support

Whitelist regex support

advanced/Scripts/database_migration/gravity-db.sh

@@ -32,4 +32,11 @@ upgrade_gravityDB(){
 			database_table_from_file "domain_audit" "${auditFile}"
 		fi
 	fi
+	if [[ "$version" == "2" ]]; then
+		# This migration script upgrades the gravity.db file by
+		# renaming the regex table to regex_blacklist, and
+		# creating a new regex_whitelist table + corresponding linking table and views
+		sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/2_to_3.sql"
+		version=3
+	fi
 }

advanced/Scripts/database_migration/gravity/1_to_2.sql

@@ -1,5 +1,7 @@
 .timeout 30000
 
+BEGIN TRANSACTION;
+
 CREATE TABLE domain_audit
 (
 	id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -8,3 +10,5 @@ CREATE TABLE domain_audit
 );
 
 UPDATE info SET value = 2 WHERE property = 'version';
+
+COMMIT;

advanced/Scripts/database_migration/gravity/2_to_3.sql

@@ -0,0 +1,65 @@
+.timeout 30000
+
+PRAGMA FOREIGN_KEYS=OFF;
+
+BEGIN TRANSACTION;
+
+ALTER TABLE regex RENAME TO regex_blacklist;
+
+CREATE TABLE regex_blacklist_by_group
+(
+	regex_blacklist_id INTEGER NOT NULL REFERENCES regex_blacklist (id),
+	group_id INTEGER NOT NULL REFERENCES "group" (id),
+	PRIMARY KEY (regex_blacklist_id, group_id)
+);
+
+INSERT INTO regex_blacklist_by_group SELECT * FROM regex_by_group;
+DROP TABLE regex_by_group;
+DROP VIEW vw_regex;
+DROP TRIGGER tr_regex_update;
+
+CREATE VIEW vw_regex_blacklist AS SELECT DISTINCT domain
+    FROM regex_blacklist
+    LEFT JOIN regex_blacklist_by_group ON regex_blacklist_by_group.regex_blacklist_id = regex_blacklist.id
+    LEFT JOIN "group" ON "group".id = regex_blacklist_by_group.group_id
+    WHERE regex_blacklist.enabled = 1 AND (regex_blacklist_by_group.group_id IS NULL OR "group".enabled = 1)
+    ORDER BY regex_blacklist.id;
+
+CREATE TRIGGER tr_regex_blacklist_update AFTER UPDATE ON regex_blacklist
+    BEGIN
+      UPDATE regex_blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
+    END;
+
+CREATE TABLE regex_whitelist
+(
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	domain TEXT UNIQUE NOT NULL,
+	enabled BOOLEAN NOT NULL DEFAULT 1,
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	comment TEXT
+);
+
+CREATE TABLE regex_whitelist_by_group
+(
+	regex_whitelist_id INTEGER NOT NULL REFERENCES regex_whitelist (id),
+	group_id INTEGER NOT NULL REFERENCES "group" (id),
+	PRIMARY KEY (regex_whitelist_id, group_id)
+);
+
+CREATE VIEW vw_regex_whitelist AS SELECT DISTINCT domain
+    FROM regex_whitelist
+    LEFT JOIN regex_whitelist_by_group ON regex_whitelist_by_group.regex_whitelist_id = regex_whitelist.id
+    LEFT JOIN "group" ON "group".id = regex_whitelist_by_group.group_id
+    WHERE regex_whitelist.enabled = 1 AND (regex_whitelist_by_group.group_id IS NULL OR "group".enabled = 1)
+    ORDER BY regex_whitelist.id;
+
+CREATE TRIGGER tr_regex_whitelist_update AFTER UPDATE ON regex_whitelist
+    BEGIN
+      UPDATE regex_whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
+    END;
+
+
+UPDATE info SET value = 3 WHERE property = 'version';
+
+COMMIT;

advanced/Scripts/list.sh

@@ -32,12 +32,18 @@ helpFunc() {
     if [[ "${listType}" == "whitelist" ]]; then
         param="w"
         type="whitelist"
-    elif [[ "${listType}" == "regex" && "${wildcard}" == true ]]; then
+    elif [[ "${listType}" == "regex_blacklist" && "${wildcard}" == true ]]; then
         param="-wild"
         type="wildcard blacklist"
-    elif [[ "${listType}" == "regex" ]]; then
+    elif [[ "${listType}" == "regex_blacklist" ]]; then
         param="-regex"
-        type="regex filter"
+        type="regex blacklist filter"
+    elif [[ "${listType}" == "regex_whitelist" && "${wildcard}" == true ]]; then
+        param="-white-wild"
+        type="wildcard whitelist"
+    elif [[ "${listType}" == "regex_whitelist" ]]; then
+        param="-white-regex"
+        type="regex whitelist filter"
     else
         param="b"
         type="blacklist"
@@ -72,7 +78,7 @@ HandleOther() {
 
     # Check validity of domain (don't check for regex entries)
     if [[ "${#domain}" -le 253 ]]; then
-        if [[ "${listType}" == "regex" && "${wildcard}" == false ]]; then
+        if [[ ( "${listType}" == "regex_blacklist" || "${listType}" == "regex_whitelist" ) && "${wildcard}" == false ]]; then
             validDomain="${domain}"
         else
             validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check
@@ -88,12 +94,19 @@ HandleOther() {
 }
 
 ProcessDomainList() {
-    if [[ "${listType}" == "regex" ]]; then
-        # Regex filter list
-        listname="regex filters"
+    local is_regexlist
+    if [[ "${listType}" == "regex_blacklist" ]]; then
+        # Regex black filter list
+        listname="regex blacklist filters"
+        is_regexlist=true
+    elif [[ "${listType}" == "regex_whitelist" ]]; then
+        # Regex white filter list
+        listname="regex whitelist filters"
+        is_regexlist=true
     else
         # Whitelist / Blacklist
         listname="${listType}"
+        is_regexlist=false
     fi
 
     for dom in "${domList[@]}"; do
@@ -106,7 +119,7 @@ ProcessDomainList() {
         # if delmode then remove from desired list but do not add to the other
         if ${addmode}; then
             AddDomain "${dom}" "${listType}"
-            if [[ ! "${listType}" == "regex" ]]; then
+            if ! ${is_regexlist}; then
                 RemoveDomain "${dom}" "${listAlt}"
             fi
         else
@@ -173,7 +186,7 @@ Displaylist() {
     data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM ${listType};" 2> /dev/null)"
 
     if [[ -z $data ]]; then
-        echo -e "Not showing empty ${listname}"
+        echo -e "Not showing empty list"
     else
         echo -e "Displaying ${listname}:"
         count=1
@@ -215,8 +228,10 @@ for var in "$@"; do
     case "${var}" in
         "-w" | "whitelist"   ) listType="whitelist"; listAlt="blacklist";;
         "-b" | "blacklist"   ) listType="blacklist"; listAlt="whitelist";;
-        "--wild" | "wildcard" ) listType="regex"; wildcard=true;;
-        "--regex" | "regex"   ) listType="regex";;
+        "--wild" | "wildcard" ) listType="regex_blacklist"; wildcard=true;;
+        "--regex" | "regex"   ) listType="regex_blacklist";;
+        "--white-regex" | "white-regex" ) listType="regex_whitelist";;
+        "--white-wild" | "white-wild" ) listType="regex_whitelist"; wildcard=true;;
         "-nr"| "--noreload"  ) reload=false;;
         "-d" | "--delmode"   ) addmode=false;;
         "-q" | "--quiet"     ) verbose=false;;

advanced/Scripts/piholeDebug.sh

@@ -1104,20 +1104,27 @@ show_db_entries() {
     IFS="$OLD_IFS"
 }
 
+show_groups() {
+    show_db_entries "Groups" "SELECT * FROM \"group\"" "4 4 30 50"
+}
+
 show_adlists() {
-    show_db_entries "Adlists" "SELECT * FROM adlist" "4 100 7 10 13 50"
+    show_db_entries "Adlists" "SELECT id,address,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM adlist" "4 100 7 19 19 50"
+    show_db_entries "Adlist groups" "SELECT * FROM adlist_by_group" "4 4"
 }
 
 show_whitelist() {
-    show_db_entries "Whitelist" "SELECT * FROM whitelist" "4 100 7 10 13 50"
+    show_db_entries "Exact whitelist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM whitelist" "4 100 7 19 19 50"
+    show_db_entries "Exact whitelist groups" "SELECT * FROM whitelist_by_group" "4 4"
+    show_db_entries "Regex whitelist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM regex_whitelist" "4 100 7 19 19 50"
+    show_db_entries "Regex whitelist groups" "SELECT * FROM regex_whitelist_by_group" "4 4"
 }
 
 show_blacklist() {
-    show_db_entries "Blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50"
-}
-
-show_regexlist() {
-    show_db_entries "Regexlist" "SELECT * FROM regex" "4 100 7 10 13 50"
+    show_db_entries "Exact blacklist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM blacklist" "4 100 7 19 19 50"
+    show_db_entries "Exact blacklist groups" "SELECT * FROM blacklist_by_group" "4 4"
+    show_db_entries "Regex blacklist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM regex_blacklist" "4 100 7 19 19 50"
+    show_db_entries "Regex blacklist groups" "SELECT * FROM regex_blacklist_by_group" "4 4"
 }
 
 analyze_gravity_list() {
@@ -1293,10 +1300,10 @@ process_status
 parse_setup_vars
 check_x_headers
 analyze_gravity_list
+show_groups
 show_adlists
 show_whitelist
 show_blacklist
-show_regexlist
 show_content_of_pihole_files
 parse_locale
 analyze_pihole_log

advanced/Scripts/query.sh

@@ -36,14 +36,14 @@ scanList(){
     # /dev/null forces filename to be printed when only one list has been generated
     # shellcheck disable=SC2086
     case "${type}" in
-		"exact" ) grep -i -E -l "(^|(?<!#)\\s)${esc_domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
-		# Create array of regexps
-		# Iterate through each regexp and check whether it matches the domainQuery
-		# If it does, print the matching regexp and continue looping
-		# Input 1 - regexps | Input 2 - domainQuery
-		"regex" ) awk 'NR==FNR{regexps[$0];next}{for (r in regexps)if($0 ~ r)print r}' \
-					<(echo "${lists}") <(echo "${domain}") 2>/dev/null;;
-		*       ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;;
+        "exact" ) grep -i -E -l "(^|(?<!#)\\s)${esc_domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
+        # Create array of regexps
+        # Iterate through each regexp and check whether it matches the domainQuery
+        # If it does, print the matching regexp and continue looping
+        # Input 1 - regexps | Input 2 - domainQuery
+        "regex" ) awk 'NR==FNR{regexps[$0];next}{for (r in regexps)if($0 ~ r)print r}' \
+                  <(echo "${lists}") <(echo "${domain}") 2>/dev/null;;
+        *       ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;;
     esac
 }
 
@@ -100,8 +100,8 @@ scanDatabaseTable() {
     # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched
     # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores.
     case "${type}" in
-		"exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";;
-		*       ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
+        "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";;
+        *       ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
     esac
 
     # Send prepared query to gravity database
@@ -130,42 +130,52 @@ scanDatabaseTable() {
     done
 }
 
+scanRegexDatabaseTable() {
+    local domain list
+    domain="${1}"
+    list="${2}"
+
+    # Query all regex from the corresponding database tables
+    mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex_${list}" 2> /dev/null)
+
+    # If we have regexps to process
+    if [[ "${#regexList[@]}" -ne 0 ]]; then
+        # Split regexps over a new line
+        str_regexList=$(printf '%s\n' "${regexList[@]}")
+        # Check domain against regexps
+        mapfile -t regexMatches < <(scanList "${domain}" "${str_regexList}" "regex")
+        # If there were regex matches
+        if [[  "${#regexMatches[@]}" -ne 0 ]]; then
+            # Split matching regexps over a new line
+            str_regexMatches=$(printf '%s\n' "${regexMatches[@]}")
+            # Form a "matched" message
+            str_message="${matchType^} found in ${COL_BOLD}Regex ${list}${COL_NC}"
+            # Form a "results" message
+            str_result="${COL_BOLD}${str_regexMatches}${COL_NC}"
+            # If we are displaying more than just the source of the block
+            if [[ -z "${blockpage}" ]]; then
+                # Set the wildcard match flag
+                wcMatch=true
+                # Echo the "matched" message, indented by one space
+                echo " ${str_message}"
+                # Echo the "results" message, each line indented by three spaces
+                # shellcheck disable=SC2001
+                echo "${str_result}" | sed 's/^/   /'
+            else
+                echo "π .wildcard"
+                exit 0
+            fi
+        fi
+    fi
+}
+
 # Scan Whitelist and Blacklist
 scanDatabaseTable "${domainQuery}" "whitelist" "${exact}"
 scanDatabaseTable "${domainQuery}" "blacklist" "${exact}"
 
 # Scan Regex table
-mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex" 2> /dev/null)
-
-# If we have regexps to process
-if [[ "${#regexList[@]}" -ne 0 ]]; then
-	# Split regexps over a new line
-	str_regexList=$(printf '%s\n' "${regexList[@]}")
-	# Check domainQuery against regexps
-	mapfile -t regexMatches < <(scanList "${domainQuery}" "${str_regexList}" "regex")
-	# If there were regex matches
-	if [[  "${#regexMatches[@]}" -ne 0 ]]; then
-		# Split matching regexps over a new line
-		str_regexMatches=$(printf '%s\n' "${regexMatches[@]}")
-		# Form a "matched" message
-		str_message="${matchType^} found in ${COL_BOLD}Regex list${COL_NC}"
-		# Form a "results" message
-		str_result="${COL_BOLD}${str_regexMatches}${COL_NC}"
-		# If we are displaying more than just the source of the block
-		if [[ -z "${blockpage}" ]]; then
-			# Set the wildcard match flag
-			wcMatch=true
-			# Echo the "matched" message, indented by one space
-			echo " ${str_message}"
-			# Echo the "results" message, each line indented by three spaces
-			# shellcheck disable=SC2001
-			echo "${str_result}" | sed 's/^/   /'
-		else
-			echo "π .wildcard"
-			exit 0
-		fi
-	fi
-fi
+scanRegexDatabaseTable "${domainQuery}" "whitelist"
+scanRegexDatabaseTable "${domainQuery}" "blacklist"
 
 # Get version sorted *.domains filenames (without dir path)
 lists=("$(cd "$piholeDir" || exit 0; printf "%s\\n" -- *.domains | sort -V)")

automated install/basic-install.sh

@@ -1910,6 +1910,9 @@ installPihole() {
             chmod a+rx /var/www/html
             # Give pihole access to the Web server group
             usermod -a -G ${LIGHTTPD_GROUP} pihole
+            # Give lighttpd access to the pihole group so the web interface can
+            # manage the gravity.db database
+            usermod -a -G pihole ${LIGHTTPD_USER}
             # If the lighttpd command is executable,
             if is_command lighty-enable-mod ; then
                 # enable fastcgi and fastcgi-php

gravity.sh

@@ -84,6 +84,10 @@ fi
 # Generate new sqlite3 file from schema template
 generate_gravity_database() {
   sqlite3 "${gravityDBfile}" < "${gravityDBschema}"
+
+  # Ensure proper permissions are set for the newly created database
+  chown pihole:pihole "${gravityDBfile}"
+  chmod g+w "${piholeDir}" "${gravityDBfile}"
 }
 
 # Import domains from file and store them in the specified database table
@@ -181,6 +185,8 @@ migrate_to_database() {
     fi
     if [ -e "${regexFile}" ]; then
       # Store regex domains in database
+      # Important note: We need to add the domains to the "regex" table
+      # as it will only later be renamed to "regex_blacklist"!
       echo -e "  ${INFO} Migrating content of ${regexFile} into new database"
       database_table_from_file "regex" "${regexFile}"
     fi
@@ -590,9 +596,10 @@ gravity_Table_Count() {
 
 # Output count of blacklisted domains and regex filters
 gravity_ShowCount() {
-  gravity_Table_Count "blacklist" "blacklisted domains"
-  gravity_Table_Count "whitelist" "whitelisted domains"
-  gravity_Table_Count "regex" "regex filters"
+  gravity_Table_Count "blacklist" "exact blacklisted domains"
+  gravity_Table_Count "regex_blacklist" "regex blacklist filters"
+  gravity_Table_Count "whitelist" "exact whitelisted domains"
+  gravity_Table_Count "regex_whitelist" "regex whitelist filters"
 }
 
 # Parse list of domains into hosts format

manpages/pihole.8

@@ -66,14 +66,24 @@ Available commands and options:
     Adds or removes specified domain or domains to the blacklist
 .br
 
+\fB--regex, regex\fR [options] [<regex1> <regex2 ...>]
+.br
+    Add or removes specified regex filter to the regex blacklist
+.br
+
+\fB--white-regex\fR [options] [<regex1> <regex2 ...>]
+.br
+    Add or removes specified regex filter to the regex whitelist
+.br
+
 \fB--wild, wildcard\fR [options] [<domain1> <domain2 ...>]
 .br
     Add or removes specified domain to the wildcard blacklist
 .br
 
-\fB--regex, regex\fR [options] [<regex1> <regex2 ...>]
+\fB--white-wild\fR [options] [<domain1> <domain2 ...>]
 .br
-    Add or removes specified regex filter to the regex blacklist
+    Add or removes specified domain to the wildcard whitelist
 .br
 
     (Whitelist/Blacklist manipulation options):

pihole

@@ -375,8 +375,10 @@ Add '-h' after specific commands for more information on usage
 Whitelist/Blacklist Options:
   -w, whitelist       Whitelist domain(s)
   -b, blacklist       Blacklist domain(s)
-  --wild, wildcard     Wildcard blacklist domain(s)
-  --regex, regex       Regex blacklist domains(s)
+  --regex, regex      Regex blacklist domains(s)
+  --white-regex       Regex whitelist domains(s)
+  --wild, wildcard    Wildcard blacklist domain(s)
+  --white-wild        Wildcard whitelist domain(s)
                         Add '-h' for more info on whitelist/blacklist usage
 
 Debugging Options:
@@ -438,6 +440,8 @@ case "${1}" in
   "-b" | "blacklist"            ) listFunc "$@";;
   "--wild" | "wildcard"         ) listFunc "$@";;
   "--regex" | "regex"           ) listFunc "$@";;
+  "--white-regex" | "white-regex" ) listFunc "$@";;
+  "--white-wild" | "white-wild"   ) listFunc "$@";;
   "-d" | "debug"                ) debugFunc "$@";;
   "-f" | "flush"                ) flushFunc "$@";;
   "-up" | "updatePihole"        ) updatePiholeFunc "$@";;