Merge pull request #2935 from pi-hole/new/internal-blocking

Per-client blocking changes

advanced/Scripts/database_migration/gravity-db.sh

@@ -10,7 +10,7 @@
 # This file is copyright under the latest version of the EUPL.
 # Please see LICENSE file for your rights under this license.
 
-scriptPath="/etc/.pihole/advanced/Scripts/database_migration/gravity"
+readonly scriptPath="/etc/.pihole/advanced/Scripts/database_migration/gravity"
 
 upgrade_gravityDB(){
 	local database piholeDir auditFile version
@@ -48,6 +48,20 @@ upgrade_gravityDB(){
 		# lists into a single table with a UNIQUE domain constraint
 		echo -e "  ${INFO} Upgrading gravity database from version 3 to 4"
 		sqlite3 "${database}" < "${scriptPath}/3_to_4.sql"
+		version=4
+	fi
+	if [[ "$version" == "4" ]]; then
+		# This migration script upgrades the gravity and list views
+		# implementing necessary changes for per-client blocking
+		echo -e "  ${INFO} Upgrading gravity database from version 4 to 5"
+		sqlite3 "${database}" < "${scriptPath}/4_to_5.sql"
+		version=5
+	fi
+	if [[ "$version" == "5" ]]; then
+		# This migration script upgrades the adlist view
+		# to return an ID used in gravity.sh
+		echo -e "  ${INFO} Upgrading gravity database from version 5 to 6"
+		sqlite3 "${database}" < "${scriptPath}/5_to_6.sql"
 		version=6
 	fi
 }

advanced/Scripts/database_migration/gravity/3_to_4.sql

@@ -91,6 +91,6 @@ CREATE VIEW vw_regex_blacklist AS SELECT domain, domainlist.id AS id, domainlist
     AND domainlist.type = 3
     ORDER BY domainlist.id;
 
-UPDATE info SET value = 6 WHERE property = 'version';
+UPDATE info SET value = 4 WHERE property = 'version';
 
 COMMIT;

advanced/Scripts/database_migration/gravity/4_to_5.sql

@@ -0,0 +1,38 @@
+.timeout 30000
+
+PRAGMA FOREIGN_KEYS=OFF;
+
+BEGIN TRANSACTION;
+
+DROP TABLE gravity;
+CREATE TABLE gravity
+(
+	domain TEXT NOT NULL,
+	adlist_id INTEGER NOT NULL REFERENCES adlist (id),
+	PRIMARY KEY(domain, adlist_id)
+);
+
+DROP VIEW vw_gravity;
+CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id
+    FROM gravity
+    LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = gravity.adlist_id
+    LEFT JOIN adlist ON adlist.id = gravity.adlist_id
+    LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
+    WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1);
+
+CREATE TABLE client
+(
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	ip TEXT NOL NULL UNIQUE
+);
+
+CREATE TABLE client_by_group
+(
+	client_id INTEGER NOT NULL REFERENCES client (id),
+	group_id INTEGER NOT NULL REFERENCES "group" (id),
+	PRIMARY KEY (client_id, group_id)
+);
+
+UPDATE info SET value = 5 WHERE property = 'version';
+
+COMMIT;

advanced/Scripts/database_migration/gravity/5_to_6.sql

@@ -0,0 +1,18 @@
+.timeout 30000
+
+PRAGMA FOREIGN_KEYS=OFF;
+
+BEGIN TRANSACTION;
+
+DROP VIEW vw_adlist;
+CREATE VIEW vw_adlist AS SELECT DISTINCT address, adlist.id AS id
+    FROM adlist
+    LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = adlist.id
+    LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
+    WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1)
+    ORDER BY adlist.id;
+
+UPDATE info SET value = 6 WHERE property = 'version';
+
+COMMIT;
+

gravity.sh

@@ -40,9 +40,6 @@ gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql"
 optimize_database=false
 
 domainsExtension="domains"
-matterAndLight="${basename}.0.matterandlight.txt"
-parsedMatter="${basename}.1.parsedmatter.txt"
-preEventHorizon="list.preEventHorizon"
 
 resolver="pihole-FTL"
 
@@ -92,30 +89,44 @@ generate_gravity_database() {
 
 update_gravity_timestamp() {
   # Update timestamp when the gravity table was last updated successfully
-  output=$( { sqlite3 "${gravityDBfile}" <<< "INSERT OR REPLACE INTO info (property,value) values (\"updated\",cast(strftime('%s', 'now') as int));"; } 2>&1 )
+  output=$( { sqlite3 "${gravityDBfile}" <<< "INSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%s', 'now') as int));"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
     echo -e "\\n  ${CROSS} Unable to update gravity timestamp in database ${gravityDBfile}\\n  ${output}"
+    return 1
   fi
+  return 0
 }
 
-# Import domains from file and store them in the specified database table
+database_truncate_table() {
-database_table_from_file() {
+  local table
-  # Define locals
-  local table source backup_path backup_file
   table="${1}"
-  source="${2}"
-  backup_path="${piholeDir}/migration_backup"
-  backup_file="${backup_path}/$(basename "${2}")"
 
-  # Truncate table
   output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
     echo -e "\\n  ${CROSS} Unable to truncate ${table} database ${gravityDBfile}\\n  ${output}"
     gravity_Cleanup "error"
+    return 1
+  fi
+  return 0
+}
+
+# Import domains from file and store them in the specified database table
+database_table_from_file() {
+  # Define locals
+  local table source backup_path backup_file arg
+  table="${1}"
+  source="${2}"
+  arg="${3}"
+  backup_path="${piholeDir}/migration_backup"
+  backup_file="${backup_path}/$(basename "${2}")"
+
+  # Truncate table only if not gravity (we add multiple times to this table)
+  if [[ "${table}" != "gravity" ]]; then
+    database_truncate_table "${table}"
   fi
 
   local tmpFile
@@ -123,15 +134,17 @@ database_table_from_file() {
   local timestamp
   timestamp="$(date --utc +'%s')"
   local inputfile
+  # Apply format for white-, blacklist, regex, and adlist tables
+  # Read file line by line
+  local rowid
+  declare -i rowid
+  rowid=1
+
   if [[ "${table}" == "gravity" ]]; then
-    # No need to modify the input data for the gravity table
+    #Append ,${arg} to every line and then remove blank lines before import
-    inputfile="${source}"
+    sed -e "s/$/,${arg}/" "${source}" > "${tmpFile}"
+    sed -i '/^$/d' "${tmpFile}"
   else
-    # Apply format for white-, blacklist, regex, and adlist tables
-    # Read file line by line
-    local rowid
-    declare -i rowid
-    rowid=1
     grep -v '^ *#' < "${source}" | while IFS= read -r domain
     do
       # Only add non-empty lines
@@ -146,8 +159,11 @@ database_table_from_file() {
         rowid+=1
       fi
     done
-    inputfile="${tmpFile}"
   fi
+
+
+  inputfile="${tmpFile}"
+
   # Store domains in database table specified by ${table}
   # Use printf as .mode and .import need to be on separate lines
   # see https://unix.stackexchange.com/a/445615/83260
@@ -216,7 +232,7 @@ gravity_CheckDNSResolutionAvailable() {
   fi
 
   # Determine if $lookupDomain is resolvable
-  if timeout 1 getent hosts "${lookupDomain}" &> /dev/null; then
+  if timeout 4 getent hosts "${lookupDomain}" &> /dev/null; then
     # Print confirmation of resolvability if it had previously failed
     if [[ -n "${secs:-}" ]]; then
       echo -e "${OVER}  ${TICK} DNS resolution is now available\\n"
@@ -230,7 +246,7 @@ gravity_CheckDNSResolutionAvailable() {
   # If the /etc/resolv.conf contains resolvers other than 127.0.0.1 then the local dnsmasq will not be queried and pi.hole is NXDOMAIN.
   # This means that even though name resolution is working, the getent hosts check fails and the holddown timer keeps ticking and eventualy fails
   # So we check the output of the last command and if it failed, attempt to use dig +short as a fallback
-  if timeout 1 dig +short "${lookupDomain}" &> /dev/null; then
+  if timeout 4 dig +short "${lookupDomain}" &> /dev/null; then
     if [[ -n "${secs:-}" ]]; then
       echo -e "${OVER}  ${TICK} DNS resolution is now available\\n"
     fi
@@ -263,12 +279,13 @@ gravity_CheckDNSResolutionAvailable() {
 }
 
 # Retrieve blocklist URLs and parse domains from adlist.list
-gravity_GetBlocklistUrls() {
+gravity_DownloadBlocklists() {
   echo -e "  ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..."
 
   # Retrieve source URLs from gravity database
   # We source only enabled adlists, sqlite3 stores boolean values as 0 (false) or 1 (true)
   mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)"
+  mapfile -t sourceIDs <<< "$(sqlite3 "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)"
 
   # Parse source domains from $sources
   mapfile -t sourceDomains <<< "$(
@@ -285,21 +302,23 @@ gravity_GetBlocklistUrls() {
 
   if [[ -n "${sources[*]}" ]] && [[ -n "${sourceDomains[*]}" ]]; then
     echo -e "${OVER}  ${TICK} ${str}"
-    return 0
   else
     echo -e "${OVER}  ${CROSS} ${str}"
     echo -e "  ${INFO} No source list found, or it is empty"
     echo ""
     return 1
   fi
-}
 
-# Define options for when retrieving blocklists
-gravity_SetDownloadOptions() {
   local url domain agent cmd_ext str
-
   echo ""
 
+  # Flush gravity table once before looping over sources
+  str="Flushing gravity table"
+  echo -ne "  ${INFO} ${str}..."
+  if database_truncate_table "gravity"; then
+    echo -e "${OVER}  ${TICK} ${str}"
+  fi
+
   # Loop through $sources and download each one
   for ((i = 0; i < "${#sources[@]}"; i++)); do
     url="${sources[$i]}"
@@ -319,7 +338,7 @@ gravity_SetDownloadOptions() {
     esac
 
     echo -e "  ${INFO} Target: ${domain} (${url##*/})"
-    gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}"
+    gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}" "${sourceIDs[$i]}"
     echo ""
   done
   gravity_Blackbody=true
@@ -327,7 +346,7 @@ gravity_SetDownloadOptions() {
 
 # Download specified URL and perform checks on HTTP status and file content
 gravity_DownloadBlocklistFromUrl() {
-  local url="${1}" cmd_ext="${2}" agent="${3}" heisenbergCompensator="" patternBuffer str httpCode success=""
+  local url="${1}" cmd_ext="${2}" agent="${3}" adlistID="${4}" heisenbergCompensator="" patternBuffer str httpCode success=""
 
   # Create temp file to store content on disk instead of RAM
   patternBuffer=$(mktemp -p "/tmp" --suffix=".phgpb")
@@ -408,11 +427,20 @@ gravity_DownloadBlocklistFromUrl() {
   # Determine if the blocklist was downloaded and saved correctly
   if [[ "${success}" == true ]]; then
     if [[ "${httpCode}" == "304" ]]; then
-      : # Do not attempt to re-parse file
+      # Add domains to database table
+      str="Adding adlist with ID ${adlistID} to database table"
+      echo -ne "  ${INFO} ${str}..."
+      database_table_from_file "gravity" "${saveLocation}" "${adlistID}"
+      echo -e "${OVER}  ${TICK} ${str}"
     # Check if $patternbuffer is a non-zero length file
     elif [[ -s "${patternBuffer}" ]]; then
       # Determine if blocklist is non-standard and parse as appropriate
       gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}"
+      # Add domains to database table
+      str="Adding adlist with ID ${adlistID} to database table"
+      echo -ne "  ${INFO} ${str}..."
+      database_table_from_file "gravity" "${saveLocation}" "${adlistID}"
+      echo -e "${OVER}  ${TICK} ${str}"
     else
       # Fall back to previously cached list if $patternBuffer is empty
       echo -e "  ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
@@ -421,6 +449,11 @@ gravity_DownloadBlocklistFromUrl() {
     # Determine if cached list has read permission
     if [[ -r "${saveLocation}" ]]; then
       echo -e "  ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
+      # Add domains to database table
+      str="Adding to database table"
+      echo -ne "  ${INFO} ${str}..."
+      database_table_from_file "gravity" "${saveLocation}" "${adlistID}"
+      echo -e "${OVER}  ${TICK} ${str}"
     else
       echo -e "  ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}"
     fi
@@ -429,10 +462,10 @@ gravity_DownloadBlocklistFromUrl() {
 
 # Parse source files into domains format
 gravity_ParseFileIntoDomains() {
-  local source="${1}" destination="${2}" firstLine abpFilter
+  local source="${1}" destination="${2}" firstLine
 
   # Determine if we are parsing a consolidated list
-  if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then
+  #if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then
     # Remove comments and print only the domain name
     # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious
     # This helps with that and makes it easier to read
@@ -449,7 +482,7 @@ gravity_ParseFileIntoDomains() {
     sed -r '/([^\.]+\.)+[^\.]{2,}/!d' >  "${destination}"
     chmod 644 "${destination}"
     return 0
-  fi
+  #fi
 
   # Individual file parsing: Keep comments, while parsing domains from each line
   # We keep comments to respect the list maintainer's licensing
@@ -495,80 +528,28 @@ gravity_ParseFileIntoDomains() {
   fi
 }
 
-# Create (unfiltered) "Matter and Light" consolidated list
-gravity_ConsolidateDownloadedBlocklists() {
-  local str lastLine
-
-  str="Consolidating blocklists"
-  echo -ne "  ${INFO} ${str}..."
-
-  # Empty $matterAndLight if it already exists, otherwise, create it
-  : > "${piholeDir}/${matterAndLight}"
-  chmod 644 "${piholeDir}/${matterAndLight}"
-
-  # Loop through each *.domains file
-  for i in "${activeDomains[@]}"; do
-    # Determine if file has read permissions, as download might have failed
-    if [[ -r "${i}" ]]; then
-      # Remove windows CRs from file, convert list to lower case, and append into $matterAndLight
-      tr -d '\r' < "${i}" | tr '[:upper:]' '[:lower:]' >> "${piholeDir}/${matterAndLight}"
-
-      # Ensure that the first line of a new list is on a new line
-      lastLine=$(tail -1 "${piholeDir}/${matterAndLight}")
-      if [[ "${#lastLine}" -gt 0 ]]; then
-        echo "" >> "${piholeDir}/${matterAndLight}"
-      fi
-    fi
-  done
-  echo -e "${OVER}  ${TICK} ${str}"
-
-}
-
-# Parse consolidated list into (filtered, unique) domains-only format
-gravity_SortAndFilterConsolidatedList() {
-  local str num
-
-  str="Extracting domains from blocklists"
-  echo -ne "  ${INFO} ${str}..."
-
-  # Parse into file
-  gravity_ParseFileIntoDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${parsedMatter}"
-
-  # Format $parsedMatter line total as currency
-  num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${parsedMatter}")")
-
-  echo -e "${OVER}  ${TICK} ${str}"
-  echo -e "  ${INFO} Gravity pulled in ${COL_BLUE}${num}${COL_NC} domains"
-
-  str="Removing duplicate domains"
-  echo -ne "  ${INFO} ${str}..."
-  sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}"
-  chmod 644 "${piholeDir}/${preEventHorizon}"
-  echo -e "${OVER}  ${TICK} ${str}"
-
-  # Format $preEventHorizon line total as currency
-  num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
-  str="Storing ${COL_BLUE}${num}${COL_NC} unique blocking domains in database"
-  echo -ne "  ${INFO} ${str}..."
-  database_table_from_file "gravity" "${piholeDir}/${preEventHorizon}"
-  echo -e "${OVER}  ${TICK} ${str}"
-}
-
 # Report number of entries in a table
 gravity_Table_Count() {
   local table="${1}"
   local str="${2}"
   local num
-  num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table} WHERE enabled = 1;")"
+  num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table};")"
-  echo -e "  ${INFO} Number of ${str}: ${num}"
+  if [[ "${table}" == "vw_gravity" ]]; then
+    local unique
+    unique="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(DISTINCT domain) FROM ${table};")"
+    echo -e "  ${INFO} Number of ${str}: ${num} (${unique} unique domains)"
+  else
+    echo -e "  ${INFO} Number of ${str}: ${num}"
+  fi
 }
 
 # Output count of blacklisted domains and regex filters
 gravity_ShowCount() {
-  gravity_Table_Count "blacklist" "exact blacklisted domains"
+  gravity_Table_Count "vw_gravity" "gravity domains" ""
-  gravity_Table_Count "regex_blacklist" "regex blacklist filters"
+  gravity_Table_Count "vw_blacklist" "exact blacklisted domains"
-  gravity_Table_Count "whitelist" "exact whitelisted domains"
+  gravity_Table_Count "vw_regex_blacklist" "regex blacklist filters"
-  gravity_Table_Count "regex_whitelist" "regex whitelist filters"
+  gravity_Table_Count "vw_whitelist" "exact whitelisted domains"
+  gravity_Table_Count "vw_regex_whitelist" "regex whitelist filters"
 }
 
 # Parse list of domains into hosts format
@@ -634,7 +615,7 @@ gravity_Cleanup() {
   # Ensure this function only runs when gravity_SetDownloadOptions() has completed
   if [[ "${gravity_Blackbody:-}" == true ]]; then
     # Remove any unused .domains files
-    for file in ${piholeDir}/*.${domainsExtension}; do
+    for file in "${piholeDir}"/*."${domainsExtension}"; do
       # If list is not in active array, then remove it
       if [[ ! "${activeDomains[*]}" == *"${file}"* ]]; then
         rm -f "${file}" 2> /dev/null || \
@@ -687,6 +668,7 @@ for var in "$@"; do
   case "${var}" in
     "-f" | "--force" ) forceDelete=true;;
     "-o" | "--optimize" ) optimize_database=true;;
+    "-r" | "--recreate" ) recreate_database=true;;
     "-h" | "--help" ) helpFunc;;
   esac
 done
@@ -694,6 +676,16 @@ done
 # Trap Ctrl-C
 gravity_Trap
 
+if [[ "${recreate_database:-}" == true ]]; then
+  str="Restoring from migration backup"
+  echo -ne "${INFO} ${str}..."
+  rm "${gravityDBfile}"
+  pushd "${piholeDir}" > /dev/null || exit
+  cp migration_backup/* .
+  popd > /dev/null || exit
+  echo -e "${OVER}  ${TICK} ${str}"
+fi
+
 # Move possibly existing legacy files to the gravity database
 migrate_to_database
 
@@ -707,12 +699,7 @@ fi
 
 # Gravity downloads blocklists next
 gravity_CheckDNSResolutionAvailable
-if gravity_GetBlocklistUrls; then
+gravity_DownloadBlocklists
-  gravity_SetDownloadOptions
-  # Build preEventHorizon
-  gravity_ConsolidateDownloadedBlocklists
-  gravity_SortAndFilterConsolidatedList
-fi
 
 # Create local.list
 gravity_generateLocalList