Merge pull request #2832 from pi-hole/new/audit_database

Migrate audit list to gravity.db database table

advanced/Scripts/database_migration/gravity-db.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1090
+
+# Pi-hole: A black hole for Internet advertisements
+# (c) 2019 Pi-hole, LLC (https://pi-hole.net)
+# Network-wide ad blocking via your own hardware.
+#
+# Updates gravity.db database
+#
+# This file is copyright under the latest version of the EUPL.
+# Please see LICENSE file for your rights under this license.
+
+upgrade_gravityDB(){
+	local database piholeDir auditFile version
+	database="${1}"
+	piholeDir="${2}"
+	auditFile="${piholeDir}/auditlog.list"
+
+	# Get database version
+	version="$(sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")"
+
+	if [[ "$version" == "1" ]]; then
+		# This migration script upgrades the gravity.db file by
+		# adding the domain_audit table
+		sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql"
+		version=2
+
+		# Store audit domains in database table
+		if [ -e "${auditFile}" ]; then
+			echo -e "  ${INFO} Migrating content of ${auditFile} into new database"
+			# database_table_from_file is defined in gravity.sh
+			database_table_from_file "domain_audit" "${auditFile}"
+		fi
+	fi
+}

advanced/Scripts/database_migration/gravity/1_to_2.sql

@@ -0,0 +1,10 @@
+.timeout 30000
+
+CREATE TABLE domain_audit
+(
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	domain TEXT UNIQUE NOT NULL,
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int))
+);
+
+UPDATE info SET value = 2 WHERE property = 'version';

advanced/Scripts/webpage.sh

@@ -542,21 +542,45 @@ Teleporter() {
     php /var/www/html/admin/scripts/pi-hole/php/teleporter.php > "pi-hole-teleporter_${datetimestamp}.tar.gz"
 }
 
+checkDomain()
+{
+    local domain validDomain
+    # Convert to lowercase
+    domain="${1,,}"
+    validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check
+    validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label
+    echo "${validDomain}"
+}
+
 addAudit()
 {
     shift # skip "-a"
     shift # skip "audit"
-    for var in "$@"
+    local domains validDomain
+    domains=""
+    for domain in "$@"
     do
-        echo "${var}" >> /etc/pihole/auditlog.list
+      # Check domain to be added. Only continue if it is valid
+      validDomain="$(checkDomain "${domain}")"
+      if [[ -n "${validDomain}" ]]; then
+        # Put comma in between domains when there is
+        # more than one domains to be added
+        # SQL INSERT allows adding multiple rows at once using the format
+        ## INSERT INTO table (domain) VALUES ('abc.de'),('fgh.ij'),('klm.no'),('pqr.st');
+        if [[ -n "${domains}" ]]; then
+          domains="${domains},"
+        fi
+        domains="${domains}('${domain}')"
+      fi
     done
-    chmod 644 /etc/pihole/auditlog.list
+    # Insert only the domain here. The date_added field will be
+    # filled with its default value (date_added = current timestamp)
+    sqlite3 "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};"
 }
 
 clearAudit()
 {
-    echo -n "" > /etc/pihole/auditlog.list
+    sqlite3 "${gravityDBfile}" "DELETE FROM domain_audit;"
-    chmod 644 /etc/pihole/auditlog.list
 }
 
 SetPrivacyLevel() {

advanced/Templates/gravity.db.sql

@@ -80,6 +80,7 @@ CREATE TABLE gravity
 (
 	domain TEXT PRIMARY KEY
 );
+
 CREATE TABLE info
 (
 	property TEXT PRIMARY KEY,

gravity.sh

@@ -17,6 +17,8 @@ coltable="/opt/pihole/COL_TABLE"
 source "${coltable}"
 regexconverter="/opt/pihole/wildcard_regex_converter.sh"
 source "${regexconverter}"
+# shellcheck disable=SC1091
+source "/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh"
 
 basename="pihole"
 PIHOLE_COMMAND="/usr/local/bin/${basename}"
@@ -112,15 +114,21 @@ database_table_from_file() {
     inputfile="${source}"
   else
     # Apply format for white-, blacklist, regex, and adlist tables
+    # Read file line by line
     local rowid
     declare -i rowid
     rowid=1
-    # Read file line by line
     grep -v '^ *#' < "${source}" | while IFS= read -r domain
     do
       # Only add non-empty lines
-      if [[ ! -z "${domain}" ]]; then
+      if [[ -n "${domain}" ]]; then
-        echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}"
+        if [[ "${table}" == "domain_audit" ]]; then
+          # domain_audit table format (no enable or modified fields)
+          echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}"
+        else
+          # White-, black-, and regexlist format
+          echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}"
+        fi
         rowid+=1
       fi
     done
@@ -150,34 +158,36 @@ database_table_from_file() {
 # Migrate pre-v5.0 list files to database-based Pi-hole versions
 migrate_to_database() {
   # Create database file only if not present
-  if [ -e "${gravityDBfile}" ]; then
+  if [ ! -e "${gravityDBfile}" ]; then
-    return 0
+    # Create new database file - note that this will be created in version 1
+    echo -e "  ${INFO} Creating new gravity database"
+    generate_gravity_database
+
+    # Migrate list files to new database
+    if [ -e "${adListFile}" ]; then
+      # Store adlist domains in database
+      echo -e "  ${INFO} Migrating content of ${adListFile} into new database"
+      database_table_from_file "adlist" "${adListFile}"
+    fi
+    if [ -e "${blacklistFile}" ]; then
+      # Store blacklisted domains in database
+      echo -e "  ${INFO} Migrating content of ${blacklistFile} into new database"
+      database_table_from_file "blacklist" "${blacklistFile}"
+    fi
+    if [ -e "${whitelistFile}" ]; then
+      # Store whitelisted domains in database
+      echo -e "  ${INFO} Migrating content of ${whitelistFile} into new database"
+      database_table_from_file "whitelist" "${whitelistFile}"
+    fi
+    if [ -e "${regexFile}" ]; then
+      # Store regex domains in database
+      echo -e "  ${INFO} Migrating content of ${regexFile} into new database"
+      database_table_from_file "regex" "${regexFile}"
+    fi
   fi
 
-  echo -e "  ${INFO} Creating new gravity database"
+  # Check if gravity database needs to be updated
-  generate_gravity_database
+  upgrade_gravityDB "${gravityDBfile}" "${piholeDir}"
-
-  # Migrate list files to new database
-  if [[ -e "${adListFile}" ]]; then
-    # Store adlist domains in database
-    echo -e "  ${INFO} Migrating content of ${adListFile} into new database"
-    database_table_from_file "adlist" "${adListFile}"
-  fi
-  if [[ -e "${blacklistFile}" ]]; then
-    # Store blacklisted domains in database
-    echo -e "  ${INFO} Migrating content of ${blacklistFile} into new database"
-    database_table_from_file "blacklist" "${blacklistFile}"
-  fi
-  if [[ -e "${whitelistFile}" ]]; then
-    # Store whitelisted domains in database
-    echo -e "  ${INFO} Migrating content of ${whitelistFile} into new database"
-    database_table_from_file "whitelist" "${whitelistFile}"
-  fi
-  if [[ -e "${regexFile}" ]]; then
-    # Store regex domains in database
-    echo -e "  ${INFO} Migrating content of ${regexFile} into new database"
-    database_table_from_file "regex" "${regexFile}"
-  fi
 }
 
 # Determine if DNS resolution is available before proceeding