From 1dbe6c83c38e64d2fb1026d65524a377ac990102 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 5 Jul 2019 13:54:18 +0200
Subject: [PATCH 01/14] Add database upgrading mechanism for adding the audit
 table.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 .../Scripts/database_migration/gravity-db.sh  | 21 +++++++++++++++++++
 .../database_migration/gravity/1_to_2.sql     |  9 ++++++++
 advanced/Templates/gravity.db.sql             | 11 +++++++++-
 3 files changed, 40 insertions(+), 1 deletion(-)
 create mode 100644 advanced/Scripts/database_migration/gravity-db.sh
 create mode 100644 advanced/Scripts/database_migration/gravity/1_to_2.sql

diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh
new file mode 100644
index 00000000..2c5669f0
--- /dev/null
+++ b/advanced/Scripts/database_migration/gravity-db.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1090
+
+# Pi-hole: A black hole for Internet advertisements
+# (c) 2019 Pi-hole, LLC (https://pi-hole.net)
+# Network-wide ad blocking via your own hardware.
+#
+# Updates gravity.db database
+#
+# This file is copyright under the latest version of the EUPL.
+# Please see LICENSE file for your rights under this license.
+
+upgrade_gravityDB(){
+	version=$(sqlite3 "$1" "SELECT "value" FROM "info" WHERE "property" = 'version';")
+	echo $version
+	case "$version" in
+	1)
+		sqlite3 "$1" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql"
+		;;
+	esac
+}
diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql
new file mode 100644
index 00000000..63b7f56f
--- /dev/null
+++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql
@@ -0,0 +1,9 @@
+CREATE TABLE audit
+(
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	domain TEXT UNIQUE NOT NULL,
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	comment TEXT
+);
+
+UPDATE info SET value = 2 WHERE property = 'version';
diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index 91a38ef7..eef9ac80 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -80,13 +80,22 @@ CREATE TABLE gravity
 (
 	domain TEXT PRIMARY KEY
 );
+
+CREATE TABLE audit
+(
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	domain TEXT UNIQUE NOT NULL,
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	comment TEXT
+);
+
 CREATE TABLE info
 (
 	property TEXT PRIMARY KEY,
 	value TEXT NOT NULL
 );
 
-INSERT INTO info VALUES("version","1");
+INSERT INTO info VALUES("version","2");
 
 CREATE VIEW vw_gravity AS SELECT domain
     FROM gravity

From 4f4a12bb40ab6f09a8651ad5ad908edc8c9b59b1 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 5 Jul 2019 14:03:57 +0200
Subject: [PATCH 02/14] Upgrade database if necessary and store audit domains
 therein.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 .../Scripts/database_migration/gravity-db.sh  |  4 +-
 gravity.sh                                    | 54 +++++++++++++------
 2 files changed, 40 insertions(+), 18 deletions(-)

diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh
index 2c5669f0..492546be 100644
--- a/advanced/Scripts/database_migration/gravity-db.sh
+++ b/advanced/Scripts/database_migration/gravity-db.sh
@@ -11,8 +11,8 @@
 # Please see LICENSE file for your rights under this license.
 
 upgrade_gravityDB(){
-	version=$(sqlite3 "$1" "SELECT "value" FROM "info" WHERE "property" = 'version';")
-	echo $version
+	local version=$(sqlite3 "$1" "SELECT "value" FROM "info" WHERE "property" = 'version';")
+
 	case "$version" in
 	1)
 		sqlite3 "$1" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql"
diff --git a/gravity.sh b/gravity.sh
index 1a325ba7..724238fa 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -17,6 +17,8 @@ coltable="/opt/pihole/COL_TABLE"
 source "${coltable}"
 regexconverter="/opt/pihole/wildcard_regex_converter.sh"
 source "${regexconverter}"
+readonly databaseMigrationScript="/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh"
+source "${databaseMigrationScript}"
 
 basename="pihole"
 PIHOLE_COMMAND="/usr/local/bin/${basename}"
@@ -28,6 +30,7 @@ whitelistFile="${piholeDir}/whitelist.txt"
 blacklistFile="${piholeDir}/blacklist.txt"
 regexFile="${piholeDir}/regex.list"
 adListFile="${piholeDir}/adlists.list"
+auditFile="${piholeDir}/audit.list"
 
 localList="${piholeDir}/local.list"
 VPNList="/etc/openvpn/ipp.txt"
@@ -116,14 +119,27 @@ database_table_from_file() {
     declare -i rowid
     rowid=1
     # Read file line by line
-    grep -v '^ *#' < "${source}" | while IFS= read -r domain
-    do
-      # Only add non-empty lines
-      if [[ ! -z "${domain}" ]]; then
-        echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}"
-        rowid+=1
-      fi
-    done
+    if [[ "${table}" == "audit" ]]; then
+      grep -v '^ *#' < "${source}" | while IFS= read -r domain
+      do
+        # Only add non-empty lines
+        if [[ ! -z "${domain}" ]]; then
+          # Audit table format
+          echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}"
+          rowid+=1
+        fi
+      done
+    else
+      grep -v '^ *#' < "${source}" | while IFS= read -r domain
+      do
+        # Only add non-empty lines
+        if [[ ! -z "${domain}" ]]; then
+          # White-, black-, and regexlist format
+          echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}"
+          rowid+=1
+        fi
+      done
+    fi
     inputfile="${tmpFile}"
   fi
   # Store domains in database table specified by ${table}
@@ -150,34 +166,40 @@ database_table_from_file() {
 # Migrate pre-v5.0 list files to database-based Pi-hole versions
 migrate_to_database() {
   # Create database file only if not present
-  if [ -e "${gravityDBfile}" ]; then
-    return 0
+  if [ ! -e "${gravityDBfile}" ]; then
+    echo -e "  ${INFO} Creating new gravity database"
+    generate_gravity_database
   fi
 
-  echo -e "  ${INFO} Creating new gravity database"
-  generate_gravity_database
+  # Check if gravity database needs to be updated
+  upgrade_gravityDB "${gravityDBfile}"
 
   # Migrate list files to new database
-  if [[ -e "${adListFile}" ]]; then
+  if [ -e "${adListFile}" ]; then
     # Store adlist domains in database
     echo -e "  ${INFO} Migrating content of ${adListFile} into new database"
     database_table_from_file "adlist" "${adListFile}"
   fi
-  if [[ -e "${blacklistFile}" ]]; then
+  if [ -e "${blacklistFile}" ]; then
     # Store blacklisted domains in database
     echo -e "  ${INFO} Migrating content of ${blacklistFile} into new database"
     database_table_from_file "blacklist" "${blacklistFile}"
   fi
-  if [[ -e "${whitelistFile}" ]]; then
+  if [ -e "${whitelistFile}" ]; then
     # Store whitelisted domains in database
     echo -e "  ${INFO} Migrating content of ${whitelistFile} into new database"
     database_table_from_file "whitelist" "${whitelistFile}"
   fi
-  if [[ -e "${regexFile}" ]]; then
+  if [ -e "${regexFile}" ]; then
     # Store regex domains in database
     echo -e "  ${INFO} Migrating content of ${regexFile} into new database"
     database_table_from_file "regex" "${regexFile}"
   fi
+  if [ -e "${auditFile}" ]; then
+    # Store audit domains in database
+    echo -e "  ${INFO} Migrating content of ${auditFile} into new database"
+    database_table_from_file "audit" "${auditFile}"
+  fi
 }
 
 # Determine if DNS resolution is available before proceeding

From 0c8f5f12215eaceb370e5110d8383fe34a428b17 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 5 Jul 2019 14:06:05 +0200
Subject: [PATCH 03/14] Remove comment field from audit table

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/database_migration/gravity/1_to_2.sql | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql
index 63b7f56f..4e560df5 100644
--- a/advanced/Scripts/database_migration/gravity/1_to_2.sql
+++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql
@@ -2,8 +2,7 @@ CREATE TABLE audit
 (
 	id INTEGER PRIMARY KEY AUTOINCREMENT,
 	domain TEXT UNIQUE NOT NULL,
-	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	comment TEXT
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int))
 );
 
 UPDATE info SET value = 2 WHERE property = 'version';

From 5293beeb77665fc60503f7566352e183c31e4d04 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 5 Jul 2019 14:10:33 +0200
Subject: [PATCH 04/14] Update audit script to store domains in new database
 table.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/webpage.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh
index f744d1b9..9a6ddfc9 100755
--- a/advanced/Scripts/webpage.sh
+++ b/advanced/Scripts/webpage.sh
@@ -546,17 +546,17 @@ addAudit()
 {
     shift # skip "-a"
     shift # skip "audit"
-    for var in "$@"
+    for domain in "$@"
     do
-        echo "${var}" >> /etc/pihole/auditlog.list
+      # Insert only the domain here. The date_added field will be
+      # filled with its default value (date_added = current timestamp)
+      sqlite3 "${gravityDBfile}" "INSERT INTO \"audit\" (domain) VALUES ('${domain}');"
     done
-    chmod 644 /etc/pihole/auditlog.list
 }
 
 clearAudit()
 {
-    echo -n "" > /etc/pihole/auditlog.list
-    chmod 644 /etc/pihole/auditlog.list
+    sqlite3 "${gravityDBfile}" "DELETE FROM \"audit\";"
 }
 
 SetPrivacyLevel() {

From 82476138c151eba69c223943eb7ef1aa60cfed78 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 5 Jul 2019 16:09:13 +0200
Subject: [PATCH 05/14] Instead of calling sqlite3 multiple times within a
 loop, we use the ability to add multiple rows within one INSERT clause. This
 is supported since sqlite3 3.7.11 (2012-03-20) and should be available on all
 systems.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/webpage.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh
index 9a6ddfc9..3996f7d0 100755
--- a/advanced/Scripts/webpage.sh
+++ b/advanced/Scripts/webpage.sh
@@ -546,12 +546,15 @@ addAudit()
 {
     shift # skip "-a"
     shift # skip "audit"
+    local domains="('${1}')"
+    shift # skip first domain, as it has already been added
     for domain in "$@"
     do
       # Insert only the domain here. The date_added field will be
       # filled with its default value (date_added = current timestamp)
-      sqlite3 "${gravityDBfile}" "INSERT INTO \"audit\" (domain) VALUES ('${domain}');"
+      domains="${domains},('${domain}')"
     done
+    sqlite3 "${gravityDBfile}" "INSERT INTO \"audit\" (domain) VALUES ${domains};"
 }
 
 clearAudit()

From 2fb4256f84b1de7b264c5bf62849c5b7c58399bd Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 5 Jul 2019 16:28:36 +0200
Subject: [PATCH 06/14] Rename table to "auditlist"

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/database_migration/gravity/1_to_2.sql | 2 +-
 advanced/Scripts/webpage.sh                            | 4 ++--
 advanced/Templates/gravity.db.sql                      | 5 ++---
 gravity.sh                                             | 8 ++++----
 4 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql
index 4e560df5..ba051e25 100644
--- a/advanced/Scripts/database_migration/gravity/1_to_2.sql
+++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql
@@ -1,4 +1,4 @@
-CREATE TABLE audit
+CREATE TABLE auditlist
 (
 	id INTEGER PRIMARY KEY AUTOINCREMENT,
 	domain TEXT UNIQUE NOT NULL,
diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh
index 3996f7d0..91d35d98 100755
--- a/advanced/Scripts/webpage.sh
+++ b/advanced/Scripts/webpage.sh
@@ -554,12 +554,12 @@ addAudit()
       # filled with its default value (date_added = current timestamp)
       domains="${domains},('${domain}')"
     done
-    sqlite3 "${gravityDBfile}" "INSERT INTO \"audit\" (domain) VALUES ${domains};"
+    sqlite3 "${gravityDBfile}" "INSERT INTO \"auditlist\" (domain) VALUES ${domains};"
 }
 
 clearAudit()
 {
-    sqlite3 "${gravityDBfile}" "DELETE FROM \"audit\";"
+    sqlite3 "${gravityDBfile}" "DELETE FROM \"auditlist\";"
 }
 
 SetPrivacyLevel() {
diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index eef9ac80..78999e7c 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -81,12 +81,11 @@ CREATE TABLE gravity
 	domain TEXT PRIMARY KEY
 );
 
-CREATE TABLE audit
+CREATE TABLE auditlist
 (
 	id INTEGER PRIMARY KEY AUTOINCREMENT,
 	domain TEXT UNIQUE NOT NULL,
-	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	comment TEXT
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int))
 );
 
 CREATE TABLE info
diff --git a/gravity.sh b/gravity.sh
index 724238fa..09c66c4f 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -30,7 +30,7 @@ whitelistFile="${piholeDir}/whitelist.txt"
 blacklistFile="${piholeDir}/blacklist.txt"
 regexFile="${piholeDir}/regex.list"
 adListFile="${piholeDir}/adlists.list"
-auditFile="${piholeDir}/audit.list"
+auditFile="${piholeDir}/auditlog.list"
 
 localList="${piholeDir}/local.list"
 VPNList="/etc/openvpn/ipp.txt"
@@ -119,12 +119,12 @@ database_table_from_file() {
     declare -i rowid
     rowid=1
     # Read file line by line
-    if [[ "${table}" == "audit" ]]; then
+    if [[ "${table}" == "auditlist" ]]; then
       grep -v '^ *#' < "${source}" | while IFS= read -r domain
       do
         # Only add non-empty lines
         if [[ ! -z "${domain}" ]]; then
-          # Audit table format
+          # Auditlist table format
           echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}"
           rowid+=1
         fi
@@ -198,7 +198,7 @@ migrate_to_database() {
   if [ -e "${auditFile}" ]; then
     # Store audit domains in database
     echo -e "  ${INFO} Migrating content of ${auditFile} into new database"
-    database_table_from_file "audit" "${auditFile}"
+    database_table_from_file "auditlist" "${auditFile}"
   fi
 }
 

From 0405aaa3dac94da58585ff9ba7b131d5fc7c490f Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sat, 6 Jul 2019 09:32:41 +0200
Subject: [PATCH 07/14] Review comments and fixing stickler complaints.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 .../Scripts/database_migration/gravity-db.sh  | 10 ++++-----
 .../database_migration/gravity/1_to_2.sql     |  2 +-
 advanced/Scripts/webpage.sh                   | 22 +++++++++++++++----
 advanced/Templates/gravity.db.sql             |  2 +-
 gravity.sh                                    | 15 ++++++++-----
 5 files changed, 34 insertions(+), 17 deletions(-)

diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh
index 492546be..55411214 100644
--- a/advanced/Scripts/database_migration/gravity-db.sh
+++ b/advanced/Scripts/database_migration/gravity-db.sh
@@ -11,11 +11,11 @@
 # Please see LICENSE file for your rights under this license.
 
 upgrade_gravityDB(){
-	local version=$(sqlite3 "$1" "SELECT "value" FROM "info" WHERE "property" = 'version';")
+	local version
+	version=$(sqlite3 "$1" "SELECT "value" FROM "info" WHERE "property" = 'version';")
 
-	case "$version" in
-	1)
+	if [[ "$version" == "1" ]]; then
 		sqlite3 "$1" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql"
-		;;
-	esac
+		version=2
+	fi
 }
diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql
index ba051e25..073eced5 100644
--- a/advanced/Scripts/database_migration/gravity/1_to_2.sql
+++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql
@@ -1,4 +1,4 @@
-CREATE TABLE auditlist
+CREATE TABLE domain_auditlist
 (
 	id INTEGER PRIMARY KEY AUTOINCREMENT,
 	domain TEXT UNIQUE NOT NULL,
diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh
index 91d35d98..516ea4e4 100755
--- a/advanced/Scripts/webpage.sh
+++ b/advanced/Scripts/webpage.sh
@@ -542,24 +542,38 @@ Teleporter() {
     php /var/www/html/admin/scripts/pi-hole/php/teleporter.php > "pi-hole-teleporter_${datetimestamp}.tar.gz"
 }
 
+checkDomain()
+{
+    local domain validDomain
+    # Convert to lowercase
+    domain="${1,,}"
+    validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check
+    validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label
+    echo "${validDomain}"
+}
+
 addAudit()
 {
     shift # skip "-a"
     shift # skip "audit"
-    local domains="('${1}')"
+    local domains validDomain
+    domains="('$(checkDomain "${1}")')"
     shift # skip first domain, as it has already been added
     for domain in "$@"
     do
       # Insert only the domain here. The date_added field will be
       # filled with its default value (date_added = current timestamp)
-      domains="${domains},('${domain}')"
+      validDomain="$(checkDomain "${domain}")"
+      if [[ -n "${validDomain}" ]]; then
+        domains="${domains},('${domain}')"
+      fi
     done
-    sqlite3 "${gravityDBfile}" "INSERT INTO \"auditlist\" (domain) VALUES ${domains};"
+    sqlite3 "${gravityDBfile}" "INSERT INTO \"domain_auditlist\" (domain) VALUES ${domains};"
 }
 
 clearAudit()
 {
-    sqlite3 "${gravityDBfile}" "DELETE FROM \"auditlist\";"
+    sqlite3 "${gravityDBfile}" "DELETE FROM \"domain_auditlist\";"
 }
 
 SetPrivacyLevel() {
diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index 78999e7c..113c035f 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -81,7 +81,7 @@ CREATE TABLE gravity
 	domain TEXT PRIMARY KEY
 );
 
-CREATE TABLE auditlist
+CREATE TABLE domain_auditlist
 (
 	id INTEGER PRIMARY KEY AUTOINCREMENT,
 	domain TEXT UNIQUE NOT NULL,
diff --git a/gravity.sh b/gravity.sh
index 09c66c4f..3a8afc7a 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -115,25 +115,28 @@ database_table_from_file() {
     inputfile="${source}"
   else
     # Apply format for white-, blacklist, regex, and adlist tables
-    local rowid
-    declare -i rowid
-    rowid=1
     # Read file line by line
     if [[ "${table}" == "auditlist" ]]; then
+      local rowid
+      declare -i rowid
+      rowid=1
       grep -v '^ *#' < "${source}" | while IFS= read -r domain
       do
         # Only add non-empty lines
-        if [[ ! -z "${domain}" ]]; then
+        if [[ -n "${domain}" ]]; then
           # Auditlist table format
           echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}"
           rowid+=1
         fi
       done
     else
+      local rowid
+      declare -i rowid
+      rowid=1
       grep -v '^ *#' < "${source}" | while IFS= read -r domain
       do
         # Only add non-empty lines
-        if [[ ! -z "${domain}" ]]; then
+        if [[ -n "${domain}" ]]; then
           # White-, black-, and regexlist format
           echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}"
           rowid+=1
@@ -198,7 +201,7 @@ migrate_to_database() {
   if [ -e "${auditFile}" ]; then
     # Store audit domains in database
     echo -e "  ${INFO} Migrating content of ${auditFile} into new database"
-    database_table_from_file "auditlist" "${auditFile}"
+    database_table_from_file "domain_auditlist" "${auditFile}"
   fi
 }
 

From efe8216445fbb7345a381f91fed3a3fefd5f5117 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sat, 6 Jul 2019 09:45:07 +0200
Subject: [PATCH 08/14] Fix further stickler complaint.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/database_migration/gravity-db.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh
index 55411214..46ff0a72 100644
--- a/advanced/Scripts/database_migration/gravity-db.sh
+++ b/advanced/Scripts/database_migration/gravity-db.sh
@@ -12,7 +12,7 @@
 
 upgrade_gravityDB(){
 	local version
-	version=$(sqlite3 "$1" "SELECT "value" FROM "info" WHERE "property" = 'version';")
+	version="$(sqlite3 "$1" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")"
 
 	if [[ "$version" == "1" ]]; then
 		sqlite3 "$1" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql"

From acc50b709efab4a88e2c1f2e7940b5708a47b6aa Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sun, 7 Jul 2019 10:33:08 +0200
Subject: [PATCH 09/14] Only migrate files once (domain and adlist lists druing
 initial creation of gravity.db auditlog.list on database upgrade from version
 1 to 2.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 .../Scripts/database_migration/gravity-db.sh  |  9 ++
 advanced/Templates/gravity.db.sql             |  9 +-
 gravity.sh                                    | 87 ++++++++-----------
 3 files changed, 46 insertions(+), 59 deletions(-)

diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh
index 46ff0a72..714676f4 100644
--- a/advanced/Scripts/database_migration/gravity-db.sh
+++ b/advanced/Scripts/database_migration/gravity-db.sh
@@ -15,7 +15,16 @@ upgrade_gravityDB(){
 	version="$(sqlite3 "$1" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")"
 
 	if [[ "$version" == "1" ]]; then
+		# This migration script upgrades the gravity.db file by
+		# adding the domain_auditlist table
 		sqlite3 "$1" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql"
 		version=2
+
+		# Store audit domains in database table
+		if [ -e "${auditFile}" ]; then
+			echo -e "  ${INFO} Migrating content of ${auditFile} into new database"
+			# database_table_from_file is defined in gravity.sh
+			database_table_from_file "domain_auditlist" "${auditFile}"
+		fi
 	fi
 }
diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index 113c035f..09d581f0 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -81,20 +81,13 @@ CREATE TABLE gravity
 	domain TEXT PRIMARY KEY
 );
 
-CREATE TABLE domain_auditlist
-(
-	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	domain TEXT UNIQUE NOT NULL,
-	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int))
-);
-
 CREATE TABLE info
 (
 	property TEXT PRIMARY KEY,
 	value TEXT NOT NULL
 );
 
-INSERT INTO info VALUES("version","2");
+INSERT INTO info VALUES("version","1");
 
 CREATE VIEW vw_gravity AS SELECT domain
     FROM gravity
diff --git a/gravity.sh b/gravity.sh
index 3a8afc7a..e2f2a9a8 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -17,8 +17,7 @@ coltable="/opt/pihole/COL_TABLE"
 source "${coltable}"
 regexconverter="/opt/pihole/wildcard_regex_converter.sh"
 source "${regexconverter}"
-readonly databaseMigrationScript="/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh"
-source "${databaseMigrationScript}"
+source "/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh"
 
 basename="pihole"
 PIHOLE_COMMAND="/usr/local/bin/${basename}"
@@ -116,33 +115,23 @@ database_table_from_file() {
   else
     # Apply format for white-, blacklist, regex, and adlist tables
     # Read file line by line
-    if [[ "${table}" == "auditlist" ]]; then
-      local rowid
-      declare -i rowid
-      rowid=1
-      grep -v '^ *#' < "${source}" | while IFS= read -r domain
-      do
-        # Only add non-empty lines
-        if [[ -n "${domain}" ]]; then
+    local rowid
+    declare -i rowid
+    rowid=1
+    grep -v '^ *#' < "${source}" | while IFS= read -r domain
+    do
+      # Only add non-empty lines
+      if [[ -n "${domain}" ]]; then
+        if [[ "${table}" == "auditlist" ]]; then
           # Auditlist table format
           echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}"
-          rowid+=1
-        fi
-      done
-    else
-      local rowid
-      declare -i rowid
-      rowid=1
-      grep -v '^ *#' < "${source}" | while IFS= read -r domain
-      do
-        # Only add non-empty lines
-        if [[ -n "${domain}" ]]; then
+        else
           # White-, black-, and regexlist format
           echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}"
-          rowid+=1
         fi
-      done
-    fi
+        rowid+=1
+      fi
+    done
     inputfile="${tmpFile}"
   fi
   # Store domains in database table specified by ${table}
@@ -170,39 +159,35 @@ database_table_from_file() {
 migrate_to_database() {
   # Create database file only if not present
   if [ ! -e "${gravityDBfile}" ]; then
+    # Create new database file - note that this will be created in version 1
     echo -e "  ${INFO} Creating new gravity database"
     generate_gravity_database
+
+    # Migrate list files to new database
+    if [ -e "${adListFile}" ]; then
+      # Store adlist domains in database
+      echo -e "  ${INFO} Migrating content of ${adListFile} into new database"
+      database_table_from_file "adlist" "${adListFile}"
+    fi
+    if [ -e "${blacklistFile}" ]; then
+      # Store blacklisted domains in database
+      echo -e "  ${INFO} Migrating content of ${blacklistFile} into new database"
+      database_table_from_file "blacklist" "${blacklistFile}"
+    fi
+    if [ -e "${whitelistFile}" ]; then
+      # Store whitelisted domains in database
+      echo -e "  ${INFO} Migrating content of ${whitelistFile} into new database"
+      database_table_from_file "whitelist" "${whitelistFile}"
+    fi
+    if [ -e "${regexFile}" ]; then
+      # Store regex domains in database
+      echo -e "  ${INFO} Migrating content of ${regexFile} into new database"
+      database_table_from_file "regex" "${regexFile}"
+    fi
   fi
 
   # Check if gravity database needs to be updated
   upgrade_gravityDB "${gravityDBfile}"
-
-  # Migrate list files to new database
-  if [ -e "${adListFile}" ]; then
-    # Store adlist domains in database
-    echo -e "  ${INFO} Migrating content of ${adListFile} into new database"
-    database_table_from_file "adlist" "${adListFile}"
-  fi
-  if [ -e "${blacklistFile}" ]; then
-    # Store blacklisted domains in database
-    echo -e "  ${INFO} Migrating content of ${blacklistFile} into new database"
-    database_table_from_file "blacklist" "${blacklistFile}"
-  fi
-  if [ -e "${whitelistFile}" ]; then
-    # Store whitelisted domains in database
-    echo -e "  ${INFO} Migrating content of ${whitelistFile} into new database"
-    database_table_from_file "whitelist" "${whitelistFile}"
-  fi
-  if [ -e "${regexFile}" ]; then
-    # Store regex domains in database
-    echo -e "  ${INFO} Migrating content of ${regexFile} into new database"
-    database_table_from_file "regex" "${regexFile}"
-  fi
-  if [ -e "${auditFile}" ]; then
-    # Store audit domains in database
-    echo -e "  ${INFO} Migrating content of ${auditFile} into new database"
-    database_table_from_file "domain_auditlist" "${auditFile}"
-  fi
 }
 
 # Determine if DNS resolution is available before proceeding

From be3e198f9a83989a6db71011a294cdce0ef7f56c Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sun, 7 Jul 2019 10:46:20 +0200
Subject: [PATCH 10/14] Address linting errors.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/database_migration/gravity-db.sh | 8 +++++---
 gravity.sh                                        | 3 ++-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh
index 714676f4..294fd32a 100644
--- a/advanced/Scripts/database_migration/gravity-db.sh
+++ b/advanced/Scripts/database_migration/gravity-db.sh
@@ -11,13 +11,15 @@
 # Please see LICENSE file for your rights under this license.
 
 upgrade_gravityDB(){
-	local version
-	version="$(sqlite3 "$1" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")"
+	local database auditFile version
+	database="${1}"
+	auditFile="${2}"
+	version="$(sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")"
 
 	if [[ "$version" == "1" ]]; then
 		# This migration script upgrades the gravity.db file by
 		# adding the domain_auditlist table
-		sqlite3 "$1" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql"
+		sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql"
 		version=2
 
 		# Store audit domains in database table
diff --git a/gravity.sh b/gravity.sh
index e2f2a9a8..89f77ce0 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -17,6 +17,7 @@ coltable="/opt/pihole/COL_TABLE"
 source "${coltable}"
 regexconverter="/opt/pihole/wildcard_regex_converter.sh"
 source "${regexconverter}"
+# shellcheck disable=SC1091
 source "/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh"
 
 basename="pihole"
@@ -187,7 +188,7 @@ migrate_to_database() {
   fi
 
   # Check if gravity database needs to be updated
-  upgrade_gravityDB "${gravityDBfile}"
+  upgrade_gravityDB "${gravityDBfile}" "${auditFile}"
 }
 
 # Determine if DNS resolution is available before proceeding

From 8382f4d7274e893e19dbe09de5dd090daa590c24 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sun, 7 Jul 2019 21:21:56 +0200
Subject: [PATCH 11/14] Rename table to domain_audit and simplify subroutine
 addAudit().

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/database_migration/gravity-db.sh  |  4 ++--
 .../Scripts/database_migration/gravity/1_to_2.sql  |  2 +-
 advanced/Scripts/webpage.sh                        | 14 +++++++++-----
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh
index 294fd32a..51a3480b 100644
--- a/advanced/Scripts/database_migration/gravity-db.sh
+++ b/advanced/Scripts/database_migration/gravity-db.sh
@@ -18,7 +18,7 @@ upgrade_gravityDB(){
 
 	if [[ "$version" == "1" ]]; then
 		# This migration script upgrades the gravity.db file by
-		# adding the domain_auditlist table
+		# adding the domain_audit table
 		sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql"
 		version=2
 
@@ -26,7 +26,7 @@ upgrade_gravityDB(){
 		if [ -e "${auditFile}" ]; then
 			echo -e "  ${INFO} Migrating content of ${auditFile} into new database"
 			# database_table_from_file is defined in gravity.sh
-			database_table_from_file "domain_auditlist" "${auditFile}"
+			database_table_from_file "domain_audit" "${auditFile}"
 		fi
 	fi
 }
diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql
index 073eced5..90a48418 100644
--- a/advanced/Scripts/database_migration/gravity/1_to_2.sql
+++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql
@@ -1,4 +1,4 @@
-CREATE TABLE domain_auditlist
+CREATE TABLE domain_audit
 (
 	id INTEGER PRIMARY KEY AUTOINCREMENT,
 	domain TEXT UNIQUE NOT NULL,
diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh
index 516ea4e4..918fde27 100755
--- a/advanced/Scripts/webpage.sh
+++ b/advanced/Scripts/webpage.sh
@@ -557,23 +557,27 @@ addAudit()
     shift # skip "-a"
     shift # skip "audit"
     local domains validDomain
-    domains="('$(checkDomain "${1}")')"
-    shift # skip first domain, as it has already been added
+    domains=""
     for domain in "$@"
     do
       # Insert only the domain here. The date_added field will be
       # filled with its default value (date_added = current timestamp)
       validDomain="$(checkDomain "${domain}")"
       if [[ -n "${validDomain}" ]]; then
-        domains="${domains},('${domain}')"
+        # Put comma in between () when there is
+        # more than one domains to be added
+        if [[ -n "${domains}" ]]; then
+          domains="${domains},"
+        fi
+        domains="${domains}('${domain}')"
       fi
     done
-    sqlite3 "${gravityDBfile}" "INSERT INTO \"domain_auditlist\" (domain) VALUES ${domains};"
+    sqlite3 "${gravityDBfile}" "INSERT INTO \"domain_audit\" (domain) VALUES ${domains};"
 }
 
 clearAudit()
 {
-    sqlite3 "${gravityDBfile}" "DELETE FROM \"domain_auditlist\";"
+    sqlite3 "${gravityDBfile}" "DELETE FROM \"domain_audit\";"
 }
 
 SetPrivacyLevel() {

From 3d3fc2947e5d848e7ea73bef7b6a34a9faa48091 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 8 Jul 2019 19:22:35 +0200
Subject: [PATCH 12/14] Review comments

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/webpage.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh
index 918fde27..227363f0 100755
--- a/advanced/Scripts/webpage.sh
+++ b/advanced/Scripts/webpage.sh
@@ -564,20 +564,22 @@ addAudit()
       # filled with its default value (date_added = current timestamp)
       validDomain="$(checkDomain "${domain}")"
       if [[ -n "${validDomain}" ]]; then
-        # Put comma in between () when there is
+        # Put comma in between domains when there is
         # more than one domains to be added
+        # SQL INSERT allows adding multiple rows at once using the format
+        ## INSERT INTO table (domain) VALUES ('abc.de'),('fgh.ij'),('klm.no'),('pqr.st');
         if [[ -n "${domains}" ]]; then
           domains="${domains},"
         fi
         domains="${domains}('${domain}')"
       fi
     done
-    sqlite3 "${gravityDBfile}" "INSERT INTO \"domain_audit\" (domain) VALUES ${domains};"
+    sqlite3 "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};"
 }
 
 clearAudit()
 {
-    sqlite3 "${gravityDBfile}" "DELETE FROM \"domain_audit\";"
+    sqlite3 "${gravityDBfile}" "DELETE FROM domain_audit;"
 }
 
 SetPrivacyLevel() {

From 5ff90522002576d9a0202a7d458d1d38117659e1 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Tue, 9 Jul 2019 11:41:44 +0200
Subject: [PATCH 13/14] Review comments

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/database_migration/gravity-db.sh | 7 +++++--
 advanced/Scripts/webpage.sh                       | 5 +++--
 gravity.sh                                        | 7 +++----
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh
index 51a3480b..65b42b95 100644
--- a/advanced/Scripts/database_migration/gravity-db.sh
+++ b/advanced/Scripts/database_migration/gravity-db.sh
@@ -11,9 +11,12 @@
 # Please see LICENSE file for your rights under this license.
 
 upgrade_gravityDB(){
-	local database auditFile version
+	local database piholeDir auditFile version
 	database="${1}"
-	auditFile="${2}"
+	piholeDir="${2}"
+	auditFile="${piholeDir}/auditlog.list"
+
+	# Get database version
 	version="$(sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")"
 
 	if [[ "$version" == "1" ]]; then
diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh
index 227363f0..9db10bda 100755
--- a/advanced/Scripts/webpage.sh
+++ b/advanced/Scripts/webpage.sh
@@ -560,8 +560,7 @@ addAudit()
     domains=""
     for domain in "$@"
     do
-      # Insert only the domain here. The date_added field will be
-      # filled with its default value (date_added = current timestamp)
+      # Check domain to be added. Only continue if it is valid
       validDomain="$(checkDomain "${domain}")"
       if [[ -n "${validDomain}" ]]; then
         # Put comma in between domains when there is
@@ -574,6 +573,8 @@ addAudit()
         domains="${domains}('${domain}')"
       fi
     done
+    # Insert only the domain here. The date_added field will be
+    # filled with its default value (date_added = current timestamp)
     sqlite3 "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};"
 }
 
diff --git a/gravity.sh b/gravity.sh
index 89f77ce0..d7c66d68 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -30,7 +30,6 @@ whitelistFile="${piholeDir}/whitelist.txt"
 blacklistFile="${piholeDir}/blacklist.txt"
 regexFile="${piholeDir}/regex.list"
 adListFile="${piholeDir}/adlists.list"
-auditFile="${piholeDir}/auditlog.list"
 
 localList="${piholeDir}/local.list"
 VPNList="/etc/openvpn/ipp.txt"
@@ -123,8 +122,8 @@ database_table_from_file() {
     do
       # Only add non-empty lines
       if [[ -n "${domain}" ]]; then
-        if [[ "${table}" == "auditlist" ]]; then
-          # Auditlist table format
+        if [[ "${table}" == "domain_audit" ]]; then
+          # domain_audit table format (no enable or modified fields)
           echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}"
         else
           # White-, black-, and regexlist format
@@ -188,7 +187,7 @@ migrate_to_database() {
   fi
 
   # Check if gravity database needs to be updated
-  upgrade_gravityDB "${gravityDBfile}" "${auditFile}"
+  upgrade_gravityDB "${gravityDBfile}" "${piholeDir}"
 }
 
 # Determine if DNS resolution is available before proceeding

From 420f60b5c7748ccf939c104a77c9dbf30116f668 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 10 Jul 2019 11:56:39 +0200
Subject: [PATCH 14/14] Add timeout to migration script (1->2).

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/database_migration/gravity/1_to_2.sql | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql
index 90a48418..45b5fa02 100644
--- a/advanced/Scripts/database_migration/gravity/1_to_2.sql
+++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql
@@ -1,3 +1,5 @@
+.timeout 30000
+
 CREATE TABLE domain_audit
 (
 	id INTEGER PRIMARY KEY AUTOINCREMENT,