From 185319d560829d9304f38db568c61e8e6955bc90 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sat, 30 Nov 2019 12:33:16 +0000
Subject: [PATCH 01/13] Unite four domain tables into a single domainlist
 table.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 .../Scripts/database_migration/gravity-db.sh  |  7 ++
 .../database_migration/gravity/3_to_4.sql     | 96 +++++++++++++++++++
 advanced/Scripts/list.sh                      | 34 +++++--
 3 files changed, 128 insertions(+), 9 deletions(-)
 create mode 100644 advanced/Scripts/database_migration/gravity/3_to_4.sql

diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh
index 0fe90d8a..d7de0ec5 100644
--- a/advanced/Scripts/database_migration/gravity-db.sh
+++ b/advanced/Scripts/database_migration/gravity-db.sh
@@ -41,4 +41,11 @@ upgrade_gravityDB(){
 		sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/2_to_3.sql"
 		version=3
 	fi
+	if [[ "$version" == "3" ]]; then
+		# This migration script unifies the formally separated domain
+		# lists into a single table with a UNIQUE domain constraint
+		echo -e "  ${INFO} Upgrading gravity database from version 3 to 4"
+		sqlite3 "${database}" < "${scriptPath}/3_to_4.sql"
+		version=6
+	fi
 }
diff --git a/advanced/Scripts/database_migration/gravity/3_to_4.sql b/advanced/Scripts/database_migration/gravity/3_to_4.sql
new file mode 100644
index 00000000..8d1c1d26
--- /dev/null
+++ b/advanced/Scripts/database_migration/gravity/3_to_4.sql
@@ -0,0 +1,96 @@
+.timeout 30000
+
+PRAGMA FOREIGN_KEYS=OFF;
+
+BEGIN TRANSACTION;
+
+CREATE TABLE domainlist
+(
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	type INTEGER NOT NULL DEFAULT 0,
+	domain TEXT UNIQUE NOT NULL,
+	enabled BOOLEAN NOT NULL DEFAULT 1,
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	comment TEXT
+);
+
+ALTER TABLE whitelist ADD COLUMN type INTEGER;
+UPDATE whitelist SET type = 0;
+INSERT INTO domainlist (type,domain,enabled,date_added,date_modified,comment)
+    SELECT type,domain,enabled,date_added,date_modified,comment FROM whitelist;
+
+ALTER TABLE blacklist ADD COLUMN type INTEGER;
+UPDATE blacklist SET type = 1;
+INSERT INTO domainlist (type,domain,enabled,date_added,date_modified,comment)
+    SELECT type,domain,enabled,date_added,date_modified,comment FROM blacklist;
+
+ALTER TABLE regex_whitelist ADD COLUMN type INTEGER;
+UPDATE regex_whitelist SET type = 2;
+INSERT INTO domainlist (type,domain,enabled,date_added,date_modified,comment)
+    SELECT type,domain,enabled,date_added,date_modified,comment FROM regex_whitelist;
+
+ALTER TABLE regex_blacklist ADD COLUMN type INTEGER;
+UPDATE regex_blacklist SET type = 3;
+INSERT INTO domainlist (type,domain,enabled,date_added,date_modified,comment)
+    SELECT type,domain,enabled,date_added,date_modified,comment FROM regex_blacklist;
+
+DROP TABLE whitelist_by_group;
+DROP TABLE blacklist_by_group;
+DROP TABLE regex_whitelist_by_group;
+DROP TABLE regex_blacklist_by_group;
+CREATE TABLE domainlist_by_group
+(
+	domainlist_id INTEGER NOT NULL REFERENCES domainlist (id),
+	group_id INTEGER NOT NULL REFERENCES "group" (id),
+	PRIMARY KEY (domainlist_id, group_id)
+);
+
+DROP TRIGGER tr_whitelist_update;
+DROP TRIGGER tr_blacklist_update;
+DROP TRIGGER tr_regex_whitelist_update;
+DROP TRIGGER tr_regex_blacklist_update;
+CREATE TRIGGER tr_domainlist_update AFTER UPDATE ON domainlist
+    BEGIN
+      UPDATE domainlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
+    END;
+
+DROP VIEW vw_whitelist;
+CREATE VIEW vw_whitelist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
+    FROM domainlist
+    LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id
+    LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id
+    WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1)
+    AND domainlist.type = 0
+    ORDER BY domainlist.id;
+
+DROP VIEW vw_blacklist;
+CREATE VIEW vw_blacklist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
+    FROM domainlist
+    LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id
+    LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id
+    WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1)
+    AND domainlist.type = 1
+    ORDER BY domainlist.id;
+
+DROP VIEW vw_regex_whitelist;
+CREATE VIEW vw_regex_whitelist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
+    FROM domainlist
+    LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id
+    LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id
+    WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1)
+    AND domainlist.type = 2
+    ORDER BY domainlist.id;
+
+DROP VIEW vw_regex_blacklist;
+CREATE VIEW vw_regex_blacklist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
+    FROM domainlist
+    LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id
+    LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id
+    WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1)
+    AND domainlist.type = 3
+    ORDER BY domainlist.id;
+
+UPDATE info SET value = 6 WHERE property = 'version';
+
+COMMIT;
diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 6a606665..483b7153 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -27,6 +27,17 @@ listname=""
 colfile="/opt/pihole/COL_TABLE"
 source ${colfile}
 
+getTypeID() {
+    if [[ "$1" == "whitelist" ]]; then
+        echo "0"
+    elif  [[ "$1" == "blacklist" ]]; then
+        echo "1"
+    elif  [[ "$1" == "regex_whitelist" ]]; then
+        echo "2"
+    elif  [[ "$1" == "regex_blacklist" ]]; then
+        echo "3"
+    fi
+}
 
 helpFunc() {
     if [[ "${listType}" == "whitelist" ]]; then
@@ -129,13 +140,14 @@ ProcessDomainList() {
 }
 
 AddDomain() {
-    local domain list num
+    local domain list num typeID
     # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input
     domain="$1"
     list="$2"
+    typeID="$(getTypeID "${list}")"
 
     # Is the domain in the list we want to add it to?
-    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = '${domain}';")"
+    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeID};")"
 
     if [[ "${num}" -ne 0 ]]; then
       if [[ "${verbose}" == true ]]; then
@@ -151,17 +163,18 @@ AddDomain() {
     reload=true
     # Insert only the domain here. The enabled and date_added fields will be filled
     # with their default values (enabled = true, date_added = current timestamp)
-    sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain) VALUES ('${domain}');"
+    sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeID});"
 }
 
 RemoveDomain() {
-    local domain list num
+    local domain list num typeID
     # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input
     domain="$1"
     list="$2"
+    typeID="$(getTypeID "${list}")"
 
     # Is the domain in the list we want to remove it from?
-    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = '${domain}';")"
+    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeID};")"
 
     if [[ "${num}" -eq 0 ]]; then
       if [[ "${verbose}" == true ]]; then
@@ -176,14 +189,15 @@ RemoveDomain() {
     fi
     reload=true
     # Remove it from the current list
-    sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE domain = '${domain}';"
+    sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeID};"
 }
 
 Displaylist() {
-    local list listname count num_pipes domain enabled status nicedate
+    local list listname count num_pipes domain enabled status nicedate typeID
 
     listname="${listType}"
-    data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM ${listType};" 2> /dev/null)"
+    typeID="$(getTypeID "${listType}")"
+    data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeID};" 2> /dev/null)"
 
     if [[ -z $data ]]; then
         echo -e "Not showing empty list"
@@ -221,7 +235,9 @@ Displaylist() {
 }
 
 NukeList() {
-    sqlite3 "${gravityDBfile}" "DELETE FROM ${listType};"
+    local typeID
+    typeID=$(getTypeID "${list}")
+    sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeID};"
 }
 
 for var in "$@"; do

From a1f120b2ff5fd7801202a481723c12871a8ab902 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sat, 30 Nov 2019 12:43:07 +0000
Subject: [PATCH 02/13] Address stickler's complaint

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/database_migration/gravity-db.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh
index d7de0ec5..a12be864 100644
--- a/advanced/Scripts/database_migration/gravity-db.sh
+++ b/advanced/Scripts/database_migration/gravity-db.sh
@@ -10,6 +10,8 @@
 # This file is copyright under the latest version of the EUPL.
 # Please see LICENSE file for your rights under this license.
 
+scriptPath="/etc/.pihole/advanced/Scripts/database_migration/gravity"
+
 upgrade_gravityDB(){
 	local database piholeDir auditFile version
 	database="${1}"
@@ -23,7 +25,7 @@ upgrade_gravityDB(){
 		# This migration script upgrades the gravity.db file by
 		# adding the domain_audit table
 		echo -e "  ${INFO} Upgrading gravity database from version 1 to 2"
-		sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql"
+		sqlite3 "${database}" < "${scriptPath}/1_to_2.sql"
 		version=2
 
 		# Store audit domains in database table
@@ -38,7 +40,7 @@ upgrade_gravityDB(){
 		# renaming the regex table to regex_blacklist, and
 		# creating a new regex_whitelist table + corresponding linking table and views
 		echo -e "  ${INFO} Upgrading gravity database from version 2 to 3"
-		sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/2_to_3.sql"
+		sqlite3 "${database}" < "${scriptPath}/2_to_3.sql"
 		version=3
 	fi
 	if [[ "$version" == "3" ]]; then

From d0de5fda3086614b1f8266a9a4ed12f0b6ef9c97 Mon Sep 17 00:00:00 2001
From: Adam Warner <me@adamwarner.co.uk>
Date: Sat, 30 Nov 2019 13:13:26 +0000
Subject: [PATCH 03/13] Simplify removal of domain from one list when it is
 requested for another

Signed-off-by: Adam Warner <me@adamwarner.co.uk>
---
 advanced/Scripts/list.sh | 36 +++++++++++++++++++++++++-----------
 1 file changed, 25 insertions(+), 11 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 483b7153..55272cde 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -39,6 +39,18 @@ getTypeID() {
     fi
 }
 
+getListnameFromType() {
+    if [[ "$1" == "0" ]]; then
+        echo "whitelist"
+    elif  [[ "$1" == "1" ]]; then
+        echo "blacklist"
+    elif  [[ "$1" == "2" ]]; then
+        echo "regex_whitelist"
+    elif  [[ "$1" == "3" ]]; then
+        echo "regex_blacklist"
+    fi
+}
+
 helpFunc() {
     if [[ "${listType}" == "whitelist" ]]; then
         param="w"
@@ -105,19 +117,15 @@ HandleOther() {
 }
 
 ProcessDomainList() {
-    local is_regexlist
     if [[ "${listType}" == "regex_blacklist" ]]; then
         # Regex black filter list
         listname="regex blacklist filters"
-        is_regexlist=true
     elif [[ "${listType}" == "regex_whitelist" ]]; then
         # Regex white filter list
         listname="regex whitelist filters"
-        is_regexlist=true
     else
         # Whitelist / Blacklist
         listname="${listType}"
-        is_regexlist=false
     fi
 
     for dom in "${domList[@]}"; do
@@ -130,9 +138,6 @@ ProcessDomainList() {
         # if delmode then remove from desired list but do not add to the other
         if ${addmode}; then
             AddDomain "${dom}" "${listType}"
-            if ! ${is_regexlist}; then
-                RemoveDomain "${dom}" "${listAlt}"
-            fi
         else
             RemoveDomain "${dom}" "${listType}"
         fi
@@ -140,18 +145,27 @@ ProcessDomainList() {
 }
 
 AddDomain() {
-    local domain list num typeID
+    local domain list num currTypeID currListName typeID
     # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input
     domain="$1"
     list="$2"
     typeID="$(getTypeID "${list}")"
 
     # Is the domain in the list we want to add it to?
-    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeID};")"
+    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")"
 
     if [[ "${num}" -ne 0 ]]; then
-      if [[ "${verbose}" == true ]]; then
-          echo -e "  ${INFO} ${1} already exists in ${listname}, no need to add!"
+      currTypeID="$(sqlite3 "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")"
+      if [[ "${currTypeID}" == "${typeID}" ]]; then
+        if [[ "${verbose}" == true ]]; then
+            echo -e "  ${INFO} ${1} already exists in ${listname}, no need to add!"
+        fi
+      else
+        currListName="$(getListnameFromType "${currTypeID}")"
+        sqlite3 "${gravityDBfile}" "UPDATE domainlist SET type = ${typeID} WHERE domain='${domain}';"
+        if [[ "${verbose}" == true ]]; then
+            echo -e "  ${INFO} ${1} already exists in ${currListName}, it has been updated to the requested list type."
+        fi
       fi
       return
     fi

From 6a881545b0d99952af2e26b50e9e859415fe7254 Mon Sep 17 00:00:00 2001
From: Adam Warner <me@adamwarner.co.uk>
Date: Sat, 30 Nov 2019 13:19:58 +0000
Subject: [PATCH 04/13] tweak wording Signed-off-by: Adam Warner
 <me@adamwarner.co.uk>

---
 advanced/Scripts/list.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 55272cde..596a6fb7 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -164,7 +164,7 @@ AddDomain() {
         currListName="$(getListnameFromType "${currTypeID}")"
         sqlite3 "${gravityDBfile}" "UPDATE domainlist SET type = ${typeID} WHERE domain='${domain}';"
         if [[ "${verbose}" == true ]]; then
-            echo -e "  ${INFO} ${1} already exists in ${currListName}, it has been updated to the requested list type."
+            echo -e "  ${INFO} ${1} already exists in ${currListName}, it has been moved to ${listname}"
         fi
       fi
       return

From 77bfb3fb671a1049119ae3d075c450277b668187 Mon Sep 17 00:00:00 2001
From: Adam Warner <me@adamwarner.co.uk>
Date: Sat, 30 Nov 2019 14:18:12 +0000
Subject: [PATCH 05/13] tidy up variable usage in list.sh Remove some that are
 redundant

Signed-off-by: Adam Warner <me@adamwarner.co.uk>
---
 advanced/Scripts/list.sh | 104 ++++++++++++++-------------------------
 1 file changed, 38 insertions(+), 66 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 596a6fb7..43f84008 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -21,25 +21,12 @@ web=false
 
 domList=()
 
-listType=""
-listname=""
+typeId=""
 
 colfile="/opt/pihole/COL_TABLE"
 source ${colfile}
 
-getTypeID() {
-    if [[ "$1" == "whitelist" ]]; then
-        echo "0"
-    elif  [[ "$1" == "blacklist" ]]; then
-        echo "1"
-    elif  [[ "$1" == "regex_whitelist" ]]; then
-        echo "2"
-    elif  [[ "$1" == "regex_blacklist" ]]; then
-        echo "3"
-    fi
-}
-
-getListnameFromType() {
+getListnameFromTypeId() {
     if [[ "$1" == "0" ]]; then
         echo "whitelist"
     elif  [[ "$1" == "1" ]]; then
@@ -52,19 +39,19 @@ getListnameFromType() {
 }
 
 helpFunc() {
-    if [[ "${listType}" == "whitelist" ]]; then
+    if [[ "${typeId}" == "0" ]]; then
         param="w"
         type="whitelist"
-    elif [[ "${listType}" == "regex_blacklist" && "${wildcard}" == true ]]; then
+    elif [[ "${typeId}" == "3" && "${wildcard}" == true ]]; then
         param="-wild"
         type="wildcard blacklist"
-    elif [[ "${listType}" == "regex_blacklist" ]]; then
+    elif [[ "${typeId}" == "3" ]]; then
         param="-regex"
         type="regex blacklist filter"
-    elif [[ "${listType}" == "regex_whitelist" && "${wildcard}" == true ]]; then
+    elif [[ "${typeId}" == "2" && "${wildcard}" == true ]]; then
         param="-white-wild"
         type="wildcard whitelist"
-    elif [[ "${listType}" == "regex_whitelist" ]]; then
+    elif [[ "${typeId}" == "2" ]]; then
         param="-white-regex"
         type="regex whitelist filter"
     else
@@ -101,7 +88,7 @@ HandleOther() {
 
     # Check validity of domain (don't check for regex entries)
     if [[ "${#domain}" -le 253 ]]; then
-        if [[ ( "${listType}" == "regex_blacklist" || "${listType}" == "regex_whitelist" ) && "${wildcard}" == false ]]; then
+        if [[ ( "${typeId}" == "3" || "${typeId}" == "2" ) && "${wildcard}" == false ]]; then
             validDomain="${domain}"
         else
             validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check
@@ -117,17 +104,6 @@ HandleOther() {
 }
 
 ProcessDomainList() {
-    if [[ "${listType}" == "regex_blacklist" ]]; then
-        # Regex black filter list
-        listname="regex blacklist filters"
-    elif [[ "${listType}" == "regex_whitelist" ]]; then
-        # Regex white filter list
-        listname="regex whitelist filters"
-    else
-        # Whitelist / Blacklist
-        listname="${listType}"
-    fi
-
     for dom in "${domList[@]}"; do
         # Format domain into regex filter if requested
         if [[ "${wildcard}" == true ]]; then
@@ -137,34 +113,33 @@ ProcessDomainList() {
         # Logic: If addmode then add to desired list and remove from the other;
         # if delmode then remove from desired list but do not add to the other
         if ${addmode}; then
-            AddDomain "${dom}" "${listType}"
+            AddDomain "${dom}"
         else
-            RemoveDomain "${dom}" "${listType}"
+            RemoveDomain "${dom}"
         fi
   done
 }
 
 AddDomain() {
-    local domain list num currTypeID currListName typeID
+    local domain num requestedListname existingTypeId existingListname
     # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input
     domain="$1"
-    list="$2"
-    typeID="$(getTypeID "${list}")"
 
     # Is the domain in the list we want to add it to?
     num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")"
+    requestedListname="$(getListnameFromTypeId "${typeId}")"
 
     if [[ "${num}" -ne 0 ]]; then
-      currTypeID="$(sqlite3 "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")"
-      if [[ "${currTypeID}" == "${typeID}" ]]; then
+      existingTypeId="$(sqlite3 "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")"
+      if [[ "${existingTypeId}" == "${typeId}" ]]; then
         if [[ "${verbose}" == true ]]; then
-            echo -e "  ${INFO} ${1} already exists in ${listname}, no need to add!"
+            echo -e "  ${INFO} ${1} already exists in ${requestedListname}, no need to add!"
         fi
       else
-        currListName="$(getListnameFromType "${currTypeID}")"
-        sqlite3 "${gravityDBfile}" "UPDATE domainlist SET type = ${typeID} WHERE domain='${domain}';"
+        existingListname="$(getListnameFromTypeId "${existingTypeId}")"
+        sqlite3 "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';"
         if [[ "${verbose}" == true ]]; then
-            echo -e "  ${INFO} ${1} already exists in ${currListName}, it has been moved to ${listname}"
+            echo -e "  ${INFO} ${1} already exists in ${existingListname}, it has been moved to ${requestedListname}!"
         fi
       fi
       return
@@ -172,51 +147,50 @@ AddDomain() {
 
     # Domain not found in the table, add it!
     if [[ "${verbose}" == true ]]; then
-        echo -e "  ${INFO} Adding ${1} to the ${listname}..."
+        echo -e "  ${INFO} Adding ${domain} to the ${requestedListname}..."
     fi
     reload=true
     # Insert only the domain here. The enabled and date_added fields will be filled
     # with their default values (enabled = true, date_added = current timestamp)
-    sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeID});"
+    sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});"
 }
 
 RemoveDomain() {
-    local domain list num typeID
+    local domain num requestedListname
     # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input
     domain="$1"
-    list="$2"
-    typeID="$(getTypeID "${list}")"
 
     # Is the domain in the list we want to remove it from?
-    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeID};")"
+    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")"
+
+    requestedListname="$(getListnameFromTypeId "${typeId}")"
 
     if [[ "${num}" -eq 0 ]]; then
       if [[ "${verbose}" == true ]]; then
-          echo -e "  ${INFO} ${1} does not exist in ${list}, no need to remove!"
+          echo -e "  ${INFO} ${domain} does not exist in ${requestedListname}, no need to remove!"
       fi
       return
     fi
 
     # Domain found in the table, remove it!
     if [[ "${verbose}" == true ]]; then
-        echo -e "  ${INFO} Removing ${1} from the ${listname}..."
+        echo -e "  ${INFO} Removing ${domain} from the ${requestedListname}..."
     fi
     reload=true
     # Remove it from the current list
-    sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeID};"
+    sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};"
 }
 
 Displaylist() {
-    local list listname count num_pipes domain enabled status nicedate typeID
+    local count num_pipes domain enabled status nicedate requestedListname
 
-    listname="${listType}"
-    typeID="$(getTypeID "${listType}")"
-    data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeID};" 2> /dev/null)"
+    requestedListname="$(getListnameFromTypeId "${typeId}")"
+    data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)"
 
     if [[ -z $data ]]; then
         echo -e "Not showing empty list"
     else
-        echo -e "Displaying ${listname}:"
+        echo -e "Displaying ${requestedListname}:"
         count=1
         while IFS= read -r line
         do
@@ -249,19 +223,17 @@ Displaylist() {
 }
 
 NukeList() {
-    local typeID
-    typeID=$(getTypeID "${list}")
-    sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeID};"
+    sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};"
 }
 
 for var in "$@"; do
     case "${var}" in
-        "-w" | "whitelist"   ) listType="whitelist"; listAlt="blacklist";;
-        "-b" | "blacklist"   ) listType="blacklist"; listAlt="whitelist";;
-        "--wild" | "wildcard" ) listType="regex_blacklist"; wildcard=true;;
-        "--regex" | "regex"   ) listType="regex_blacklist";;
-        "--white-regex" | "white-regex" ) listType="regex_whitelist";;
-        "--white-wild" | "white-wild" ) listType="regex_whitelist"; wildcard=true;;
+        "-w" | "whitelist"   ) typeId=0;;
+        "-b" | "blacklist"   ) typeId=1;;
+        "--white-regex" | "white-regex" ) typeId=2;;
+        "--white-wild" | "white-wild" ) typeId=2; wildcard=true;;
+        "--wild" | "wildcard" ) typeId=3; wildcard=true;;
+        "--regex" | "regex"   ) typeId=3;;
         "-nr"| "--noreload"  ) reload=false;;
         "-d" | "--delmode"   ) addmode=false;;
         "-q" | "--quiet"     ) verbose=false;;

From edaee4e9626223574b4432a03bc7d86f99baa205 Mon Sep 17 00:00:00 2001
From: Adam Warner <me@adamwarner.co.uk>
Date: Sat, 30 Nov 2019 16:02:50 +0000
Subject: [PATCH 06/13] remove redundant function and comments

Signed-off-by: Adam Warner <me@adamwarner.co.uk>
---
 advanced/Scripts/list.sh | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 43f84008..7707ceea 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -74,14 +74,6 @@ Options:
   exit 0
 }
 
-EscapeRegexp() {
-    # This way we may safely insert an arbitrary
-    # string in our regular expressions
-    # This sed is intentionally executed in three steps to ease maintainability
-    # The first sed removes any amount of leading dots
-    echo $* | sed 's/^\.*//' | sed "s/[]\.|$(){}?+*^]/\\\\&/g" | sed "s/\\//\\\\\//g"
-}
-
 HandleOther() {
     # Convert to lowercase
     domain="${1,,}"
@@ -122,7 +114,6 @@ ProcessDomainList() {
 
 AddDomain() {
     local domain num requestedListname existingTypeId existingListname
-    # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input
     domain="$1"
 
     # Is the domain in the list we want to add it to?
@@ -157,7 +148,6 @@ AddDomain() {
 
 RemoveDomain() {
     local domain num requestedListname
-    # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input
     domain="$1"
 
     # Is the domain in the list we want to remove it from?

From 4b8a72fda71bae5cc0ed2cfb6ba5bbb70a623b89 Mon Sep 17 00:00:00 2001
From: Adam Warner <me@adamwarner.co.uk>
Date: Sat, 30 Nov 2019 16:26:26 +0000
Subject: [PATCH 07/13] functionise parameter discovery Rename HandleOther to
 ValidateDomain Capital letters on the new functions

Signed-off-by: Adam Warner <me@adamwarner.co.uk>
---
 advanced/Scripts/list.sh | 59 ++++++++++++++++++++--------------------
 1 file changed, 30 insertions(+), 29 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 7707ceea..5fbe831f 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -26,7 +26,7 @@ typeId=""
 colfile="/opt/pihole/COL_TABLE"
 source ${colfile}
 
-getListnameFromTypeId() {
+GetListnameFromTypeId() {
     if [[ "$1" == "0" ]]; then
         echo "whitelist"
     elif  [[ "$1" == "1" ]]; then
@@ -38,43 +38,44 @@ getListnameFromTypeId() {
     fi
 }
 
-helpFunc() {
+GetListParamFromTypeId() {
     if [[ "${typeId}" == "0" ]]; then
-        param="w"
-        type="whitelist"
-    elif [[ "${typeId}" == "3" && "${wildcard}" == true ]]; then
-        param="-wild"
-        type="wildcard blacklist"
-    elif [[ "${typeId}" == "3" ]]; then
-        param="-regex"
-        type="regex blacklist filter"
-    elif [[ "${typeId}" == "2" && "${wildcard}" == true ]]; then
-        param="-white-wild"
-        type="wildcard whitelist"
-    elif [[ "${typeId}" == "2" ]]; then
-        param="-white-regex"
-        type="regex whitelist filter"
-    else
-        param="b"
-        type="blacklist"
+        echo "w"
+    elif  [[ "${typeId}" == "1" ]]; then
+        echo "b"
+    elif  [[ "${typeId}" == "2" && "${wildcard}" == true ]]; then
+        echo "-white-wild"
+    elif  [[ "${typeId}" == "2" ]]; then
+        echo "regex_blacklist"
+    elif  [[ "${typeId}" == "3" && "${wildcard}" == true ]]; then
+        echo "-regex"
+    elif  [[ "${typeId}" == "3" ]]; then
+        echo "-wild"
     fi
+}
+
+helpFunc() {
+    local listname param
+
+    listname="$(GetListnameFromTypeId "${typeId}")"
+    param="$(GetListParamFromTypeId)"
 
     echo "Usage: pihole -${param} [options] <domain> <domain2 ...>
 Example: 'pihole -${param} site.com', or 'pihole -${param} site1.com site2.com'
-${type^} one or more domains
+${listname^} one or more domains
 
 Options:
-  -d, --delmode       Remove domain(s) from the ${type}
-  -nr, --noreload     Update ${type} without reloading the DNS server
+  -d, --delmode       Remove domain(s) from the ${listname}
+  -nr, --noreload     Update ${listname} without reloading the DNS server
   -q, --quiet         Make output less verbose
   -h, --help          Show this help dialog
-  -l, --list          Display all your ${type}listed domains
+  -l, --list          Display all your ${listname}listed domains
   --nuke              Removes all entries in a list"
 
   exit 0
 }
 
-HandleOther() {
+ValidateDomain() {
     # Convert to lowercase
     domain="${1,,}"
 
@@ -118,7 +119,7 @@ AddDomain() {
 
     # Is the domain in the list we want to add it to?
     num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")"
-    requestedListname="$(getListnameFromTypeId "${typeId}")"
+    requestedListname="$(GetListnameFromTypeId "${typeId}")"
 
     if [[ "${num}" -ne 0 ]]; then
       existingTypeId="$(sqlite3 "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")"
@@ -127,7 +128,7 @@ AddDomain() {
             echo -e "  ${INFO} ${1} already exists in ${requestedListname}, no need to add!"
         fi
       else
-        existingListname="$(getListnameFromTypeId "${existingTypeId}")"
+        existingListname="$(GetListnameFromTypeId "${existingTypeId}")"
         sqlite3 "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';"
         if [[ "${verbose}" == true ]]; then
             echo -e "  ${INFO} ${1} already exists in ${existingListname}, it has been moved to ${requestedListname}!"
@@ -153,7 +154,7 @@ RemoveDomain() {
     # Is the domain in the list we want to remove it from?
     num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")"
 
-    requestedListname="$(getListnameFromTypeId "${typeId}")"
+    requestedListname="$(GetListnameFromTypeId "${typeId}")"
 
     if [[ "${num}" -eq 0 ]]; then
       if [[ "${verbose}" == true ]]; then
@@ -174,7 +175,7 @@ RemoveDomain() {
 Displaylist() {
     local count num_pipes domain enabled status nicedate requestedListname
 
-    requestedListname="$(getListnameFromTypeId "${typeId}")"
+    requestedListname="$(GetListnameFromTypeId "${typeId}")"
     data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)"
 
     if [[ -z $data ]]; then
@@ -231,7 +232,7 @@ for var in "$@"; do
         "-l" | "--list"      ) Displaylist;;
         "--nuke"             ) NukeList;;
         "--web"              ) web=true;;
-        *                    ) HandleOther "${var}";;
+        *                    ) ValidateDomain "${var}";;
     esac
 done
 

From 76460f01e9c70905b7ac82d9a294855c6bfd19bc Mon Sep 17 00:00:00 2001
From: Adam Warner <me@adamwarner.co.uk>
Date: Sat, 30 Nov 2019 17:45:07 +0000
Subject: [PATCH 08/13] Change the regex used for domain validation

Signed-off-by: Adam Warner <me@adamwarner.co.uk>
---
 advanced/Scripts/list.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 5fbe831f..50541872 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -84,8 +84,8 @@ ValidateDomain() {
         if [[ ( "${typeId}" == "3" || "${typeId}" == "2" ) && "${wildcard}" == false ]]; then
             validDomain="${domain}"
         else
-            validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check
-            validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label
+            # Use regex to check the validity of the passed domain. see https://regexr.com/3abjr
+            validDomain=$(grep -P "^((?!-))(xn--)?[a-z0-9][a-z0-9-_]{0,61}[a-z0-9]{0,1}\.(xn--)?([a-z0-9\-]{1,61}|[a-z0-9-]{1,30}\.[a-z]{2,})$" <<< "${domain}")
         fi
     fi
 

From 44e1455b12e0dafba4ba9cfca7708bdb9aa2d188 Mon Sep 17 00:00:00 2001
From: Adam Warner <me@adamwarner.co.uk>
Date: Sun, 1 Dec 2019 12:44:48 +0000
Subject: [PATCH 09/13] Update advanced/Scripts/list.sh

Co-Authored-By: DL6ER <DL6ER@users.noreply.github.com>
---
 advanced/Scripts/list.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 50541872..40fabd54 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -46,7 +46,7 @@ GetListParamFromTypeId() {
     elif  [[ "${typeId}" == "2" && "${wildcard}" == true ]]; then
         echo "-white-wild"
     elif  [[ "${typeId}" == "2" ]]; then
-        echo "regex_blacklist"
+        echo "-white-regex"
     elif  [[ "${typeId}" == "3" && "${wildcard}" == true ]]; then
         echo "-regex"
     elif  [[ "${typeId}" == "3" ]]; then

From 0251117c779f9f3ebb2056862e1009b8a0e88e28 Mon Sep 17 00:00:00 2001
From: Adam Warner <me@adamwarner.co.uk>
Date: Sun, 1 Dec 2019 12:45:06 +0000
Subject: [PATCH 10/13] Update advanced/Scripts/list.sh

Co-Authored-By: DL6ER <DL6ER@users.noreply.github.com>
---
 advanced/Scripts/list.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 40fabd54..064181f0 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -50,7 +50,7 @@ GetListParamFromTypeId() {
     elif  [[ "${typeId}" == "3" && "${wildcard}" == true ]]; then
         echo "-regex"
     elif  [[ "${typeId}" == "3" ]]; then
-        echo "-wild"
+        echo "-regex"
     fi
 }
 

From 63e407cfdc9f7fd1ffddd6c49b3bf6b2dfd63c63 Mon Sep 17 00:00:00 2001
From: Adam Warner <me@adamwarner.co.uk>
Date: Sun, 1 Dec 2019 12:45:22 +0000
Subject: [PATCH 11/13] Update advanced/Scripts/list.sh

Co-Authored-By: DL6ER <DL6ER@users.noreply.github.com>
---
 advanced/Scripts/list.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 064181f0..a5c84661 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -48,7 +48,7 @@ GetListParamFromTypeId() {
     elif  [[ "${typeId}" == "2" ]]; then
         echo "-white-regex"
     elif  [[ "${typeId}" == "3" && "${wildcard}" == true ]]; then
-        echo "-regex"
+        echo "-wild"
     elif  [[ "${typeId}" == "3" ]]; then
         echo "-regex"
     fi

From 869473172c2d8620e180aa037fb9cde12cef1fb3 Mon Sep 17 00:00:00 2001
From: Adam Warner <me@adamwarner.co.uk>
Date: Sun, 1 Dec 2019 12:50:24 +0000
Subject: [PATCH 12/13] remove _ from regex descibers

Signed-off-by: Adam Warner <me@adamwarner.co.uk>
---
 advanced/Scripts/list.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index a5c84661..a7d12657 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -32,9 +32,9 @@ GetListnameFromTypeId() {
     elif  [[ "$1" == "1" ]]; then
         echo "blacklist"
     elif  [[ "$1" == "2" ]]; then
-        echo "regex_whitelist"
+        echo "regex whitelist"
     elif  [[ "$1" == "3" ]]; then
-        echo "regex_blacklist"
+        echo "regex blacklist"
     fi
 }
 

From b6cd7b8e3d36a5d7208136a671e65913076aef79 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 2 Dec 2019 17:27:32 +0000
Subject: [PATCH 13/13] Use more descriptive names instead of directly using
 the IDs in list.sh

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/list.sh | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index a7d12657..c5bf5b2a 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -26,30 +26,37 @@ typeId=""
 colfile="/opt/pihole/COL_TABLE"
 source ${colfile}
 
+# IDs are hard-wired to domain interpretation in the gravity database scheme
+# Clients (including FTL) will read them through the corresponding views
+readonly whitelist="0"
+readonly blacklist="1"
+readonly regex_whitelist="2"
+readonly regex_blacklist="3"
+
 GetListnameFromTypeId() {
-    if [[ "$1" == "0" ]]; then
+    if [[ "$1" == "${whitelist}" ]]; then
         echo "whitelist"
-    elif  [[ "$1" == "1" ]]; then
+    elif  [[ "$1" == "${blacklist}" ]]; then
         echo "blacklist"
-    elif  [[ "$1" == "2" ]]; then
+    elif  [[ "$1" == "${regex_whitelist}" ]]; then
         echo "regex whitelist"
-    elif  [[ "$1" == "3" ]]; then
+    elif  [[ "$1" == "${regex_blacklist}" ]]; then
         echo "regex blacklist"
     fi
 }
 
 GetListParamFromTypeId() {
-    if [[ "${typeId}" == "0" ]]; then
+    if [[ "${typeId}" == "${whitelist}" ]]; then
         echo "w"
-    elif  [[ "${typeId}" == "1" ]]; then
+    elif  [[ "${typeId}" == "${blacklist}" ]]; then
         echo "b"
-    elif  [[ "${typeId}" == "2" && "${wildcard}" == true ]]; then
+    elif  [[ "${typeId}" == "${regex_whitelist}" && "${wildcard}" == true ]]; then
         echo "-white-wild"
-    elif  [[ "${typeId}" == "2" ]]; then
+    elif  [[ "${typeId}" == "${regex_whitelist}" ]]; then
         echo "-white-regex"
-    elif  [[ "${typeId}" == "3" && "${wildcard}" == true ]]; then
+    elif  [[ "${typeId}" == "${regex_blacklist}" && "${wildcard}" == true ]]; then
         echo "-wild"
-    elif  [[ "${typeId}" == "3" ]]; then
+    elif  [[ "${typeId}" == "${regex_blacklist}" ]]; then
         echo "-regex"
     fi
 }
@@ -81,7 +88,7 @@ ValidateDomain() {
 
     # Check validity of domain (don't check for regex entries)
     if [[ "${#domain}" -le 253 ]]; then
-        if [[ ( "${typeId}" == "3" || "${typeId}" == "2" ) && "${wildcard}" == false ]]; then
+        if [[ ( "${typeId}" == "${regex_blacklist}" || "${typeId}" == "${regex_whitelist}" ) && "${wildcard}" == false ]]; then
             validDomain="${domain}"
         else
             # Use regex to check the validity of the passed domain. see https://regexr.com/3abjr