Add antigravity support to gravity (#5330)

advanced/Scripts/database_migration/gravity-db.sh

@@ -134,4 +134,11 @@ upgrade_gravityDB(){
 		pihole-FTL sqlite3 "${database}" < "${scriptPath}/15_to_16.sql"
 		version=16
 	fi
+	if [[ "$version" == "16" ]]; then
+		# Add antigravity table
+		# Add column type to adlist table (to support adlist types)
+		echo -e "  ${INFO} Upgrading gravity database from version 16 to 17"
+		pihole-FTL sqlite3 "${database}" < "${scriptPath}/16_to_17.sql"
+		version=17
+	fi
 }

advanced/Scripts/database_migration/gravity/16_to_17.sql

@@ -0,0 +1,27 @@
+.timeout 30000
+
+PRAGMA FOREIGN_KEYS=OFF;
+
+BEGIN TRANSACTION;
+
+ALTER TABLE adlist ADD COLUMN type INTEGER NOT NULL DEFAULT 0;
+
+UPDATE adlist SET type = 0;
+
+CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id
+    FROM antigravity
+    LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id
+    LEFT JOIN adlist ON adlist.id = antigravity.adlist_id
+    LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
+    WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1;
+
+DROP VIEW vw_adlist;
+
+CREATE VIEW vw_adlist AS SELECT DISTINCT address, id, type
+    FROM adlist
+    WHERE enabled = 1
+    ORDER BY id;
+
+UPDATE info SET value = 17 WHERE property = 'version';
+
+COMMIT;

advanced/Templates/gravity.db.sql

@@ -36,7 +36,8 @@ CREATE TABLE adlist
 	number INTEGER NOT NULL DEFAULT 0,
 	invalid_domains INTEGER NOT NULL DEFAULT 0,
 	status INTEGER NOT NULL DEFAULT 0,
-	abp_entries INTEGER NOT NULL DEFAULT 0
+	abp_entries INTEGER NOT NULL DEFAULT 0,
+	type INTEGER NOT NULL DEFAULT 0
 );
 
 CREATE TABLE adlist_by_group
@@ -52,13 +53,19 @@ CREATE TABLE gravity
 	adlist_id INTEGER NOT NULL REFERENCES adlist (id)
 );
 
+CREATE TABLE antigravity
+(
+	domain TEXT NOT NULL,
+	adlist_id INTEGER NOT NULL REFERENCES adlist (id)
+);
+
 CREATE TABLE info
 (
 	property TEXT PRIMARY KEY,
 	value TEXT NOT NULL
 );
 
-INSERT INTO "info" VALUES('version','16');
+INSERT INTO "info" VALUES('version','17');
 
 CREATE TABLE domain_audit
 (
@@ -144,7 +151,14 @@ CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id
     LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
     WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1);
 
-CREATE VIEW vw_adlist AS SELECT DISTINCT address, id
+CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id
+    FROM antigravity
+    LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id
+    LEFT JOIN adlist ON adlist.id = antigravity.adlist_id
+    LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
+    WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1;
+
+CREATE VIEW vw_adlist AS SELECT DISTINCT address, id, type
     FROM adlist
     WHERE enabled = 1
     ORDER BY id;

gravity.sh

@@ -178,7 +178,7 @@ database_table_from_file() {
         echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}"
       elif [[ "${table}" == "adlist" ]]; then
         # Adlist table format
-        echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0,0" >> "${tmpFile}"
+        echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0,0,0" >> "${tmpFile}"
       else
         # White-, black-, and regexlist table format
         echo "${rowid},${list_type},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\"" >> "${tmpFile}"
@@ -361,6 +361,7 @@ gravity_DownloadBlocklists() {
   # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true)
   mapfile -t sources <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)"
   mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)"
+  mapfile -t sourceTypes <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2> /dev/null)"
 
   # Parse source domains from $sources
   mapfile -t sourceDomains <<< "$(
@@ -382,7 +383,7 @@ gravity_DownloadBlocklists() {
     unset sources
   fi
 
-  local url domain str target compression
+  local url domain str target compression adlist_type
   echo ""
 
   # Prepare new gravity database
@@ -433,6 +434,15 @@ gravity_DownloadBlocklists() {
     url="${sources[$i]}"
     domain="${sourceDomains[$i]}"
     id="${sourceIDs[$i]}"
+    if [[ "${sourceTypes[$i]}" -eq "0" ]]; then
+      # Gravity list
+      str="blocklist"
+      adlist_type="gravity"
+    else
+      # AntiGravity list
+      str="allowlist"
+      adlist_type="antigravity"
+    fi
 
     # Save the file as list.#.domain
     saveLocation="${piholeDir}/list.${id}.${domain}.${domainsExtension}"
@@ -450,7 +460,7 @@ gravity_DownloadBlocklists() {
     if [[ "${check_url}" =~ ${regex} ]]; then
       echo -e "  ${CROSS} Invalid Target"
     else
-      gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}"
+      gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" "${adlist_type}"
     fi
     echo ""
   done
@@ -482,7 +492,7 @@ compareLists() {
 
 # Download specified URL and perform checks on HTTP status and file content
 gravity_DownloadBlocklistFromUrl() {
-  local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}"
+  local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" gravity_type="${6}"
   local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext
 
   # Create temp file to store content on disk instead of RAM
@@ -576,7 +586,7 @@ gravity_DownloadBlocklistFromUrl() {
   if [[ "${success}" == true ]]; then
     if [[ "${httpCode}" == "304" ]]; then
       # Add domains to database table file
-      pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
+      pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
       database_adlist_status "${adlistID}" "2"
       done="true"
     # Check if $listCurlBuffer is a non-zero length file
@@ -586,7 +596,7 @@ gravity_DownloadBlocklistFromUrl() {
       # Remove curl buffer file after its use
       rm "${listCurlBuffer}"
       # Add domains to database table file
-      pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
+      pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
       # Compare lists, are they identical?
       compareLists "${adlistID}" "${saveLocation}"
       done="true"
@@ -602,7 +612,7 @@ gravity_DownloadBlocklistFromUrl() {
     if [[ -r "${saveLocation}" ]]; then
       echo -e "  ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
       # Add domains to database table file
-      pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
+      pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
       database_adlist_status "${adlistID}" "3"
     else
       echo -e "  ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}"