diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index b31bbdc5..2bfeef2d 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1077,7 +1077,7 @@ show_db_entries() { } show_adlists() { - show_db_entries "Adlists" "SELECT * FROM adlists" "4 100 7 10 13 50" + show_db_entries "Adlists" "SELECT * FROM adlist" "4 100 7 10 13 50" } show_whitelist() { diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index b7edaccf..4fc82744 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -201,7 +201,7 @@ fi # Get adlist file content as array if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then # Retrieve source URLs from gravity database - mapfile -t adlists <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)" + mapfile -t adlists <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" fi # Print "Exact matches for" title diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 583579b6..f744d1b9 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -400,13 +400,13 @@ CustomizeAdLists() { address="${args[3]}" if [[ "${args[2]}" == "enable" ]]; then - sqlite3 "${gravityDBfile}" "UPDATE adlists SET enabled = 1 WHERE address = '${address}'" + sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'" elif [[ "${args[2]}" == "disable" ]]; then - sqlite3 "${gravityDBfile}" "UPDATE adlists SET enabled = 0 WHERE address = '${address}'" + sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'" elif [[ "${args[2]}" == "add" ]]; then - sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlists (address) VALUES ('${address}')" + sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address) VALUES ('${address}')" elif [[ "${args[2]}" == "del" ]]; then - sqlite3 "${gravityDBfile}" "DELETE FROM adlists WHERE address = '${address}'" + sqlite3 "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'" else echo "Not permitted" return 1 diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 372a4a29..91a38ef7 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,3 +1,13 @@ +PRAGMA FOREIGN_KEYS=ON; + +CREATE TABLE "group" +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + enabled BOOLEAN NOT NULL DEFAULT 1, + name TEXT NOT NULL, + description TEXT +); + CREATE TABLE whitelist ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -7,6 +17,14 @@ CREATE TABLE whitelist date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT ); + +CREATE TABLE whitelist_by_group +( + whitelist_id INTEGER NOT NULL REFERENCES whitelist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (whitelist_id, group_id) +); + CREATE TABLE blacklist ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -16,6 +34,14 @@ CREATE TABLE blacklist date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT ); + +CREATE TABLE blacklist_by_group +( + blacklist_id INTEGER NOT NULL REFERENCES blacklist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (blacklist_id, group_id) +); + CREATE TABLE regex ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -25,7 +51,15 @@ CREATE TABLE regex date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT ); -CREATE TABLE adlists + +CREATE TABLE regex_by_group +( + regex_id INTEGER NOT NULL REFERENCES regex (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (regex_id, group_id) +); + +CREATE TABLE adlist ( id INTEGER PRIMARY KEY AUTOINCREMENT, address TEXT UNIQUE NOT NULL, @@ -34,6 +68,14 @@ CREATE TABLE adlists date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT ); + +CREATE TABLE adlist_by_group +( + adlist_id INTEGER NOT NULL REFERENCES adlist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (adlist_id, group_id) +); + CREATE TABLE gravity ( domain TEXT PRIMARY KEY @@ -46,47 +88,55 @@ CREATE TABLE info INSERT INTO info VALUES("version","1"); -CREATE VIEW vw_gravity AS SELECT a.domain - FROM gravity a - WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1); +CREATE VIEW vw_gravity AS SELECT domain + FROM gravity + WHERE domain NOT IN (SELECT domain from vw_whitelist); -CREATE VIEW vw_whitelist AS SELECT a.domain - FROM whitelist a - WHERE a.enabled == 1 - ORDER BY a.id; +CREATE VIEW vw_whitelist AS SELECT DISTINCT domain + FROM whitelist + LEFT JOIN whitelist_by_group ON whitelist_by_group.whitelist_id = whitelist.id + LEFT JOIN "group" ON "group".id = whitelist_by_group.group_id + WHERE whitelist.enabled = 1 AND (whitelist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY whitelist.id; CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist BEGIN UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; END; -CREATE VIEW vw_blacklist AS SELECT a.domain - FROM blacklist a - WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist - ORDER BY a.id; +CREATE VIEW vw_blacklist AS SELECT DISTINCT domain + FROM blacklist + LEFT JOIN blacklist_by_group ON blacklist_by_group.blacklist_id = blacklist.id + LEFT JOIN "group" ON "group".id = blacklist_by_group.group_id + WHERE blacklist.enabled = 1 AND (blacklist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY blacklist.id; CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist BEGIN UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; END; -CREATE VIEW vw_regex AS SELECT a.domain - FROM regex a - WHERE a.enabled == 1 - ORDER BY a.id; +CREATE VIEW vw_regex AS SELECT DISTINCT domain + FROM regex + LEFT JOIN regex_by_group ON regex_by_group.regex_id = regex.id + LEFT JOIN "group" ON "group".id = regex_by_group.group_id + WHERE regex.enabled = 1 AND (regex_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY regex.id; CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex BEGIN UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; END; -CREATE VIEW vw_adlists AS SELECT a.address - FROM adlists a - WHERE a.enabled == 1 - ORDER BY a.id; +CREATE VIEW vw_adlist AS SELECT DISTINCT address + FROM adlist + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = adlist.id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY adlist.id; -CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists +CREATE TRIGGER tr_adlist_update AFTER UPDATE ON adlist BEGIN - UPDATE adlists SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address; + UPDATE adlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address; END; diff --git a/gravity.sh b/gravity.sh index 1c142ffd..1a325ba7 100755 --- a/gravity.sh +++ b/gravity.sh @@ -111,7 +111,7 @@ database_table_from_file() { # No need to modify the input data for the gravity table inputfile="${source}" else - # Apply format for white-, blacklist, regex, and adlists tables + # Apply format for white-, blacklist, regex, and adlist tables local rowid declare -i rowid rowid=1 @@ -159,9 +159,9 @@ migrate_to_database() { # Migrate list files to new database if [[ -e "${adListFile}" ]]; then - # Store adlists domains in database + # Store adlist domains in database echo -e " ${INFO} Migrating content of ${adListFile} into new database" - database_table_from_file "adlists" "${adListFile}" + database_table_from_file "adlist" "${adListFile}" fi if [[ -e "${blacklistFile}" ]]; then # Store blacklisted domains in database @@ -236,13 +236,13 @@ gravity_CheckDNSResolutionAvailable() { gravity_CheckDNSResolutionAvailable } -# Retrieve blocklist URLs and parse domains from adlists.list +# Retrieve blocklist URLs and parse domains from adlist.list gravity_GetBlocklistUrls() { echo -e " ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..." # Retrieve source URLs from gravity database # We source only enabled adlists, sqlite3 stores boolean values as 0 (false) or 1 (true) - mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)" + mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" # Parse source domains from $sources mapfile -t sourceDomains <<< "$(