arno/pi-hole
1
0
Fork 0
mirror of https://github.com/pi-hole/pi-hole synced 2024-12-22 14:58:08 +00:00
Code Issues Releases Wiki Activity

Merge pull request #2803 from pi-hole/new/gravity.db_domain_groups

Browse Source 
Implement groups for lists
This commit is contained in:
Mark Drobnak 2019-07-04 13:52:52 -04:00 committed by GitHub
commit ada8b53423
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 83 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
advanced/Scripts/piholeDebug.sh
View File

@ -1077,7 +1077,7 @@ show_db_entries() {
} }
show_adlists() { show_adlists() {
show_db_entries "Adlists" "SELECT * FROM adlists" "4 100 7 10 13 50" show_db_entries "Adlists" "SELECT * FROM adlist" "4 100 7 10 13 50"
} }
show_whitelist() { show_whitelist() {

2
advanced/Scripts/query.sh
View File

@ -201,7 +201,7 @@ fi
# Get adlist file content as array # Get adlist file content as array
if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then
# Retrieve source URLs from gravity database # Retrieve source URLs from gravity database
mapfile -t adlists <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)" mapfile -t adlists <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)"
fi fi
# Print "Exact matches for" title # Print "Exact matches for" title

8
advanced/Scripts/webpage.sh
View File

@ -400,13 +400,13 @@ CustomizeAdLists() {
address="${args[3]}" address="${args[3]}"
if [[ "${args[2]}" == "enable" ]]; then if [[ "${args[2]}" == "enable" ]]; then
sqlite3 "${gravityDBfile}" "UPDATE adlists SET enabled = 1 WHERE address = '${address}'" sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'"
elif [[ "${args[2]}" == "disable" ]]; then elif [[ "${args[2]}" == "disable" ]]; then
sqlite3 "${gravityDBfile}" "UPDATE adlists SET enabled = 0 WHERE address = '${address}'" sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'"
elif [[ "${args[2]}" == "add" ]]; then elif [[ "${args[2]}" == "add" ]]; then
sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlists (address) VALUES ('${address}')" sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address) VALUES ('${address}')"
elif [[ "${args[2]}" == "del" ]]; then elif [[ "${args[2]}" == "del" ]]; then
sqlite3 "${gravityDBfile}" "DELETE FROM adlists WHERE address = '${address}'" sqlite3 "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'"
else else
echo "Not permitted" echo "Not permitted"
return 1 return 1

94
advanced/Templates/gravity.db.sql
View File

@ -1,3 +1,13 @@
PRAGMA FOREIGN_KEYS=ON;
CREATE TABLE "group"
(
id INTEGER PRIMARY KEY AUTOINCREMENT,
enabled BOOLEAN NOT NULL DEFAULT 1,
name TEXT NOT NULL,
description TEXT
);
CREATE TABLE whitelist CREATE TABLE whitelist
( (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
@ -7,6 +17,14 @@ CREATE TABLE whitelist
date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
comment TEXT comment TEXT
); );
CREATE TABLE whitelist_by_group
(
whitelist_id INTEGER NOT NULL REFERENCES whitelist (id),
group_id INTEGER NOT NULL REFERENCES "group" (id),
PRIMARY KEY (whitelist_id, group_id)
);
CREATE TABLE blacklist CREATE TABLE blacklist
( (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
@ -16,6 +34,14 @@ CREATE TABLE blacklist
date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
comment TEXT comment TEXT
); );
CREATE TABLE blacklist_by_group
(
blacklist_id INTEGER NOT NULL REFERENCES blacklist (id),
group_id INTEGER NOT NULL REFERENCES "group" (id),
PRIMARY KEY (blacklist_id, group_id)
);
CREATE TABLE regex CREATE TABLE regex
( (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
@ -25,7 +51,15 @@ CREATE TABLE regex
date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
comment TEXT comment TEXT
); );
CREATE TABLE adlists
CREATE TABLE regex_by_group
(
regex_id INTEGER NOT NULL REFERENCES regex (id),
group_id INTEGER NOT NULL REFERENCES "group" (id),
PRIMARY KEY (regex_id, group_id)
);
CREATE TABLE adlist
( (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
address TEXT UNIQUE NOT NULL, address TEXT UNIQUE NOT NULL,
@ -34,6 +68,14 @@ CREATE TABLE adlists
date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
comment TEXT comment TEXT
); );
CREATE TABLE adlist_by_group
(
adlist_id INTEGER NOT NULL REFERENCES adlist (id),
group_id INTEGER NOT NULL REFERENCES "group" (id),
PRIMARY KEY (adlist_id, group_id)
);
CREATE TABLE gravity CREATE TABLE gravity
( (
domain TEXT PRIMARY KEY domain TEXT PRIMARY KEY
@ -46,47 +88,55 @@ CREATE TABLE info
INSERT INTO info VALUES("version","1"); INSERT INTO info VALUES("version","1");
CREATE VIEW vw_gravity AS SELECT a.domain CREATE VIEW vw_gravity AS SELECT domain
FROM gravity a FROM gravity
WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1); WHERE domain NOT IN (SELECT domain from vw_whitelist);
CREATE VIEW vw_whitelist AS SELECT a.domain CREATE VIEW vw_whitelist AS SELECT DISTINCT domain
FROM whitelist a FROM whitelist
WHERE a.enabled == 1 LEFT JOIN whitelist_by_group ON whitelist_by_group.whitelist_id = whitelist.id
ORDER BY a.id; LEFT JOIN "group" ON "group".id = whitelist_by_group.group_id
WHERE whitelist.enabled = 1 AND (whitelist_by_group.group_id IS NULL OR "group".enabled = 1)
ORDER BY whitelist.id;
CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist
BEGIN BEGIN
UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
END; END;
CREATE VIEW vw_blacklist AS SELECT a.domain CREATE VIEW vw_blacklist AS SELECT DISTINCT domain
FROM blacklist a FROM blacklist
WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist LEFT JOIN blacklist_by_group ON blacklist_by_group.blacklist_id = blacklist.id
ORDER BY a.id; LEFT JOIN "group" ON "group".id = blacklist_by_group.group_id
WHERE blacklist.enabled = 1 AND (blacklist_by_group.group_id IS NULL OR "group".enabled = 1)
ORDER BY blacklist.id;
CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist
BEGIN BEGIN
UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
END; END;
CREATE VIEW vw_regex AS SELECT a.domain CREATE VIEW vw_regex AS SELECT DISTINCT domain
FROM regex a FROM regex
WHERE a.enabled == 1 LEFT JOIN regex_by_group ON regex_by_group.regex_id = regex.id
ORDER BY a.id; LEFT JOIN "group" ON "group".id = regex_by_group.group_id
WHERE regex.enabled = 1 AND (regex_by_group.group_id IS NULL OR "group".enabled = 1)
ORDER BY regex.id;
CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex
BEGIN BEGIN
UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
END; END;
CREATE VIEW vw_adlists AS SELECT a.address CREATE VIEW vw_adlist AS SELECT DISTINCT address
FROM adlists a FROM adlist
WHERE a.enabled == 1 LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = adlist.id
ORDER BY a.id; LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1)
ORDER BY adlist.id;
CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists CREATE TRIGGER tr_adlist_update AFTER UPDATE ON adlist
BEGIN BEGIN
UPDATE adlists SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address; UPDATE adlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address;
END; END;

10
gravity.sh
View File

@ -111,7 +111,7 @@ database_table_from_file() {
# No need to modify the input data for the gravity table # No need to modify the input data for the gravity table
inputfile="${source}" inputfile="${source}"
else else
# Apply format for white-, blacklist, regex, and adlists tables # Apply format for white-, blacklist, regex, and adlist tables
local rowid local rowid
declare -i rowid declare -i rowid
rowid=1 rowid=1
@ -159,9 +159,9 @@ migrate_to_database() {
# Migrate list files to new database # Migrate list files to new database
if [[ -e "${adListFile}" ]]; then if [[ -e "${adListFile}" ]]; then
# Store adlists domains in database # Store adlist domains in database
echo -e " ${INFO} Migrating content of ${adListFile} into new database" echo -e " ${INFO} Migrating content of ${adListFile} into new database"
database_table_from_file "adlists" "${adListFile}" database_table_from_file "adlist" "${adListFile}"
fi fi
if [[ -e "${blacklistFile}" ]]; then if [[ -e "${blacklistFile}" ]]; then
# Store blacklisted domains in database # Store blacklisted domains in database
@ -236,13 +236,13 @@ gravity_CheckDNSResolutionAvailable() {
gravity_CheckDNSResolutionAvailable gravity_CheckDNSResolutionAvailable
} }
# Retrieve blocklist URLs and parse domains from adlists.list # Retrieve blocklist URLs and parse domains from adlist.list
gravity_GetBlocklistUrls() { gravity_GetBlocklistUrls() {
echo -e " ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..." echo -e " ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..."
# Retrieve source URLs from gravity database # Retrieve source URLs from gravity database
# We source only enabled adlists, sqlite3 stores boolean values as 0 (false) or 1 (true) # We source only enabled adlists, sqlite3 stores boolean values as 0 (false) or 1 (true)
mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)" mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)"
# Parse source domains from $sources # Parse source domains from $sources
mapfile -t sourceDomains <<< "$( mapfile -t sourceDomains <<< "$(