From 10c2dad48ad2e600d016004166ab5d88bec16424 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 24 Jan 2020 18:39:13 +0100 Subject: [PATCH] Improve gravity performance (#3100) * Gravity performance improvements. Signed-off-by: DL6ER * Do not move downloaded lists into migration_backup directory. Signed-off-by: DL6ER * Do not (strictly) sort domains. Random-leaf access is faster than always-last-leaf access (on average). Signed-off-by: DL6ER * Append instead of overwrite gravity_new collection list. Signed-off-by: DL6ER * Rename table gravity_new to gravity_temp to clarify that this is only an intermediate table. Signed-off-by: DL6ER * Add timers for each of the calls to compute intense parts. They are to be removed before this finally hits the release/v5.0 branch. Signed-off-by: DL6ER * Fix legacy list files import. It currently doesn't work when the gravity database has already been updated to using the single domainlist table. Signed-off-by: DL6ER * Simplify database_table_from_file(), remove all to this function for gravity lost downloads. Signed-off-by: DL6ER * Update gravity.db.sql to version 10 to have newle created databases already reflect the most recent state. Signed-off-by: DL6ER * Create second gravity database and swap them on success. This has a number of advantages such as instantaneous gravity updates (as seen from FTL) and always available gravity blocking. Furthermore, this saves disk space as the old database is removed on completion. * Add timing output for the database swapping SQLite3 call. Signed-off-by: DL6ER * Explicitly generate index as a separate process. Signed-off-by: DL6ER * Remove time measurements. Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 190 ++++++++++++++---------- advanced/Templates/gravity_copy.sql | 21 +++ gravity.sh | 219 +++++++++++++++++----------- 3 files changed, 272 insertions(+), 158 deletions(-) create mode 100644 advanced/Templates/gravity_copy.sql diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index d0c744f4..a7dc12df 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,16 +1,21 @@ -PRAGMA FOREIGN_KEYS=ON; +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; CREATE TABLE "group" ( id INTEGER PRIMARY KEY AUTOINCREMENT, enabled BOOLEAN NOT NULL DEFAULT 1, - name TEXT NOT NULL, + name TEXT UNIQUE NOT NULL, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), description TEXT ); +INSERT INTO "group" (id,enabled,name) VALUES (0,1,'Unassociated'); -CREATE TABLE whitelist +CREATE TABLE domainlist ( id INTEGER PRIMARY KEY AUTOINCREMENT, + type INTEGER NOT NULL DEFAULT 0, domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), @@ -18,125 +23,158 @@ CREATE TABLE whitelist comment TEXT ); -CREATE TABLE whitelist_by_group -( - whitelist_id INTEGER NOT NULL REFERENCES whitelist (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (whitelist_id, group_id) -); - -CREATE TABLE blacklist +CREATE TABLE adlist ( id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, + address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT ); -CREATE TABLE blacklist_by_group +CREATE TABLE adlist_by_group ( - blacklist_id INTEGER NOT NULL REFERENCES blacklist (id), + adlist_id INTEGER NOT NULL REFERENCES adlist (id), group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (blacklist_id, group_id) + PRIMARY KEY (adlist_id, group_id) ); -CREATE TABLE regex +CREATE TABLE gravity ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT + domain TEXT NOT NULL, + adlist_id INTEGER NOT NULL REFERENCES adlist (id) ); -CREATE TABLE regex_by_group +CREATE TABLE info ( - regex_id INTEGER NOT NULL REFERENCES regex (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (regex_id, group_id) + property TEXT PRIMARY KEY, + value TEXT NOT NULL ); -CREATE TABLE adlist +INSERT INTO "info" VALUES('version','10'); + +CREATE TABLE domain_audit ( id INTEGER PRIMARY KEY AUTOINCREMENT, - address TEXT UNIQUE NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT + domain TEXT UNIQUE NOT NULL, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) ); -CREATE TABLE adlist_by_group +CREATE TABLE domainlist_by_group ( - adlist_id INTEGER NOT NULL REFERENCES adlist (id), + domainlist_id INTEGER NOT NULL REFERENCES domainlist (id), group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (adlist_id, group_id) + PRIMARY KEY (domainlist_id, group_id) ); -CREATE TABLE gravity +CREATE TABLE client ( - domain TEXT PRIMARY KEY + id INTEGER PRIMARY KEY AUTOINCREMENT, + ip TEXT NOL NULL UNIQUE ); -CREATE TABLE info +CREATE TABLE client_by_group ( - property TEXT PRIMARY KEY, - value TEXT NOT NULL + client_id INTEGER NOT NULL REFERENCES client (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (client_id, group_id) ); -INSERT INTO info VALUES("version","1"); +CREATE TRIGGER tr_adlist_update AFTER UPDATE ON adlist + BEGIN + UPDATE adlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address; + END; + +CREATE TRIGGER tr_domainlist_update AFTER UPDATE ON domainlist + BEGIN + UPDATE domainlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; + +CREATE VIEW vw_whitelist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id + FROM domainlist + LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id + LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id + WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1) + AND domainlist.type = 0 + ORDER BY domainlist.id; + +CREATE VIEW vw_blacklist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id + FROM domainlist + LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id + LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id + WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1) + AND domainlist.type = 1 + ORDER BY domainlist.id; + +CREATE VIEW vw_regex_whitelist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id + FROM domainlist + LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id + LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id + WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1) + AND domainlist.type = 2 + ORDER BY domainlist.id; + +CREATE VIEW vw_regex_blacklist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id + FROM domainlist + LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id + LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id + WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1) + AND domainlist.type = 3 + ORDER BY domainlist.id; + +CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id + FROM gravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = gravity.adlist_id + LEFT JOIN adlist ON adlist.id = gravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1); + +CREATE VIEW vw_adlist AS SELECT DISTINCT address, adlist.id AS id + FROM adlist + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = adlist.id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY adlist.id; -CREATE VIEW vw_whitelist AS SELECT DISTINCT domain - FROM whitelist - LEFT JOIN whitelist_by_group ON whitelist_by_group.whitelist_id = whitelist.id - LEFT JOIN "group" ON "group".id = whitelist_by_group.group_id - WHERE whitelist.enabled = 1 AND (whitelist_by_group.group_id IS NULL OR "group".enabled = 1) - ORDER BY whitelist.id; +CREATE TRIGGER tr_domainlist_add AFTER INSERT ON domainlist + BEGIN + INSERT INTO domainlist_by_group (domainlist_id, group_id) VALUES (NEW.id, 0); + END; -CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist +CREATE TRIGGER tr_client_add AFTER INSERT ON client BEGIN - UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + INSERT INTO client_by_group (client_id, group_id) VALUES (NEW.id, 0); END; -CREATE VIEW vw_blacklist AS SELECT DISTINCT domain - FROM blacklist - LEFT JOIN blacklist_by_group ON blacklist_by_group.blacklist_id = blacklist.id - LEFT JOIN "group" ON "group".id = blacklist_by_group.group_id - WHERE blacklist.enabled = 1 AND (blacklist_by_group.group_id IS NULL OR "group".enabled = 1) - ORDER BY blacklist.id; +CREATE TRIGGER tr_adlist_add AFTER INSERT ON adlist + BEGIN + INSERT INTO adlist_by_group (adlist_id, group_id) VALUES (NEW.id, 0); + END; -CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist +CREATE TRIGGER tr_group_update AFTER UPDATE ON "group" BEGIN - UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + UPDATE "group" SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE id = NEW.id; END; -CREATE VIEW vw_regex AS SELECT DISTINCT domain - FROM regex - LEFT JOIN regex_by_group ON regex_by_group.regex_id = regex.id - LEFT JOIN "group" ON "group".id = regex_by_group.group_id - WHERE regex.enabled = 1 AND (regex_by_group.group_id IS NULL OR "group".enabled = 1) - ORDER BY regex.id; +CREATE TRIGGER tr_group_zero AFTER DELETE ON "group" + BEGIN + INSERT OR IGNORE INTO "group" (id,enabled,name) VALUES (0,1,'Unassociated'); + END; -CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex +CREATE TRIGGER tr_domainlist_delete AFTER DELETE ON domainlist BEGIN - UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + DELETE FROM domainlist_by_group WHERE domainlist_id = OLD.id; END; -CREATE VIEW vw_adlist AS SELECT DISTINCT address - FROM adlist - LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = adlist.id - LEFT JOIN "group" ON "group".id = adlist_by_group.group_id - WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) - ORDER BY adlist.id; +CREATE TRIGGER tr_adlist_delete AFTER DELETE ON adlist + BEGIN + DELETE FROM adlist_by_group WHERE adlist_id = OLD.id; + END; -CREATE TRIGGER tr_adlist_update AFTER UPDATE ON adlist +CREATE TRIGGER tr_client_delete AFTER DELETE ON client BEGIN - UPDATE adlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address; + DELETE FROM client_by_group WHERE client_id = OLD.id; END; -CREATE VIEW vw_gravity AS SELECT domain - FROM gravity - WHERE domain NOT IN (SELECT domain from vw_whitelist); +COMMIT; diff --git a/advanced/Templates/gravity_copy.sql b/advanced/Templates/gravity_copy.sql new file mode 100644 index 00000000..e14d9d8c --- /dev/null +++ b/advanced/Templates/gravity_copy.sql @@ -0,0 +1,21 @@ +.timeout 30000 + +ATTACH DATABASE '/etc/pihole/gravity.db' AS OLD; + +BEGIN TRANSACTION; + +INSERT OR REPLACE INTO "group" SELECT * FROM OLD."group"; +INSERT OR REPLACE INTO domain_audit SELECT * FROM OLD.domain_audit; + +INSERT OR REPLACE INTO domainlist SELECT * FROM OLD.domainlist; +INSERT OR REPLACE INTO domainlist_by_group SELECT * FROM OLD.domainlist_by_group; + +INSERT OR REPLACE INTO adlist SELECT * FROM OLD.adlist; +INSERT OR REPLACE INTO adlist_by_group SELECT * FROM OLD.adlist_by_group; + +INSERT OR REPLACE INTO info SELECT * FROM OLD.info; + +INSERT OR REPLACE INTO client SELECT * FROM OLD.client; +INSERT OR REPLACE INTO client_by_group SELECT * FROM OLD.client_by_group; + +COMMIT; diff --git a/gravity.sh b/gravity.sh index 105febb7..26bedae7 100755 --- a/gravity.sh +++ b/gravity.sh @@ -36,7 +36,9 @@ VPNList="/etc/openvpn/ipp.txt" piholeGitDir="/etc/.pihole" gravityDBfile="${piholeDir}/gravity.db" +gravityTEMPfile="${piholeDir}/gravity_temp.db" gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql" +gravityDBcopy="${piholeGitDir}/advanced/Templates/gravity_copy.sql" optimize_database=false domainsExtension="domains" @@ -80,31 +82,49 @@ fi # Generate new sqlite3 file from schema template generate_gravity_database() { - sqlite3 "${gravityDBfile}" < "${gravityDBschema}" + sqlite3 "${1}" < "${gravityDBschema}" } -update_gravity_timestamp() { - # Update timestamp when the gravity table was last updated successfully - output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | sqlite3 "${gravityDBfile}"; } 2>&1 ) +# Copy data from old to new database file and swap them +gravity_swap_databases() { + local str + str="Building tree" + echo -ne " ${INFO} ${str}..." + + # The index is intentionally not UNIQUE as prro quality adlists may contain domains more than once + output=$( { sqlite3 "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to update gravity timestamp in database ${gravityDBfile}\\n ${output}" + echo -e "\\n ${CROSS} Unable to build gravity tree in ${gravityTEMPfile}\\n ${output}" return 1 fi - return 0 -} + echo -e "${OVER} ${TICK} ${str}" -database_truncate_table() { - local table - table="${1}" + str="Swapping databases" + echo -ne " ${INFO} ${str}..." - output=$( { printf ".timeout 30000\\nDELETE FROM %s;" "${table}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { sqlite3 "${gravityTEMPfile}" < "${gravityDBcopy}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to truncate ${table} database ${gravityDBfile}\\n ${output}" - gravity_Cleanup "error" + echo -e "\\n ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n ${output}" + return 1 + fi + echo -e "${OVER} ${TICK} ${str}" + + # Swap databases and remove old database + rm "${gravityDBfile}" + mv "${gravityTEMPfile}" "${gravityDBfile}" +} + +# Update timestamp when the gravity table was last updated successfully +update_gravity_timestamp() { + output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to update gravity timestamp in database ${gravityTEMPfile}\\n ${output}" return 1 fi return 0 @@ -113,73 +133,80 @@ database_truncate_table() { # Import domains from file and store them in the specified database table database_table_from_file() { # Define locals - local table source backup_path backup_file arg + local table source backup_path backup_file tmpFile type table="${1}" source="${2}" - arg="${3}" backup_path="${piholeDir}/migration_backup" backup_file="${backup_path}/$(basename "${2}")" - - # Truncate table only if not gravity (we add multiple times to this table) - if [[ "${table}" != "gravity" ]]; then - database_truncate_table "${table}" - fi - - local tmpFile tmpFile="$(mktemp -p "/tmp" --suffix=".gravity")" + local timestamp timestamp="$(date --utc +'%s')" - local inputfile - # Apply format for white-, blacklist, regex, and adlist tables - # Read file line by line + local rowid declare -i rowid rowid=1 - if [[ "${table}" == "gravity" ]]; then - #Append ,${arg} to every line and then remove blank lines before import - sed -e "s/$/,${arg}/" "${source}" > "${tmpFile}" - sed -i '/^$/d' "${tmpFile}" - else - grep -v '^ *#' < "${source}" | while IFS= read -r domain - do - # Only add non-empty lines - if [[ -n "${domain}" ]]; then - if [[ "${table}" == "domain_audit" ]]; then - # domain_audit table format (no enable or modified fields) - echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" - else - # White-, black-, and regexlist format - echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" - fi - rowid+=1 - fi - done + # Special handling for domains to be imported into the common domainlist table + if [[ "${table}" == "whitelist" ]]; then + type="0" + table="domainlist" + elif [[ "${table}" == "blacklist" ]]; then + type="1" + table="domainlist" + elif [[ "${table}" == "regex" ]]; then + type="3" + table="domainlist" fi - inputfile="${tmpFile}" - # Remove possible duplicates found in lower-quality adlists - sort -u -o "${inputfile}" "${inputfile}" + # Get MAX(id) from domainlist when INSERTing into this table + if [[ "${table}" == "domainlist" ]]; then + rowid="$(sqlite3 "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")" + if [[ -z "$rowid" ]]; then + rowid=0 + fi + rowid+=1 + fi + + # Loop over all domains in ${source} file + # Read file line by line + grep -v '^ *#' < "${source}" | while IFS= read -r domain + do + # Only add non-empty lines + if [[ -n "${domain}" ]]; then + if [[ "${table}" == "domain_audit" ]]; then + # domain_audit table format (no enable or modified fields) + echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" + elif [[ "${table}" == "adlist" ]]; then + # Adlist table format + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" + else + # White-, black-, and regexlist table format + echo "${rowid},${type},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" + fi + rowid+=1 + fi + done # Store domains in database table specified by ${table} # Use printf as .mode and .import need to be on separate lines # see https://unix.stackexchange.com/a/445615/83260 - output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${inputfile}" "${table}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to fill table ${table} in database ${gravityDBfile}\\n ${output}" + echo -e "\\n ${CROSS} Unable to fill table ${table}${type} in database ${gravityDBfile}\\n ${output}" gravity_Cleanup "error" fi - # Delete tmpfile - rm "${tmpFile}" > /dev/null 2>&1 || \ - echo -e " ${CROSS} Unable to remove ${tmpFile}" - # Move source file to backup directory, create directory if not existing mkdir -p "${backup_path}" mv "${source}" "${backup_file}" 2> /dev/null || \ echo -e " ${CROSS} Unable to backup ${source} to ${backup_path}" + + # Delete tmpFile + rm "${tmpFile}" > /dev/null 2>&1 || \ + echo -e " ${CROSS} Unable to remove ${tmpFile}" } # Migrate pre-v5.0 list files to database-based Pi-hole versions @@ -188,7 +215,10 @@ migrate_to_database() { if [ ! -e "${gravityDBfile}" ]; then # Create new database file - note that this will be created in version 1 echo -e " ${INFO} Creating new gravity database" - generate_gravity_database + generate_gravity_database "${gravityDBfile}" + + # Check if gravity database needs to be updated + upgrade_gravityDB "${gravityDBfile}" "${piholeDir}" # Migrate list files to new database if [ -e "${adListFile}" ]; then @@ -306,16 +336,25 @@ gravity_DownloadBlocklists() { return 1 fi - local url domain agent cmd_ext str + local url domain agent cmd_ext str target echo "" - # Flush gravity table once before looping over sources - str="Flushing gravity table" + # Prepare new gravity database + str="Preparing new gravity database" echo -ne " ${INFO} ${str}..." - if database_truncate_table "gravity"; then + rm "${gravityTEMPfile}" > /dev/null 2>&1 + output=$( { sqlite3 "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to create new database ${gravityTEMPfile}\\n ${output}" + gravity_Cleanup "error" + else echo -e "${OVER} ${TICK} ${str}" fi + target="$(mktemp -p "/tmp" --suffix=".gravity")" + # Loop through $sources and download each one for ((i = 0; i < "${#sources[@]}"; i++)); do url="${sources[$i]}" @@ -335,15 +374,32 @@ gravity_DownloadBlocklists() { esac echo -e " ${INFO} Target: ${url}" - gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}" "${sourceIDs[$i]}" + gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" echo "" done + + str="Storing downloaded domains in new gravity database" + echo -ne " ${INFO} ${str}..." + output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" gravity\\n" "${target}" | sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to fill gravity table in database ${gravityTEMPfile}\\n ${output}" + gravity_Cleanup "error" + else + echo -e "${OVER} ${TICK} ${str}" + fi + + rm "${target}" > /dev/null 2>&1 || \ + echo -e " ${CROSS} Unable to remove ${target}" + gravity_Blackbody=true } # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { - local url="${1}" cmd_ext="${2}" agent="${3}" adlistID="${4}" heisenbergCompensator="" patternBuffer str httpCode success="" + local url="${1}" cmd_ext="${2}" agent="${3}" adlistID="${4}" saveLocation="${5}" target="${6}" + local heisenbergCompensator="" patternBuffer str httpCode success="" # Create temp file to store content on disk instead of RAM patternBuffer=$(mktemp -p "/tmp" --suffix=".phgpb") @@ -424,20 +480,15 @@ gravity_DownloadBlocklistFromUrl() { # Determine if the blocklist was downloaded and saved correctly if [[ "${success}" == true ]]; then if [[ "${httpCode}" == "304" ]]; then - # Add domains to database table - str="Adding adlist with ID ${adlistID} to database table" - echo -ne " ${INFO} ${str}..." - database_table_from_file "gravity" "${saveLocation}" "${adlistID}" - echo -e "${OVER} ${TICK} ${str}" + # Add domains to database table file + #Append ,${arg} to every line and then remove blank lines before import + sed -e "s/$/,${adlistID}/;/^$/d" "${saveLocation}" >> "${target}" # Check if $patternbuffer is a non-zero length file elif [[ -s "${patternBuffer}" ]]; then # Determine if blocklist is non-standard and parse as appropriate gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}" - # Add domains to database table - str="Adding adlist with ID ${adlistID} to database table" - echo -ne " ${INFO} ${str}..." - database_table_from_file "gravity" "${saveLocation}" "${adlistID}" - echo -e "${OVER} ${TICK} ${str}" + #Append ,${arg} to every line and then remove blank lines before import + sed -e "s/$/,${adlistID}/;/^$/d" "${saveLocation}" >> "${target}" else # Fall back to previously cached list if $patternBuffer is empty echo -e " ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" @@ -446,11 +497,8 @@ gravity_DownloadBlocklistFromUrl() { # Determine if cached list has read permission if [[ -r "${saveLocation}" ]]; then echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" - # Add domains to database table - str="Adding to database table" - echo -ne " ${INFO} ${str}..." - database_table_from_file "gravity" "${saveLocation}" "${adlistID}" - echo -e "${OVER} ${TICK} ${str}" + #Append ,${arg} to every line and then remove blank lines before import + sed -e "s/$/,${adlistID}/;/^$/d" "${saveLocation}" >> "${target}" else echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" fi @@ -686,10 +734,6 @@ fi # Move possibly existing legacy files to the gravity database migrate_to_database -# Ensure proper permissions are set for the newly created database -chown pihole:pihole "${gravityDBfile}" -chmod g+w "${piholeDir}" "${gravityDBfile}" - if [[ "${forceDelete:-}" == true ]]; then str="Deleting existing list cache" echo -ne "${INFO} ${str}..." @@ -704,15 +748,26 @@ gravity_DownloadBlocklists # Create local.list gravity_generateLocalList -gravity_ShowCount +# Update gravity timestamp update_gravity_timestamp -gravity_Cleanup -echo "" +# Migrate rest of the data from old to new database +gravity_swap_databases + +# Ensure proper permissions are set for the database +chown pihole:pihole "${gravityDBfile}" +chmod g+w "${piholeDir}" "${gravityDBfile}" # Determine if DNS has been restarted by this instance of gravity if [[ -z "${dnsWasOffline:-}" ]]; then "${PIHOLE_COMMAND}" restartdns reload fi + +# Compute numbers to be displayed +gravity_ShowCount + +gravity_Cleanup +echo "" + "${PIHOLE_COMMAND}" status