From a8af2e1837946d16f273eb38331a4e1fc5c67e3d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 4 Sep 2019 23:14:29 +0200 Subject: [PATCH 01/11] Store domains without sorting and unifying them first. This allows us to preserve the relationship of the individual domains to the lists they came from. Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 5 + gravity.sh | 194 ++++++++---------- 2 files changed, 86 insertions(+), 113 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index a82d0d51..7d59a6a0 100644 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -39,4 +39,9 @@ upgrade_gravityDB(){ sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/2_to_3.sql" version=3 fi + if [[ "$version" == "3" ]]; then + # This migration script upgrades ... + sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/3_to_4.sql" + version=3 + fi } diff --git a/gravity.sh b/gravity.sh index 98747f35..86bb6a2e 100755 --- a/gravity.sh +++ b/gravity.sh @@ -97,25 +97,39 @@ update_gravity_timestamp() { if [[ "${status}" -ne 0 ]]; then echo -e "\\n ${CROSS} Unable to update gravity timestamp in database ${gravityDBfile}\\n ${output}" + return 1 fi + return 0 } -# Import domains from file and store them in the specified database table -database_table_from_file() { - # Define locals - local table source backup_path backup_file +database_truncate_table() { + local table table="${1}" - source="${2}" - backup_path="${piholeDir}/migration_backup" - backup_file="${backup_path}/$(basename "${2}")" - # Truncate table output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then echo -e "\\n ${CROSS} Unable to truncate ${table} database ${gravityDBfile}\\n ${output}" gravity_Cleanup "error" + return 1 + fi + return 0 +} + +# Import domains from file and store them in the specified database table +database_table_from_file() { + # Define locals + local table source backup_path backup_file arg + table="${1}" + source="${2}" + arg="${3}" + backup_path="${piholeDir}/migration_backup" + backup_file="${backup_path}/$(basename "${2}")" + + # Truncate table only if not gravity (we add multiple times to this table) + if [[ "${table}" != "gravity" ]]; then + database_truncate_table "${table}" fi local tmpFile @@ -123,31 +137,30 @@ database_table_from_file() { local timestamp timestamp="$(date --utc +'%s')" local inputfile - if [[ "${table}" == "gravity" ]]; then - # No need to modify the input data for the gravity table - inputfile="${source}" - else - # Apply format for white-, blacklist, regex, and adlist tables - # Read file line by line - local rowid - declare -i rowid - rowid=1 - grep -v '^ *#' < "${source}" | while IFS= read -r domain - do - # Only add non-empty lines - if [[ -n "${domain}" ]]; then - if [[ "${table}" == "domain_audit" ]]; then - # domain_audit table format (no enable or modified fields) - echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" - else - # White-, black-, and regexlist format - echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" - fi - rowid+=1 + # Apply format for white-, blacklist, regex, and adlist tables + # Read file line by line + local rowid + declare -i rowid + rowid=1 + grep -v '^ *#' < "${source}" | while IFS= read -r domain + do + # Only add non-empty lines + if [[ -n "${domain}" ]]; then + if [[ "${table}" == "domain_audit" ]]; then + # domain_audit table format (no enable or modified fields) + echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" + elif [[ "${table}" == "gravity" ]]; then + # gravity table format + echo "\"${domain}\",${arg}" >> "${tmpFile}" + else + # White-, black-, and regexlist format + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" fi - done - inputfile="${tmpFile}" - fi + rowid+=1 + fi + done + inputfile="${tmpFile}" + # Store domains in database table specified by ${table} # Use printf as .mode and .import need to be on separate lines # see https://unix.stackexchange.com/a/445615/83260 @@ -263,12 +276,13 @@ gravity_CheckDNSResolutionAvailable() { } # Retrieve blocklist URLs and parse domains from adlist.list -gravity_GetBlocklistUrls() { +gravity_DownloadBlocklists() { echo -e " ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..." # Retrieve source URLs from gravity database # We source only enabled adlists, sqlite3 stores boolean values as 0 (false) or 1 (true) mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" + mapfile -t sourceIDs <<< "$(sqlite3 "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)" # Parse source domains from $sources mapfile -t sourceDomains <<< "$( @@ -285,21 +299,23 @@ gravity_GetBlocklistUrls() { if [[ -n "${sources[*]}" ]] && [[ -n "${sourceDomains[*]}" ]]; then echo -e "${OVER} ${TICK} ${str}" - return 0 else echo -e "${OVER} ${CROSS} ${str}" echo -e " ${INFO} No source list found, or it is empty" echo "" return 1 fi -} -# Define options for when retrieving blocklists -gravity_SetDownloadOptions() { local url domain agent cmd_ext str - echo "" + # Flush gravity table once before looping over sources + str="Flushing gravity table" + echo -ne " ${INFO} ${str}..." + if database_truncate_table "gravity"; then + echo -e "${OVER} ${TICK} ${str}" + fi + # Loop through $sources and download each one for ((i = 0; i < "${#sources[@]}"; i++)); do url="${sources[$i]}" @@ -319,7 +335,7 @@ gravity_SetDownloadOptions() { esac echo -e " ${INFO} Target: ${domain} (${url##*/})" - gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}" + gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}" "${sourceIDs[$i]}" echo "" done gravity_Blackbody=true @@ -327,7 +343,7 @@ gravity_SetDownloadOptions() { # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { - local url="${1}" cmd_ext="${2}" agent="${3}" heisenbergCompensator="" patternBuffer str httpCode success="" + local url="${1}" cmd_ext="${2}" agent="${3}" adlistID="${4}" heisenbergCompensator="" patternBuffer str httpCode success="" # Create temp file to store content on disk instead of RAM patternBuffer=$(mktemp -p "/tmp" --suffix=".phgpb") @@ -408,11 +424,20 @@ gravity_DownloadBlocklistFromUrl() { # Determine if the blocklist was downloaded and saved correctly if [[ "${success}" == true ]]; then if [[ "${httpCode}" == "304" ]]; then - : # Do not attempt to re-parse file + # Add domains to database table + str="Adding to database table" + echo -ne " ${INFO} ${str}..." + database_table_from_file "gravity" "${saveLocation}" "${adlistID}" + echo -e "${OVER} ${TICK} ${str}" # Check if $patternbuffer is a non-zero length file elif [[ -s "${patternBuffer}" ]]; then # Determine if blocklist is non-standard and parse as appropriate gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}" + # Add domains to database table + str="Adding to database table" + echo -ne " ${INFO} ${str}..." + database_table_from_file "gravity" "${saveLocation}" "${adlistID}" + echo -e "${OVER} ${TICK} ${str}" else # Fall back to previously cached list if $patternBuffer is empty echo -e " ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" @@ -421,6 +446,11 @@ gravity_DownloadBlocklistFromUrl() { # Determine if cached list has read permission if [[ -r "${saveLocation}" ]]; then echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" + # Add domains to database table + str="Adding to database table" + echo -ne " ${INFO} ${str}..." + database_table_from_file "gravity" "${saveLocation}" "${adlistID}" + echo -e "${OVER} ${TICK} ${str}" else echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" fi @@ -432,7 +462,7 @@ gravity_ParseFileIntoDomains() { local source="${1}" destination="${2}" firstLine abpFilter # Determine if we are parsing a consolidated list - if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then + #if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then # Remove comments and print only the domain name # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious # This helps with that and makes it easier to read @@ -449,7 +479,7 @@ gravity_ParseFileIntoDomains() { sed -r '/([^\.]+\.)+[^\.]{2,}/!d' > "${destination}" chmod 644 "${destination}" return 0 - fi + #fi # Individual file parsing: Keep comments, while parsing domains from each line # We keep comments to respect the list maintainer's licensing @@ -536,80 +566,23 @@ gravity_ParseFileIntoDomains() { fi } -# Create (unfiltered) "Matter and Light" consolidated list -gravity_ConsolidateDownloadedBlocklists() { - local str lastLine - - str="Consolidating blocklists" - echo -ne " ${INFO} ${str}..." - - # Empty $matterAndLight if it already exists, otherwise, create it - : > "${piholeDir}/${matterAndLight}" - chmod 644 "${piholeDir}/${matterAndLight}" - - # Loop through each *.domains file - for i in "${activeDomains[@]}"; do - # Determine if file has read permissions, as download might have failed - if [[ -r "${i}" ]]; then - # Remove windows CRs from file, convert list to lower case, and append into $matterAndLight - tr -d '\r' < "${i}" | tr '[:upper:]' '[:lower:]' >> "${piholeDir}/${matterAndLight}" - - # Ensure that the first line of a new list is on a new line - lastLine=$(tail -1 "${piholeDir}/${matterAndLight}") - if [[ "${#lastLine}" -gt 0 ]]; then - echo "" >> "${piholeDir}/${matterAndLight}" - fi - fi - done - echo -e "${OVER} ${TICK} ${str}" - -} - -# Parse consolidated list into (filtered, unique) domains-only format -gravity_SortAndFilterConsolidatedList() { - local str num - - str="Extracting domains from blocklists" - echo -ne " ${INFO} ${str}..." - - # Parse into file - gravity_ParseFileIntoDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${parsedMatter}" - - # Format $parsedMatter line total as currency - num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${parsedMatter}")") - - echo -e "${OVER} ${TICK} ${str}" - echo -e " ${INFO} Gravity pulled in ${COL_BLUE}${num}${COL_NC} domains" - - str="Removing duplicate domains" - echo -ne " ${INFO} ${str}..." - sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}" - chmod 644 "${piholeDir}/${preEventHorizon}" - echo -e "${OVER} ${TICK} ${str}" - - # Format $preEventHorizon line total as currency - num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") - str="Storing ${COL_BLUE}${num}${COL_NC} unique blocking domains in database" - echo -ne " ${INFO} ${str}..." - database_table_from_file "gravity" "${piholeDir}/${preEventHorizon}" - echo -e "${OVER} ${TICK} ${str}" -} - # Report number of entries in a table gravity_Table_Count() { local table="${1}" local str="${2}" + local extra="${3}" local num - num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table} WHERE enabled = 1;")" + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table} ${extra};")" echo -e " ${INFO} Number of ${str}: ${num}" } # Output count of blacklisted domains and regex filters gravity_ShowCount() { - gravity_Table_Count "blacklist" "exact blacklisted domains" - gravity_Table_Count "regex_blacklist" "regex blacklist filters" - gravity_Table_Count "whitelist" "exact whitelisted domains" - gravity_Table_Count "regex_whitelist" "regex whitelist filters" + gravity_Table_Count "gravity" "gravity domains" "" + gravity_Table_Count "blacklist" "exact blacklisted domains" "WHERE enabled = 1" + gravity_Table_Count "regex_blacklist" "regex blacklist filters" "WHERE enabled = 1" + gravity_Table_Count "whitelist" "exact whitelisted domains" "WHERE enabled = 1" + gravity_Table_Count "regex_whitelist" "regex whitelist filters" "WHERE enabled = 1" } # Parse list of domains into hosts format @@ -748,12 +721,7 @@ fi # Gravity downloads blocklists next gravity_CheckDNSResolutionAvailable -if gravity_GetBlocklistUrls; then - gravity_SetDownloadOptions - # Build preEventHorizon - gravity_ConsolidateDownloadedBlocklists - gravity_SortAndFilterConsolidatedList -fi +gravity_DownloadBlocklists # Create local.list gravity_generateLocalList From ffc91a6c814be6badbc04e3fdc42c82fa0dbdb7e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 7 Sep 2019 11:17:53 +0200 Subject: [PATCH 02/11] Update view vw_gravity to only return domains from enabled adlists. Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 3 ++- .../database_migration/gravity/3_to_4.sql | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 advanced/Scripts/database_migration/gravity/3_to_4.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 7d59a6a0..40fa9655 100644 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -40,7 +40,8 @@ upgrade_gravityDB(){ version=3 fi if [[ "$version" == "3" ]]; then - # This migration script upgrades ... + # This migration script upgrades the gravity and adlist views + # implementing necessary changes for per-client blocking sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/3_to_4.sql" version=3 fi diff --git a/advanced/Scripts/database_migration/gravity/3_to_4.sql b/advanced/Scripts/database_migration/gravity/3_to_4.sql new file mode 100644 index 00000000..4a2f9925 --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/3_to_4.sql @@ -0,0 +1,25 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +DROP TABLE gravity; +CREATE TABLE gravity +( + domain TEXT NOT NULL, + adlist_id INTEGER NOT NULL REFERENCES adlist (id), + PRIMARY KEY(domain, adlist_id) +); + +DROP VIEW vw_gravity; +CREATE VIEW vw_gravity AS SELECT domain, gravity.adlist_id + FROM gravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = gravity.adlist_id + LEFT JOIN adlist ON adlist.id = gravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1); + +UPDATE info SET value = 4 WHERE property = 'version'; + +COMMIT; From ff08add7c0a42536d8901af8e325d7ffe861d887 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 7 Sep 2019 13:01:36 +0200 Subject: [PATCH 03/11] Update vw_whitelist and vw_blacklist to return group_id alongside domain so we can filter if the current client wants to get this domain blocked or not. Signed-off-by: DL6ER --- .../database_migration/gravity/3_to_4.sql | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/database_migration/gravity/3_to_4.sql b/advanced/Scripts/database_migration/gravity/3_to_4.sql index 4a2f9925..5bb96a47 100644 --- a/advanced/Scripts/database_migration/gravity/3_to_4.sql +++ b/advanced/Scripts/database_migration/gravity/3_to_4.sql @@ -13,13 +13,35 @@ CREATE TABLE gravity ); DROP VIEW vw_gravity; -CREATE VIEW vw_gravity AS SELECT domain, gravity.adlist_id +CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id FROM gravity LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = gravity.adlist_id LEFT JOIN adlist ON adlist.id = gravity.adlist_id LEFT JOIN "group" ON "group".id = adlist_by_group.group_id WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1); +DROP VIEW vw_whitelist; +CREATE VIEW vw_whitelist AS SELECT domain, whitelist_by_group.group_id AS group_id + FROM whitelist + LEFT JOIN whitelist_by_group ON whitelist_by_group.whitelist_id = whitelist.id + LEFT JOIN "group" ON "group".id = whitelist_by_group.group_id + WHERE whitelist.enabled = 1 AND (whitelist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY whitelist.id; + +DROP VIEW vw_blacklist; +CREATE VIEW vw_blacklist AS SELECT domain, blacklist_by_group.group_id AS group_id + FROM blacklist + LEFT JOIN blacklist_by_group ON blacklist_by_group.blacklist_id = blacklist.id + LEFT JOIN "group" ON "group".id = blacklist_by_group.group_id + WHERE blacklist.enabled = 1 AND (blacklist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY blacklist.id; + +CREATE TABLE client +( + ip TEXT NOL NULL PRIMARY KEY, + "groups" TEXT NOT NULL +); + UPDATE info SET value = 4 WHERE property = 'version'; COMMIT; From 7b484319176d5058ff4e35242348ee8760238c31 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 9 Sep 2019 00:03:57 +0200 Subject: [PATCH 04/11] Add client_by_group table like we have for the other lists. It stores associations between individual clients and list groups. Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity/3_to_4.sql | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity/3_to_4.sql b/advanced/Scripts/database_migration/gravity/3_to_4.sql index 5bb96a47..e1060da4 100644 --- a/advanced/Scripts/database_migration/gravity/3_to_4.sql +++ b/advanced/Scripts/database_migration/gravity/3_to_4.sql @@ -38,8 +38,15 @@ CREATE VIEW vw_blacklist AS SELECT domain, blacklist_by_group.group_id AS group_ CREATE TABLE client ( - ip TEXT NOL NULL PRIMARY KEY, - "groups" TEXT NOT NULL + id INTEGER PRIMARY KEY AUTOINCREMENT, + ip TEXT NOL NULL UNIQUE +); + +CREATE TABLE client_by_group +( + client_id INTEGER NOT NULL REFERENCES client (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (client_id, group_id) ); UPDATE info SET value = 4 WHERE property = 'version'; From a27c7b13985eed530a1fdf57f3cb5ef91c815aa2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 18 Sep 2019 20:58:44 +0200 Subject: [PATCH 05/11] regex white- and blacklist views need to be re-created as well as we need the ID for storing internally whether or not we try to match a given regex for a specific client. Signed-off-by: DL6ER --- .../database_migration/gravity/3_to_4.sql | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity/3_to_4.sql b/advanced/Scripts/database_migration/gravity/3_to_4.sql index e1060da4..182d24a1 100644 --- a/advanced/Scripts/database_migration/gravity/3_to_4.sql +++ b/advanced/Scripts/database_migration/gravity/3_to_4.sql @@ -21,7 +21,7 @@ CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1); DROP VIEW vw_whitelist; -CREATE VIEW vw_whitelist AS SELECT domain, whitelist_by_group.group_id AS group_id +CREATE VIEW vw_whitelist AS SELECT domain, whitelist.id AS id, whitelist_by_group.group_id AS group_id FROM whitelist LEFT JOIN whitelist_by_group ON whitelist_by_group.whitelist_id = whitelist.id LEFT JOIN "group" ON "group".id = whitelist_by_group.group_id @@ -29,13 +29,29 @@ CREATE VIEW vw_whitelist AS SELECT domain, whitelist_by_group.group_id AS group_ ORDER BY whitelist.id; DROP VIEW vw_blacklist; -CREATE VIEW vw_blacklist AS SELECT domain, blacklist_by_group.group_id AS group_id +CREATE VIEW vw_blacklist AS SELECT domain, blacklist.id AS id, blacklist_by_group.group_id AS group_id FROM blacklist LEFT JOIN blacklist_by_group ON blacklist_by_group.blacklist_id = blacklist.id LEFT JOIN "group" ON "group".id = blacklist_by_group.group_id WHERE blacklist.enabled = 1 AND (blacklist_by_group.group_id IS NULL OR "group".enabled = 1) ORDER BY blacklist.id; +DROP VIEW vw_regex_whitelist; +CREATE VIEW vw_regex_whitelist AS SELECT DISTINCT domain, regex_whitelist.id AS id, regex_whitelist_by_group.group_id AS group_id + FROM regex_whitelist + LEFT JOIN regex_whitelist_by_group ON regex_whitelist_by_group.regex_whitelist_id = regex_whitelist.id + LEFT JOIN "group" ON "group".id = regex_whitelist_by_group.group_id + WHERE regex_whitelist.enabled = 1 AND (regex_whitelist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY regex_whitelist.id; + +DROP VIEW vw_regex_blacklist; +CREATE VIEW vw_regex_blacklist AS SELECT DISTINCT domain, regex_blacklist.id AS id, regex_blacklist_by_group.group_id AS group_id + FROM regex_blacklist + LEFT JOIN regex_blacklist_by_group ON regex_blacklist_by_group.regex_blacklist_id = regex_blacklist.id + LEFT JOIN "group" ON "group".id = regex_blacklist_by_group.group_id + WHERE regex_blacklist.enabled = 1 AND (regex_blacklist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY regex_blacklist.id; + CREATE TABLE client ( id INTEGER PRIMARY KEY AUTOINCREMENT, From d883854aadb5f56075a6fe51cbe3ec59daf6751c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 3 Oct 2019 12:12:32 +0200 Subject: [PATCH 06/11] Use constant for long path. Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity-db.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 7eba43c0..773898b0 100644 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -10,6 +10,8 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. +readonly scriptPath="/etc/.pihole/advanced/Scripts/database_migration/gravity" + upgrade_gravityDB(){ local database piholeDir auditFile version database="${1}" @@ -23,7 +25,7 @@ upgrade_gravityDB(){ # This migration script upgrades the gravity.db file by # adding the domain_audit table echo -e " ${INFO} Upgrading gravity database from version 1 to 2" - sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql" + sqlite3 "${database}" < "${scriptPath}/1_to_2.sql" version=2 # Store audit domains in database table @@ -38,13 +40,13 @@ upgrade_gravityDB(){ # renaming the regex table to regex_blacklist, and # creating a new regex_whitelist table + corresponding linking table and views echo -e " ${INFO} Upgrading gravity database from version 2 to 3" - sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/2_to_3.sql" + sqlite3 "${database}" < "${scriptPath}/2_to_3.sql" version=3 fi if [[ "$version" == "3" ]]; then # This migration script upgrades the gravity and adlist views # implementing necessary changes for per-client blocking - sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/3_to_4.sql" + sqlite3 "${database}" < "${scriptPath}/3_to_4.sql" version=3 fi } From 037d52104a84ac867842e017c30879d17ee79cfe Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 26 Nov 2019 10:58:39 +0100 Subject: [PATCH 07/11] New command "pihole -g -r" recreates gravity.db based on files backed up in /etc/pihole/migration_update. This is useful to restore a working version of the database when the user destroyed the original database. Also, update gravity.db to version 5 because of a fix we needed to implement. Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 12 ++++++++++-- .../database_migration/gravity/4_to_5.sql | 17 +++++++++++++++++ gravity.sh | 17 ++++++++++++++--- 3 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 advanced/Scripts/database_migration/gravity/4_to_5.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 773898b0..1fe6a61f 100644 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -44,9 +44,17 @@ upgrade_gravityDB(){ version=3 fi if [[ "$version" == "3" ]]; then - # This migration script upgrades the gravity and adlist views + # This migration script upgrades the gravity and list views # implementing necessary changes for per-client blocking + echo -e " ${INFO} Upgrading gravity database from version 3 to 4" sqlite3 "${database}" < "${scriptPath}/3_to_4.sql" - version=3 + version=4 + fi + if [[ "$version" == "4" ]]; then + # This migration script upgrades the adlist view + # to return an ID used in gravity.sh + echo -e " ${INFO} Upgrading gravity database from version 4 to 5" + sqlite3 "${database}" < "${scriptPath}/4_to_5.sql" + version=5 fi } diff --git a/advanced/Scripts/database_migration/gravity/4_to_5.sql b/advanced/Scripts/database_migration/gravity/4_to_5.sql new file mode 100644 index 00000000..22b75d58 --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/4_to_5.sql @@ -0,0 +1,17 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +DROP VIEW vw_adlist; +CREATE VIEW vw_adlist AS SELECT DISTINCT address, adlist.id AS id + FROM adlist + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = adlist.id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY adlist.id; + +UPDATE info SET value = 5 WHERE property = 'version'; + +COMMIT; diff --git a/gravity.sh b/gravity.sh index 7a9e4f67..3225b28d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -229,7 +229,7 @@ gravity_CheckDNSResolutionAvailable() { fi # Determine if $lookupDomain is resolvable - if timeout 1 getent hosts "${lookupDomain}" &> /dev/null; then + if timeout 4 getent hosts "${lookupDomain}" &> /dev/null; then # Print confirmation of resolvability if it had previously failed if [[ -n "${secs:-}" ]]; then echo -e "${OVER} ${TICK} DNS resolution is now available\\n" @@ -243,7 +243,7 @@ gravity_CheckDNSResolutionAvailable() { # If the /etc/resolv.conf contains resolvers other than 127.0.0.1 then the local dnsmasq will not be queried and pi.hole is NXDOMAIN. # This means that even though name resolution is working, the getent hosts check fails and the holddown timer keeps ticking and eventualy fails # So we check the output of the last command and if it failed, attempt to use dig +short as a fallback - if timeout 1 dig +short "${lookupDomain}" &> /dev/null; then + if timeout 4 dig +short "${lookupDomain}" &> /dev/null; then if [[ -n "${secs:-}" ]]; then echo -e "${OVER} ${TICK} DNS resolution is now available\\n" fi @@ -425,7 +425,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ "${success}" == true ]]; then if [[ "${httpCode}" == "304" ]]; then # Add domains to database table - str="Adding to database table" + str="Adding adlist with ID ${adlistID} to database table" echo -ne " ${INFO} ${str}..." database_table_from_file "gravity" "${saveLocation}" "${adlistID}" echo -e "${OVER} ${TICK} ${str}" @@ -660,6 +660,7 @@ for var in "$@"; do case "${var}" in "-f" | "--force" ) forceDelete=true;; "-o" | "--optimize" ) optimize_database=true;; + "-r" | "--recreate" ) recreate_database=true;; "-h" | "--help" ) helpFunc;; esac done @@ -667,6 +668,16 @@ done # Trap Ctrl-C gravity_Trap +if [[ "${recreate_database:-}" == true ]]; then + str="Restoring from migration backup" + echo -ne "${INFO} ${str}..." + rm "${gravityDBfile}" + pushd "${piholeDir}" > /dev/null + cp migration_backup/* . + popd > /dev/null + echo -e "${OVER} ${TICK} ${str}" +fi + # Move possibly existing legacy files to the gravity database migrate_to_database From 0c5185f8ba0b658a8de6c2d6c9705fc54e4b0f65 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 4 Dec 2019 21:02:46 +0000 Subject: [PATCH 08/11] Also display how many unique domains we have caught in the event horizon. Signed-off-by: DL6ER --- gravity.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/gravity.sh b/gravity.sh index 3225b28d..72f39ad4 100755 --- a/gravity.sh +++ b/gravity.sh @@ -529,19 +529,24 @@ gravity_ParseFileIntoDomains() { gravity_Table_Count() { local table="${1}" local str="${2}" - local extra="${3}" local num - num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table} ${extra};")" - echo -e " ${INFO} Number of ${str}: ${num}" + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table};")" + if [[ "${table}" == "vw_gravity" ]]; then + local unique + unique="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(DISTINCT domain) FROM ${table};")" + echo -e " ${INFO} Number of ${str}: ${num} (${unique} unique domains)" + else + echo -e " ${INFO} Number of ${str}: ${num}" + fi } # Output count of blacklisted domains and regex filters gravity_ShowCount() { - gravity_Table_Count "gravity" "gravity domains" "" - gravity_Table_Count "blacklist" "exact blacklisted domains" "WHERE enabled = 1" - gravity_Table_Count "regex_blacklist" "regex blacklist filters" "WHERE enabled = 1" - gravity_Table_Count "whitelist" "exact whitelisted domains" "WHERE enabled = 1" - gravity_Table_Count "regex_whitelist" "regex whitelist filters" "WHERE enabled = 1" + gravity_Table_Count "vw_gravity" "gravity domains" "" + gravity_Table_Count "vw_blacklist" "exact blacklisted domains" + gravity_Table_Count "vw_regex_blacklist" "regex blacklist filters" + gravity_Table_Count "vw_whitelist" "exact whitelisted domains" + gravity_Table_Count "vw_regex_whitelist" "regex whitelist filters" } # Parse list of domains into hosts format From 3231e5c3ba833e715e8acf408b862e6b0c6fd3bd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 9 Dec 2019 16:52:03 +0000 Subject: [PATCH 09/11] Address stickler requests. Signed-off-by: DL6ER --- gravity.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index 72f39ad4..5db14765 100755 --- a/gravity.sh +++ b/gravity.sh @@ -434,7 +434,7 @@ gravity_DownloadBlocklistFromUrl() { # Determine if blocklist is non-standard and parse as appropriate gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}" # Add domains to database table - str="Adding to database table" + str="Adding adlist with ID ${adlistID} to database table" echo -ne " ${INFO} ${str}..." database_table_from_file "gravity" "${saveLocation}" "${adlistID}" echo -e "${OVER} ${TICK} ${str}" @@ -677,9 +677,9 @@ if [[ "${recreate_database:-}" == true ]]; then str="Restoring from migration backup" echo -ne "${INFO} ${str}..." rm "${gravityDBfile}" - pushd "${piholeDir}" > /dev/null + pushd "${piholeDir}" > /dev/null || exit cp migration_backup/* . - popd > /dev/null + popd > /dev/null || exit echo -e "${OVER} ${TICK} ${str}" fi From 1f03faddef51b2b942ab432fe2487917dae7ee5a Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 9 Dec 2019 21:35:54 +0000 Subject: [PATCH 10/11] shell check recomends Signed-off-by: Adam Warner --- gravity.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/gravity.sh b/gravity.sh index 5db14765..b04e8553 100755 --- a/gravity.sh +++ b/gravity.sh @@ -40,9 +40,6 @@ gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql" optimize_database=false domainsExtension="domains" -matterAndLight="${basename}.0.matterandlight.txt" -parsedMatter="${basename}.1.parsedmatter.txt" -preEventHorizon="list.preEventHorizon" resolver="pihole-FTL" @@ -92,7 +89,7 @@ generate_gravity_database() { update_gravity_timestamp() { # Update timestamp when the gravity table was last updated successfully - output=$( { sqlite3 "${gravityDBfile}" <<< "INSERT OR REPLACE INTO info (property,value) values (\"updated\",cast(strftime('%s', 'now') as int));"; } 2>&1 ) + output=$( { sqlite3 "${gravityDBfile}" <<< "INSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%s', 'now') as int));"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -459,7 +456,7 @@ gravity_DownloadBlocklistFromUrl() { # Parse source files into domains format gravity_ParseFileIntoDomains() { - local source="${1}" destination="${2}" firstLine abpFilter + local source="${1}" destination="${2}" firstLine # Determine if we are parsing a consolidated list #if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then @@ -612,7 +609,7 @@ gravity_Cleanup() { # Ensure this function only runs when gravity_SetDownloadOptions() has completed if [[ "${gravity_Blackbody:-}" == true ]]; then # Remove any unused .domains files - for file in ${piholeDir}/*.${domainsExtension}; do + for file in "${piholeDir}"/*."${domainsExtension}"; do # If list is not in active array, then remove it if [[ ! "${activeDomains[*]}" == *"${file}"* ]]; then rm -f "${file}" 2> /dev/null || \ From d29947ba32de5313ffbd9bedb76e0f8218812d05 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 9 Dec 2019 22:30:41 +0000 Subject: [PATCH 11/11] optimise gravity list inserts Signed-off-by: Adam Warner --- gravity.sh | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/gravity.sh b/gravity.sh index b04e8553..1d9919a1 100755 --- a/gravity.sh +++ b/gravity.sh @@ -139,23 +139,29 @@ database_table_from_file() { local rowid declare -i rowid rowid=1 - grep -v '^ *#' < "${source}" | while IFS= read -r domain - do - # Only add non-empty lines - if [[ -n "${domain}" ]]; then - if [[ "${table}" == "domain_audit" ]]; then - # domain_audit table format (no enable or modified fields) - echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" - elif [[ "${table}" == "gravity" ]]; then - # gravity table format - echo "\"${domain}\",${arg}" >> "${tmpFile}" - else - # White-, black-, and regexlist format - echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" + + if [[ "${table}" == "gravity" ]]; then + #Append ,${arg} to every line and then remove blank lines before import + sed -e "s/$/,${arg}/" "${source}" > "${tmpFile}" + sed -i '/^$/d' "${tmpFile}" + else + grep -v '^ *#' < "${source}" | while IFS= read -r domain + do + # Only add non-empty lines + if [[ -n "${domain}" ]]; then + if [[ "${table}" == "domain_audit" ]]; then + # domain_audit table format (no enable or modified fields) + echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" + else + # White-, black-, and regexlist format + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" + fi + rowid+=1 fi - rowid+=1 - fi - done + done + fi + + inputfile="${tmpFile}" # Store domains in database table specified by ${table}