diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh new file mode 100644 index 00000000..65b42b95 --- /dev/null +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090 + +# Pi-hole: A black hole for Internet advertisements +# (c) 2019 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# Updates gravity.db database +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + +upgrade_gravityDB(){ + local database piholeDir auditFile version + database="${1}" + piholeDir="${2}" + auditFile="${piholeDir}/auditlog.list" + + # Get database version + version="$(sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" + + if [[ "$version" == "1" ]]; then + # This migration script upgrades the gravity.db file by + # adding the domain_audit table + sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql" + version=2 + + # Store audit domains in database table + if [ -e "${auditFile}" ]; then + echo -e " ${INFO} Migrating content of ${auditFile} into new database" + # database_table_from_file is defined in gravity.sh + database_table_from_file "domain_audit" "${auditFile}" + fi + fi +} diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql new file mode 100644 index 00000000..45b5fa02 --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql @@ -0,0 +1,10 @@ +.timeout 30000 + +CREATE TABLE domain_audit +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) +); + +UPDATE info SET value = 2 WHERE property = 'version'; diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index f744d1b9..9db10bda 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -542,21 +542,45 @@ Teleporter() { php /var/www/html/admin/scripts/pi-hole/php/teleporter.php > "pi-hole-teleporter_${datetimestamp}.tar.gz" } +checkDomain() +{ + local domain validDomain + # Convert to lowercase + domain="${1,,}" + validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check + validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label + echo "${validDomain}" +} + addAudit() { shift # skip "-a" shift # skip "audit" - for var in "$@" + local domains validDomain + domains="" + for domain in "$@" do - echo "${var}" >> /etc/pihole/auditlog.list + # Check domain to be added. Only continue if it is valid + validDomain="$(checkDomain "${domain}")" + if [[ -n "${validDomain}" ]]; then + # Put comma in between domains when there is + # more than one domains to be added + # SQL INSERT allows adding multiple rows at once using the format + ## INSERT INTO table (domain) VALUES ('abc.de'),('fgh.ij'),('klm.no'),('pqr.st'); + if [[ -n "${domains}" ]]; then + domains="${domains}," + fi + domains="${domains}('${domain}')" + fi done - chmod 644 /etc/pihole/auditlog.list + # Insert only the domain here. The date_added field will be + # filled with its default value (date_added = current timestamp) + sqlite3 "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};" } clearAudit() { - echo -n "" > /etc/pihole/auditlog.list - chmod 644 /etc/pihole/auditlog.list + sqlite3 "${gravityDBfile}" "DELETE FROM domain_audit;" } SetPrivacyLevel() { diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 91a38ef7..09d581f0 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -80,6 +80,7 @@ CREATE TABLE gravity ( domain TEXT PRIMARY KEY ); + CREATE TABLE info ( property TEXT PRIMARY KEY, diff --git a/gravity.sh b/gravity.sh index 1a325ba7..d7c66d68 100755 --- a/gravity.sh +++ b/gravity.sh @@ -17,6 +17,8 @@ coltable="/opt/pihole/COL_TABLE" source "${coltable}" regexconverter="/opt/pihole/wildcard_regex_converter.sh" source "${regexconverter}" +# shellcheck disable=SC1091 +source "/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh" basename="pihole" PIHOLE_COMMAND="/usr/local/bin/${basename}" @@ -112,15 +114,21 @@ database_table_from_file() { inputfile="${source}" else # Apply format for white-, blacklist, regex, and adlist tables + # Read file line by line local rowid declare -i rowid rowid=1 - # Read file line by line grep -v '^ *#' < "${source}" | while IFS= read -r domain do # Only add non-empty lines - if [[ ! -z "${domain}" ]]; then - echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" + if [[ -n "${domain}" ]]; then + if [[ "${table}" == "domain_audit" ]]; then + # domain_audit table format (no enable or modified fields) + echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" + else + # White-, black-, and regexlist format + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" + fi rowid+=1 fi done @@ -150,34 +158,36 @@ database_table_from_file() { # Migrate pre-v5.0 list files to database-based Pi-hole versions migrate_to_database() { # Create database file only if not present - if [ -e "${gravityDBfile}" ]; then - return 0 + if [ ! -e "${gravityDBfile}" ]; then + # Create new database file - note that this will be created in version 1 + echo -e " ${INFO} Creating new gravity database" + generate_gravity_database + + # Migrate list files to new database + if [ -e "${adListFile}" ]; then + # Store adlist domains in database + echo -e " ${INFO} Migrating content of ${adListFile} into new database" + database_table_from_file "adlist" "${adListFile}" + fi + if [ -e "${blacklistFile}" ]; then + # Store blacklisted domains in database + echo -e " ${INFO} Migrating content of ${blacklistFile} into new database" + database_table_from_file "blacklist" "${blacklistFile}" + fi + if [ -e "${whitelistFile}" ]; then + # Store whitelisted domains in database + echo -e " ${INFO} Migrating content of ${whitelistFile} into new database" + database_table_from_file "whitelist" "${whitelistFile}" + fi + if [ -e "${regexFile}" ]; then + # Store regex domains in database + echo -e " ${INFO} Migrating content of ${regexFile} into new database" + database_table_from_file "regex" "${regexFile}" + fi fi - echo -e " ${INFO} Creating new gravity database" - generate_gravity_database - - # Migrate list files to new database - if [[ -e "${adListFile}" ]]; then - # Store adlist domains in database - echo -e " ${INFO} Migrating content of ${adListFile} into new database" - database_table_from_file "adlist" "${adListFile}" - fi - if [[ -e "${blacklistFile}" ]]; then - # Store blacklisted domains in database - echo -e " ${INFO} Migrating content of ${blacklistFile} into new database" - database_table_from_file "blacklist" "${blacklistFile}" - fi - if [[ -e "${whitelistFile}" ]]; then - # Store whitelisted domains in database - echo -e " ${INFO} Migrating content of ${whitelistFile} into new database" - database_table_from_file "whitelist" "${whitelistFile}" - fi - if [[ -e "${regexFile}" ]]; then - # Store regex domains in database - echo -e " ${INFO} Migrating content of ${regexFile} into new database" - database_table_from_file "regex" "${regexFile}" - fi + # Check if gravity database needs to be updated + upgrade_gravityDB "${gravityDBfile}" "${piholeDir}" } # Determine if DNS resolution is available before proceeding