From acc50b709efab4a88e2c1f2e7940b5708a47b6aa Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 7 Jul 2019 10:33:08 +0200 Subject: [PATCH] Only migrate files once (domain and adlist lists druing initial creation of gravity.db auditlog.list on database upgrade from version 1 to 2. Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 9 ++ advanced/Templates/gravity.db.sql | 9 +- gravity.sh | 87 ++++++++----------- 3 files changed, 46 insertions(+), 59 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 46ff0a72..714676f4 100644 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -15,7 +15,16 @@ upgrade_gravityDB(){ version="$(sqlite3 "$1" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" if [[ "$version" == "1" ]]; then + # This migration script upgrades the gravity.db file by + # adding the domain_auditlist table sqlite3 "$1" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql" version=2 + + # Store audit domains in database table + if [ -e "${auditFile}" ]; then + echo -e " ${INFO} Migrating content of ${auditFile} into new database" + # database_table_from_file is defined in gravity.sh + database_table_from_file "domain_auditlist" "${auditFile}" + fi fi } diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 113c035f..09d581f0 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -81,20 +81,13 @@ CREATE TABLE gravity domain TEXT PRIMARY KEY ); -CREATE TABLE domain_auditlist -( - id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) -); - CREATE TABLE info ( property TEXT PRIMARY KEY, value TEXT NOT NULL ); -INSERT INTO info VALUES("version","2"); +INSERT INTO info VALUES("version","1"); CREATE VIEW vw_gravity AS SELECT domain FROM gravity diff --git a/gravity.sh b/gravity.sh index 3a8afc7a..e2f2a9a8 100755 --- a/gravity.sh +++ b/gravity.sh @@ -17,8 +17,7 @@ coltable="/opt/pihole/COL_TABLE" source "${coltable}" regexconverter="/opt/pihole/wildcard_regex_converter.sh" source "${regexconverter}" -readonly databaseMigrationScript="/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh" -source "${databaseMigrationScript}" +source "/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh" basename="pihole" PIHOLE_COMMAND="/usr/local/bin/${basename}" @@ -116,33 +115,23 @@ database_table_from_file() { else # Apply format for white-, blacklist, regex, and adlist tables # Read file line by line - if [[ "${table}" == "auditlist" ]]; then - local rowid - declare -i rowid - rowid=1 - grep -v '^ *#' < "${source}" | while IFS= read -r domain - do - # Only add non-empty lines - if [[ -n "${domain}" ]]; then + local rowid + declare -i rowid + rowid=1 + grep -v '^ *#' < "${source}" | while IFS= read -r domain + do + # Only add non-empty lines + if [[ -n "${domain}" ]]; then + if [[ "${table}" == "auditlist" ]]; then # Auditlist table format echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" - rowid+=1 - fi - done - else - local rowid - declare -i rowid - rowid=1 - grep -v '^ *#' < "${source}" | while IFS= read -r domain - do - # Only add non-empty lines - if [[ -n "${domain}" ]]; then + else # White-, black-, and regexlist format echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" - rowid+=1 fi - done - fi + rowid+=1 + fi + done inputfile="${tmpFile}" fi # Store domains in database table specified by ${table} @@ -170,39 +159,35 @@ database_table_from_file() { migrate_to_database() { # Create database file only if not present if [ ! -e "${gravityDBfile}" ]; then + # Create new database file - note that this will be created in version 1 echo -e " ${INFO} Creating new gravity database" generate_gravity_database + + # Migrate list files to new database + if [ -e "${adListFile}" ]; then + # Store adlist domains in database + echo -e " ${INFO} Migrating content of ${adListFile} into new database" + database_table_from_file "adlist" "${adListFile}" + fi + if [ -e "${blacklistFile}" ]; then + # Store blacklisted domains in database + echo -e " ${INFO} Migrating content of ${blacklistFile} into new database" + database_table_from_file "blacklist" "${blacklistFile}" + fi + if [ -e "${whitelistFile}" ]; then + # Store whitelisted domains in database + echo -e " ${INFO} Migrating content of ${whitelistFile} into new database" + database_table_from_file "whitelist" "${whitelistFile}" + fi + if [ -e "${regexFile}" ]; then + # Store regex domains in database + echo -e " ${INFO} Migrating content of ${regexFile} into new database" + database_table_from_file "regex" "${regexFile}" + fi fi # Check if gravity database needs to be updated upgrade_gravityDB "${gravityDBfile}" - - # Migrate list files to new database - if [ -e "${adListFile}" ]; then - # Store adlist domains in database - echo -e " ${INFO} Migrating content of ${adListFile} into new database" - database_table_from_file "adlist" "${adListFile}" - fi - if [ -e "${blacklistFile}" ]; then - # Store blacklisted domains in database - echo -e " ${INFO} Migrating content of ${blacklistFile} into new database" - database_table_from_file "blacklist" "${blacklistFile}" - fi - if [ -e "${whitelistFile}" ]; then - # Store whitelisted domains in database - echo -e " ${INFO} Migrating content of ${whitelistFile} into new database" - database_table_from_file "whitelist" "${whitelistFile}" - fi - if [ -e "${regexFile}" ]; then - # Store regex domains in database - echo -e " ${INFO} Migrating content of ${regexFile} into new database" - database_table_from_file "regex" "${regexFile}" - fi - if [ -e "${auditFile}" ]; then - # Store audit domains in database - echo -e " ${INFO} Migrating content of ${auditFile} into new database" - database_table_from_file "domain_auditlist" "${auditFile}" - fi } # Determine if DNS resolution is available before proceeding