From 0de5132e2b2afef6e6fa1c573a9060c79d73d940 Mon Sep 17 00:00:00 2001 From: xch12i5 <40517505+xCh12i5@users.noreply.github.com> Date: Fri, 18 Jan 2019 22:14:34 +0100 Subject: [PATCH 01/86] Solves RegEx issue. Signed-off-by: xch12i5 <40517505+xCh12i5@users.noreply.github.com> --- gravity.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index 75a51608..3247a8e2 100755 --- a/gravity.sh +++ b/gravity.sh @@ -338,11 +338,11 @@ gravity_ParseFileIntoDomains() { # Awk -F splits on given IFS, we grab the right hand side (chops trailing #coments and /'s to grab the domain only. # Last awk command takes non-commented lines and if they have 2 fields, take the right field (the domain) and leave # the left (IP address), otherwise grab the single field. - - < ${source} awk -F '#' '{print $1}' | \ - awk -F '/' '{print $1}' | \ - awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' | \ - sed -nr -e 's/\.{2,}/./g' -e '/\./p' > ${destination} + < ${source} tr -d '\r' | \ + tr '[:upper:]' '[:lower:]' | \ + sed -r '/(\/|#).*$/d' | \ + sed -r 's/^.*\s+//g' | \ + sed -r '/([^\.]+\.)+[^\.]{2,}/!d' > ${destination} return 0 fi From e02da90faae124a9df3847b255f2116c126d26c7 Mon Sep 17 00:00:00 2001 From: xch12i5 <40517505+xCh12i5@users.noreply.github.com> Date: Sat, 19 Jan 2019 17:48:29 +0100 Subject: [PATCH 02/86] Replaces double quotes by single quotes and updates documentation. Signed-off-by: xch12i5 <40517505+xCh12i5@users.noreply.github.com> --- gravity.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/gravity.sh b/gravity.sh index 3247a8e2..7b9116a4 100755 --- a/gravity.sh +++ b/gravity.sh @@ -335,14 +335,16 @@ gravity_ParseFileIntoDomains() { # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious # This helps with that and makes it easier to read # It also helps with debugging so each stage of the script can be researched more in depth - # Awk -F splits on given IFS, we grab the right hand side (chops trailing #coments and /'s to grab the domain only. - # Last awk command takes non-commented lines and if they have 2 fields, take the right field (the domain) and leave - # the left (IP address), otherwise grab the single field. - < ${source} tr -d '\r' | \ - tr '[:upper:]' '[:lower:]' | \ - sed -r '/(\/|#).*$/d' | \ - sed -r 's/^.*\s+//g' | \ - sed -r '/([^\.]+\.)+[^\.]{2,}/!d' > ${destination} + # 1) Remove carriage returns + # 2) Convert all characters to lowercase + # 3) Remove lines containing "#" or "/" + # 4) Remove leading tabs, spaces, etc. + # 5) Delete lines not matching domain names + < ${source} tr -d "\r" | \ + tr "[:upper:]" "[:lower:]" | \ + sed -r "/(\/|#).*$/d" | \ + sed -r "s/^.*\s+//g" | \ + sed -r "/([^\.]+\.)+[^\.]{2,}/!d" > ${destination} return 0 fi From cd8a423b32cd3900861085f9f64affc14cb544f2 Mon Sep 17 00:00:00 2001 From: xch12i5 <40517505+xCh12i5@users.noreply.github.com> Date: Sun, 20 Jan 2019 16:04:07 +0100 Subject: [PATCH 03/86] Solves RegEx issue and updates documentation. Signed-off-by: xch12i5 <40517505+xCh12i5@users.noreply.github.com> --- gravity.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index 7b9116a4..6fe9be06 100755 --- a/gravity.sh +++ b/gravity.sh @@ -340,11 +340,11 @@ gravity_ParseFileIntoDomains() { # 3) Remove lines containing "#" or "/" # 4) Remove leading tabs, spaces, etc. # 5) Delete lines not matching domain names - < ${source} tr -d "\r" | \ - tr "[:upper:]" "[:lower:]" | \ - sed -r "/(\/|#).*$/d" | \ - sed -r "s/^.*\s+//g" | \ - sed -r "/([^\.]+\.)+[^\.]{2,}/!d" > ${destination} + < ${source} tr -d '\r' | \ + tr '[:upper:]' '[:lower:]' | \ + sed -r '/(\/|#).*$/d' | \ + sed -r 's/^.*\s+//g' | \ + sed -r '/([^\.]+\.)+[^\.]{2,}/!d' > ${destination} return 0 fi From 09c4c88a6d1794a3ddb3ee8acb96e2cfb03ad7c2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Feb 2019 13:04:31 +0100 Subject: [PATCH 04/86] Create and fill gravity.db during pihole -g Signed-off-by: DL6ER --- advanced/Templates/gravity.db.schema | 13 +++++++ gravity.sh | 56 ++++++++++++++++++++++++++-- 2 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 advanced/Templates/gravity.db.schema diff --git a/advanced/Templates/gravity.db.schema b/advanced/Templates/gravity.db.schema new file mode 100644 index 00000000..1eee8491 --- /dev/null +++ b/advanced/Templates/gravity.db.schema @@ -0,0 +1,13 @@ +CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, comment TEXT, DateAdded DATETIME); + +CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, comment TEXT, DateAdded DATETIME); + +CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); + +CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain +FROM gravity a +WHERE a.domain NOT IN (SELECT domain from whitelist); + +CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain +FROM blacklist a +WHERE a.domain NOT IN (SELECT domain from whitelist); diff --git a/gravity.sh b/gravity.sh index 75a51608..ac045a93 100755 --- a/gravity.sh +++ b/gravity.sh @@ -35,6 +35,10 @@ blackList="${piholeDir}/black.list" localList="${piholeDir}/local.list" VPNList="/etc/openvpn/ipp.txt" +piholeGitDir="/etc/.pihole" +gravityDBfile="${piholeDir}/gravity.db" +gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.schema" + domainsExtension="domains" matterAndLight="${basename}.0.matterandlight.txt" parsedMatter="${basename}.1.parsedmatter.txt" @@ -83,6 +87,11 @@ if [[ -r "${piholeDir}/pihole.conf" ]]; then echo -e " ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}" fi +# Generate new sqlite3 file from schema template +generate_gravity_database() { + sqlite3 "${gravityDBfile}" < "${gravityDBschema}" +} + # Determine if Pi-hole blocking is disabled # If this is the case, we want to update # gravity.list.bck and black.list.bck instead of @@ -582,12 +591,38 @@ gravity_ParseBlacklistDomains() { cp "${piholeDir}/${preEventHorizon}" "${piholeDir}/${accretionDisc}" fi - # Move the file over as /etc/pihole/gravity.list so dnsmasq can use it - output=$( { mv "${piholeDir}/${accretionDisc}" "${adList}"; } 2>&1 ) + # Create database file if not present + if [ ! -e "${gravityDBfile}" ]; then + generate_gravity_database + fi + + # Backup gravity database + cp "${gravityDBfile}" "${gravityDBfile}.bck" + + # Empty domains + output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM gravity;"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to move ${accretionDisc} from ${piholeDir}\\n ${output}" + echo -e "\\n ${CROSS} Unable to truncate gravity database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + fi + + # Store domains in gravity database + output=$( { sqlite3 "${gravityDBfile}" <<< ".import ${piholeDir}/${accretionDisc} gravity"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to create gravity database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + fi + + # Empty $adList if it already exists, otherwise, create it + output=$( { : > "${adList}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to create empty ${adList}\\n ${output}" gravity_Cleanup "error" fi } @@ -633,6 +668,19 @@ gravity_Cleanup() { echo -e "${OVER} ${TICK} ${str}" + str="Optimizing domains database" + echo -ne " ${INFO} ${str}..." + # Store + output=$( { sqlite3 "${gravityDBfile}" <<< "VACUUM;"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to optimize gravity database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + else + echo -e "${OVER} ${TICK} ${str}" + fi + # Only restart DNS service if offline if ! pidof ${resolver} &> /dev/null; then "${PIHOLE_COMMAND}" restartdns @@ -707,7 +755,7 @@ gravity_ShowBlockCount # Perform when downloading blocklists, or modifying the white/blacklist (not wildcards) if [[ "${skipDownload}" == false ]] || [[ "${listType}" == *"list" ]]; then - str="Parsing domains into hosts format" + str="Parsing domains" echo -ne " ${INFO} ${str}..." gravity_ParseUserDomains From 8a91fe699276cdb1b1671d61f719653c197794b9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Feb 2019 13:21:26 +0100 Subject: [PATCH 05/86] Store domains in gravit tables: gravity, whitelist, blacklist Signed-off-by: DL6ER --- gravity.sh | 104 ++++++++++++++++++++++++++--------------------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/gravity.sh b/gravity.sh index ac045a93..a8c92e8a 100755 --- a/gravity.sh +++ b/gravity.sh @@ -33,6 +33,7 @@ regexFile="${piholeDir}/regex.list" adList="${piholeDir}/gravity.list" blackList="${piholeDir}/black.list" localList="${piholeDir}/local.list" +whiteList="${piholeDir}/white.list" VPNList="/etc/openvpn/ipp.txt" piholeGitDir="/etc/.pihole" @@ -42,8 +43,6 @@ gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.schema" domainsExtension="domains" matterAndLight="${basename}.0.matterandlight.txt" parsedMatter="${basename}.1.parsedmatter.txt" -whitelistMatter="${basename}.2.whitelistmatter.txt" -accretionDisc="${basename}.3.accretionDisc.txt" preEventHorizon="list.preEventHorizon" skipDownload="false" @@ -92,6 +91,49 @@ generate_gravity_database() { sqlite3 "${gravityDBfile}" < "${gravityDBschema}" } +# Import domains from file and store them in the specified database table +gravity_store_in_database() { + # Define locals + local table="${1}" + local source="${2}" + local template="${3}" + + # Create database file if not present + if [ ! -e "${gravityDBfile}" ]; then + generate_gravity_database + fi + + # Backup gravity database + cp "${gravityDBfile}" "${gravityDBfile}.bck" + + # Empty domains + output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to truncate ${table} database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + fi + + # Store domains in gravity database + output=$( { sqlite3 "${gravityDBfile}" <<< ".import \"${source}\" ${table}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to create ${table} database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + fi + + # Empty $adList if it already exists, otherwise, create it + output=$( { : > "${template}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to create empty ${template}\\n ${output}" + gravity_Cleanup "error" + fi +} + # Determine if Pi-hole blocking is disabled # If this is the case, we want to update # gravity.list.bck and black.list.bck instead of @@ -476,7 +518,7 @@ gravity_SortAndFilterConsolidatedList() { echo -ne " ${INFO} ${str}..." fi - # Parse into hosts file + # Parse into file gravity_ParseFileIntoDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${parsedMatter}" # Format $parsedMatter line total as currency @@ -514,8 +556,8 @@ gravity_Whitelist() { str="Number of whitelisted domains: ${num}" echo -ne " ${INFO} ${str}..." - # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in $whitelistFile - comm -23 "${piholeDir}/${preEventHorizon}" <(sort "${whitelistFile}") > "${piholeDir}/${whitelistMatter}" + # Store whitelisted files in gravity database + gravity_store_in_database "whitelist" "${whitelistFile}" "${whiteList}" echo -e "${OVER} ${INFO} ${str}" } @@ -581,50 +623,8 @@ gravity_ParseLocalDomains() { gravity_ParseBlacklistDomains() { local output status - # Empty $accretionDisc if it already exists, otherwise, create it - : > "${piholeDir}/${accretionDisc}" - - if [[ -f "${piholeDir}/${whitelistMatter}" ]]; then - mv "${piholeDir}/${whitelistMatter}" "${piholeDir}/${accretionDisc}" - else - # There was no whitelist file, so use preEventHorizon instead of whitelistMatter. - cp "${piholeDir}/${preEventHorizon}" "${piholeDir}/${accretionDisc}" - fi - - # Create database file if not present - if [ ! -e "${gravityDBfile}" ]; then - generate_gravity_database - fi - - # Backup gravity database - cp "${gravityDBfile}" "${gravityDBfile}.bck" - - # Empty domains - output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM gravity;"; } 2>&1 ) - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to truncate gravity database ${gravityDBfile}\\n ${output}" - gravity_Cleanup "error" - fi - - # Store domains in gravity database - output=$( { sqlite3 "${gravityDBfile}" <<< ".import ${piholeDir}/${accretionDisc} gravity"; } 2>&1 ) - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to create gravity database ${gravityDBfile}\\n ${output}" - gravity_Cleanup "error" - fi - - # Empty $adList if it already exists, otherwise, create it - output=$( { : > "${adList}"; } 2>&1 ) - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to create empty ${adList}\\n ${output}" - gravity_Cleanup "error" - fi + # Store gravity domains in gravity database + gravity_store_in_database "gravity" "${piholeDir}/${preEventHorizon}" "${adList}" } # Create user-added blacklist entries @@ -632,9 +632,9 @@ gravity_ParseUserDomains() { if [[ ! -f "${blacklistFile}" ]]; then return 0 fi - # Copy the file over as /etc/pihole/black.list so dnsmasq can use it - cp "${blacklistFile}" "${blackList}" 2> /dev/null || \ - echo -e "\\n ${CROSS} Unable to move ${blacklistFile##*/} to ${piholeDir}" + + # Fill database table + gravity_store_in_database "blacklist" "${blacklistFile}" "${blackList}" } # Trap Ctrl-C From 710036adae672486abf35be122a5d571d681d428 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Feb 2019 13:34:08 +0100 Subject: [PATCH 06/86] Add DISABLED field to white- and blacklist defaulting to false. The two views respect the DISABLED fields for both lists. Signed-off-by: DL6ER --- advanced/Templates/gravity.db.schema | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/advanced/Templates/gravity.db.schema b/advanced/Templates/gravity.db.schema index 1eee8491..2fd9a9d1 100644 --- a/advanced/Templates/gravity.db.schema +++ b/advanced/Templates/gravity.db.schema @@ -1,13 +1,14 @@ -CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, comment TEXT, DateAdded DATETIME); +CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, comment TEXT, disabled BOOLEAN DEFAULT 0, DateAdded DATETIME); -CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, comment TEXT, DateAdded DATETIME); +CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, comment TEXT, disabled BOOLEAN DEFAULT 0, DateAdded DATETIME); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain FROM gravity a -WHERE a.domain NOT IN (SELECT domain from whitelist); +WHERE a.domain NOT IN (SELECT domain from whitelist WHERE disabled != 1); CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain FROM blacklist a -WHERE a.domain NOT IN (SELECT domain from whitelist); +WHERE a.disabled != 1 AND + a.domain NOT IN (SELECT domain from whitelist WHERE disabled != 1); From dcf0a605cf1b6e101b34206a3d5bd359f3c331a5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Feb 2019 14:14:14 +0100 Subject: [PATCH 07/86] Use views for all tables and set disabled column to false for those tables that support it Signed-off-by: DL6ER --- advanced/Templates/gravity.db.schema | 10 ++++++---- gravity.sh | 17 ++++++++++++++--- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/advanced/Templates/gravity.db.schema b/advanced/Templates/gravity.db.schema index 2fd9a9d1..83141f94 100644 --- a/advanced/Templates/gravity.db.schema +++ b/advanced/Templates/gravity.db.schema @@ -1,7 +1,5 @@ -CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, comment TEXT, disabled BOOLEAN DEFAULT 0, DateAdded DATETIME); - -CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, comment TEXT, disabled BOOLEAN DEFAULT 0, DateAdded DATETIME); - +CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME); +CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain @@ -12,3 +10,7 @@ CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain FROM blacklist a WHERE a.disabled != 1 AND a.domain NOT IN (SELECT domain from whitelist WHERE disabled != 1); + +CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain +FROM whitelist a +WHERE a.disabled != 1; diff --git a/gravity.sh b/gravity.sh index a8c92e8a..21338119 100755 --- a/gravity.sh +++ b/gravity.sh @@ -115,16 +115,27 @@ gravity_store_in_database() { gravity_Cleanup "error" fi - # Store domains in gravity database + # Store domains in gravity database table ${table} output=$( { sqlite3 "${gravityDBfile}" <<< ".import \"${source}\" ${table}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to create ${table} database ${gravityDBfile}\\n ${output}" + echo -e "\\n ${CROSS} Unable to create ${table} in database ${gravityDBfile}\\n ${output}" gravity_Cleanup "error" fi - # Empty $adList if it already exists, otherwise, create it + if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ]; then + # Set disabled to false + output=$( { sqlite3 "${gravityDBfile}" <<< "UPDATE ${table} SET disabled = 0 WHERE disabled IS NULL;"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to set disabled states (${table}) in database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + fi + fi + + # Empty $template if it already exists, otherwise, create it output=$( { : > "${template}"; } 2>&1 ) status="$?" From 93f1859babd7a28de799edb84fdeadf39300daba Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Feb 2019 15:13:18 +0100 Subject: [PATCH 08/86] Store regex in gravity.db as well Signed-off-by: DL6ER --- advanced/Templates/gravity.db.schema | 5 +++++ gravity.sh | 20 +++++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/advanced/Templates/gravity.db.schema b/advanced/Templates/gravity.db.schema index 83141f94..b7b3808a 100644 --- a/advanced/Templates/gravity.db.schema +++ b/advanced/Templates/gravity.db.schema @@ -1,5 +1,6 @@ CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME); CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME); +CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain @@ -14,3 +15,7 @@ WHERE a.disabled != 1 AND CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain FROM whitelist a WHERE a.disabled != 1; + +CREATE VIEW vw_regex AS SELECT DISTINCT a.domain +FROM regex a +WHERE a.disabled != 1; diff --git a/gravity.sh b/gravity.sh index 21338119..e3e78528 100755 --- a/gravity.sh +++ b/gravity.sh @@ -124,7 +124,7 @@ gravity_store_in_database() { gravity_Cleanup "error" fi - if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ]; then + if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then # Set disabled to false output=$( { sqlite3 "${gravityDBfile}" <<< "UPDATE ${table} SET disabled = 0 WHERE disabled IS NULL;"; } 2>&1 ) status="$?" @@ -135,13 +135,16 @@ gravity_store_in_database() { fi fi - # Empty $template if it already exists, otherwise, create it - output=$( { : > "${template}"; } 2>&1 ) - status="$?" + # Only create template file if asked to + if [ "${template}" != "-" ]; then + # Empty $template if it already exists, otherwise, create it + output=$( { : > "${template}"; } 2>&1 ) + status="$?" - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to create empty ${template}\\n ${output}" - gravity_Cleanup "error" + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to create empty ${template}\\n ${output}" + gravity_Cleanup "error" + fi fi } @@ -586,6 +589,9 @@ gravity_ShowBlockCount() { num=$(grep -cv "^#" "${regexFile}") echo -e " ${INFO} Number of regex filters: ${num}" fi + + # Store regex files in gravity database + gravity_store_in_database "regex" "${regexFile}" "-" } # Parse list of domains into hosts format From fc62cf7e2f043cc28d163386aaf7cd0134dce306 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Feb 2019 16:44:05 +0100 Subject: [PATCH 09/86] Only run VACUUM on "full" gravity runs Signed-off-by: DL6ER --- gravity.sh | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/gravity.sh b/gravity.sh index e3e78528..0a1d4e8d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -39,6 +39,7 @@ VPNList="/etc/openvpn/ipp.txt" piholeGitDir="/etc/.pihole" gravityDBfile="${piholeDir}/gravity.db" gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.schema" +optimize_database=true domainsExtension="domains" matterAndLight="${basename}.0.matterandlight.txt" @@ -685,17 +686,19 @@ gravity_Cleanup() { echo -e "${OVER} ${TICK} ${str}" - str="Optimizing domains database" - echo -ne " ${INFO} ${str}..." - # Store - output=$( { sqlite3 "${gravityDBfile}" <<< "VACUUM;"; } 2>&1 ) - status="$?" + if ${optimize_database} ; then + str="Optimizing domains database" + echo -ne " ${INFO} ${str}..." + # Store + output=$( { sqlite3 "${gravityDBfile}" <<< "VACUUM;"; } 2>&1 ) + status="$?" - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to optimize gravity database ${gravityDBfile}\\n ${output}" - gravity_Cleanup "error" - else - echo -e "${OVER} ${TICK} ${str}" + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to optimize gravity database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + else + echo -e "${OVER} ${TICK} ${str}" + fi fi # Only restart DNS service if offline @@ -726,9 +729,9 @@ for var in "$@"; do "-f" | "--force" ) forceDelete=true;; "-h" | "--help" ) helpFunc;; "-sd" | "--skip-download" ) skipDownload=true;; - "-b" | "--blacklist-only" ) listType="blacklist";; - "-w" | "--whitelist-only" ) listType="whitelist";; - "-wild" | "--wildcard-only" ) listType="wildcard"; dnsRestartType="restart";; + "-b" | "--blacklist-only" ) listType="blacklist"; optimize_database=false;; + "-w" | "--whitelist-only" ) listType="whitelist"; optimize_database=false;; + "-wild" | "--wildcard-only" ) listType="wildcard"; optimize_database=false; dnsRestartType="restart";; esac done From d5feffa117c2093e8dbd17f814bf95a7167ab2ba Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Feb 2019 16:45:19 +0100 Subject: [PATCH 10/86] Do not backup the database Signed-off-by: DL6ER --- gravity.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index 0a1d4e8d..0d312b3f 100755 --- a/gravity.sh +++ b/gravity.sh @@ -104,9 +104,6 @@ gravity_store_in_database() { generate_gravity_database fi - # Backup gravity database - cp "${gravityDBfile}" "${gravityDBfile}.bck" - # Empty domains output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 ) status="$?" From 1f9a7d0b0cdba219cc2e3ad7b5eb91e744ab915b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Feb 2019 17:01:38 +0100 Subject: [PATCH 11/86] Only run VACUUM on "pihole -g -o" Signed-off-by: DL6ER --- gravity.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index 0d312b3f..d0b44c24 100755 --- a/gravity.sh +++ b/gravity.sh @@ -39,7 +39,7 @@ VPNList="/etc/openvpn/ipp.txt" piholeGitDir="/etc/.pihole" gravityDBfile="${piholeDir}/gravity.db" gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.schema" -optimize_database=true +optimize_database=false domainsExtension="domains" matterAndLight="${basename}.0.matterandlight.txt" @@ -724,11 +724,12 @@ Options: for var in "$@"; do case "${var}" in "-f" | "--force" ) forceDelete=true;; + "-o" | "--optimize" ) optimize_database=true;; "-h" | "--help" ) helpFunc;; "-sd" | "--skip-download" ) skipDownload=true;; - "-b" | "--blacklist-only" ) listType="blacklist"; optimize_database=false;; - "-w" | "--whitelist-only" ) listType="whitelist"; optimize_database=false;; - "-wild" | "--wildcard-only" ) listType="wildcard"; optimize_database=false; dnsRestartType="restart";; + "-b" | "--blacklist-only" ) listType="blacklist";; + "-w" | "--whitelist-only" ) listType="whitelist";; + "-wild" | "--wildcard-only" ) listType="wildcard"; dnsRestartType="restart";; esac done From b2f4385232f7293d16e2a24eee6b60e3e7011308 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Feb 2019 17:05:00 +0100 Subject: [PATCH 12/86] Avoid infinity loop if gravity optimization fails Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index d0b44c24..06e75a90 100755 --- a/gravity.sh +++ b/gravity.sh @@ -692,7 +692,7 @@ gravity_Cleanup() { if [[ "${status}" -ne 0 ]]; then echo -e "\\n ${CROSS} Unable to optimize gravity database ${gravityDBfile}\\n ${output}" - gravity_Cleanup "error" + error="error" else echo -e "${OVER} ${TICK} ${str}" fi From dbbf21071b4d3a13522d63bdb873ead883f2e13f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 5 Feb 2019 19:05:11 +0100 Subject: [PATCH 13/86] Invert logic to use ENABLED instead of DISABLED. We make sure to set "enabled = 1 WHERE enabled IS NULL" in gravity_store_in_database() Signed-off-by: DL6ER --- advanced/Templates/gravity.db.schema | 21 --------------------- advanced/Templates/gravity.db.sql | 21 +++++++++++++++++++++ gravity.sh | 8 ++++---- 3 files changed, 25 insertions(+), 25 deletions(-) delete mode 100644 advanced/Templates/gravity.db.schema create mode 100644 advanced/Templates/gravity.db.sql diff --git a/advanced/Templates/gravity.db.schema b/advanced/Templates/gravity.db.schema deleted file mode 100644 index b7b3808a..00000000 --- a/advanced/Templates/gravity.db.schema +++ /dev/null @@ -1,21 +0,0 @@ -CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME); -CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME); -CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME); -CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); - -CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain -FROM gravity a -WHERE a.domain NOT IN (SELECT domain from whitelist WHERE disabled != 1); - -CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain -FROM blacklist a -WHERE a.disabled != 1 AND - a.domain NOT IN (SELECT domain from whitelist WHERE disabled != 1); - -CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain -FROM whitelist a -WHERE a.disabled != 1; - -CREATE VIEW vw_regex AS SELECT DISTINCT a.domain -FROM regex a -WHERE a.disabled != 1; diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql new file mode 100644 index 00000000..72d64bc1 --- /dev/null +++ b/advanced/Templates/gravity.db.sql @@ -0,0 +1,21 @@ +CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, DateAdded DATETIME); +CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, DateAdded DATETIME); +CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, DateAdded DATETIME); +CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); + +CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain +FROM gravity a +WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1); + +CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain +FROM blacklist a +WHERE a.enabled == 1 AND + a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1); + +CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain +FROM whitelist a +WHERE a.enabled == 1; + +CREATE VIEW vw_regex AS SELECT DISTINCT a.domain +FROM regex a +WHERE a.enabled == 1; diff --git a/gravity.sh b/gravity.sh index 06e75a90..c47f7487 100755 --- a/gravity.sh +++ b/gravity.sh @@ -38,7 +38,7 @@ VPNList="/etc/openvpn/ipp.txt" piholeGitDir="/etc/.pihole" gravityDBfile="${piholeDir}/gravity.db" -gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.schema" +gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql" optimize_database=false domainsExtension="domains" @@ -123,12 +123,12 @@ gravity_store_in_database() { fi if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then - # Set disabled to false - output=$( { sqlite3 "${gravityDBfile}" <<< "UPDATE ${table} SET disabled = 0 WHERE disabled IS NULL;"; } 2>&1 ) + # Set enabled to true where it is unspecified + output=$( { sqlite3 "${gravityDBfile}" <<< "UPDATE ${table} SET enabled = 1 WHERE enabled IS NULL;"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to set disabled states (${table}) in database ${gravityDBfile}\\n ${output}" + echo -e "\\n ${CROSS} Unable to set enabled states (${table}) in database ${gravityDBfile}\\n ${output}" gravity_Cleanup "error" fi fi From 8a2363621d084bff58ab160cc01633ae32aa4f19 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 5 Feb 2019 19:06:46 +0100 Subject: [PATCH 14/86] Rename fields from DateAdded to dateadded Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 72d64bc1..4470c21c 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,6 +1,6 @@ -CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, DateAdded DATETIME); -CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, DateAdded DATETIME); -CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, DateAdded DATETIME); +CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME); +CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME); +CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain From c3c60e10f18db58e6a3cb09beb573a4eef26f8c8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 5 Feb 2019 19:07:12 +0100 Subject: [PATCH 15/86] Rename regex field from domain to filter because this described better what it is Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 4470c21c..5290e83b 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,6 +1,6 @@ CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME); CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME); -CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME); +CREATE TABLE regex (filter TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain From ec5b16ef851e25167997fe02fd23944be65c7b4d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 5 Feb 2019 19:08:08 +0100 Subject: [PATCH 16/86] Fix incomplete comment Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index c47f7487..c45e124e 100755 --- a/gravity.sh +++ b/gravity.sh @@ -686,7 +686,7 @@ gravity_Cleanup() { if ${optimize_database} ; then str="Optimizing domains database" echo -ne " ${INFO} ${str}..." - # Store + # Run VACUUM command on database to optimize it output=$( { sqlite3 "${gravityDBfile}" <<< "VACUUM;"; } 2>&1 ) status="$?" From 9984647ebbc053f21881c8e4473a4ac5220050cf Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 5 Feb 2019 19:41:11 +0100 Subject: [PATCH 17/86] After renaming column in regex, it also needs to be renamed in vw_regex Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 5290e83b..23efa64a 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -16,6 +16,6 @@ CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain FROM whitelist a WHERE a.enabled == 1; -CREATE VIEW vw_regex AS SELECT DISTINCT a.domain +CREATE VIEW vw_regex AS SELECT DISTINCT a.filter FROM regex a WHERE a.enabled == 1; From 644ec36e6462a9b46f2e63fc5185040e3a803ebd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 6 Feb 2019 18:57:48 +0100 Subject: [PATCH 18/86] Review comments Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 9 +++++--- gravity.sh | 34 ++++++++++++++++++++----------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 23efa64a..3a2230c8 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,7 +1,10 @@ -CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME); -CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME); -CREATE TABLE regex (filter TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME); +CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); +CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); +CREATE TABLE regex (filter TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); +CREATE TABLE info (property TEXT NOT NULL, value TEXT NOT NULL); + +INSERT INTO info VALUES("version","1"); CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain FROM gravity a diff --git a/gravity.sh b/gravity.sh index c45e124e..0c3c8dee 100755 --- a/gravity.sh +++ b/gravity.sh @@ -20,6 +20,8 @@ source "${regexconverter}" basename="pihole" PIHOLE_COMMAND="/usr/local/bin/${basename}" +PIHOLE_USER="pihole" +PIHOLE_GROUP="pihole" piholeDir="/etc/${basename}" @@ -90,6 +92,7 @@ fi # Generate new sqlite3 file from schema template generate_gravity_database() { sqlite3 "${gravityDBfile}" < "${gravityDBschema}" + chown $PIHOLE_USER:$PIHOLE_GROUP "${gravityDBfile}" } # Import domains from file and store them in the specified database table @@ -113,25 +116,32 @@ gravity_store_in_database() { gravity_Cleanup "error" fi + local tmpFile=$(mktemp -p "/tmp" --suffix=".gravity") + if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then + # Apply format for white-, blacklist, and regex tables, prevent globbing + set -f + for domain in $(cat < "${source}"); do + echo "\"${domain}\",1,$(date --utc +'%s')," >> "${tmpFile}" + done + set +f + inputfile="${tmpFile}" + else + # No need to modify the input data for the gravity table + inputfile="${source}" + fi # Store domains in gravity database table ${table} - output=$( { sqlite3 "${gravityDBfile}" <<< ".import \"${source}\" ${table}"; } 2>&1 ) + # Use printf as .mode and .import need to be on separate lines + # see https://unix.stackexchange.com/a/445615/83260 + output=$( { printf ".mode csv\n.import \"${inputfile}\" ${table}\n" | sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to create ${table} in database ${gravityDBfile}\\n ${output}" + echo -e "\\n ${CROSS} Unable to fill table ${table} in database ${gravityDBfile}\\n ${output}" gravity_Cleanup "error" fi - if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then - # Set enabled to true where it is unspecified - output=$( { sqlite3 "${gravityDBfile}" <<< "UPDATE ${table} SET enabled = 1 WHERE enabled IS NULL;"; } 2>&1 ) - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to set enabled states (${table}) in database ${gravityDBfile}\\n ${output}" - gravity_Cleanup "error" - fi - fi + # Delete tmpfile + #rm "$tmpFile" > /dev/null 2>&1 || true # Only create template file if asked to if [ "${template}" != "-" ]; then From 491c828661ba203e679acc74df48a22970f2aee5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 6 Feb 2019 19:09:09 +0100 Subject: [PATCH 19/86] Improved code, ensure that gravity.sh passes shellcheck as a whole Signed-off-by: DL6ER --- gravity.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/gravity.sh b/gravity.sh index 0c3c8dee..129d0ed4 100755 --- a/gravity.sh +++ b/gravity.sh @@ -116,14 +116,15 @@ gravity_store_in_database() { gravity_Cleanup "error" fi - local tmpFile=$(mktemp -p "/tmp" --suffix=".gravity") + local tmpFile + tmpFile="$(mktemp -p "/tmp" --suffix=".gravity")" if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then - # Apply format for white-, blacklist, and regex tables, prevent globbing - set -f - for domain in $(cat < "${source}"); do + # Apply format for white-, blacklist, and regex tables + # Read file line by line + grep -v '^ *#' < "${source}" | while IFS= read -r domain + do echo "\"${domain}\",1,$(date --utc +'%s')," >> "${tmpFile}" done - set +f inputfile="${tmpFile}" else # No need to modify the input data for the gravity table @@ -132,7 +133,7 @@ gravity_store_in_database() { # Store domains in gravity database table ${table} # Use printf as .mode and .import need to be on separate lines # see https://unix.stackexchange.com/a/445615/83260 - output=$( { printf ".mode csv\n.import \"${inputfile}\" ${table}\n" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".mode csv\n.import \"%s\" ${table}\n" "${inputfile}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -412,10 +413,10 @@ gravity_ParseFileIntoDomains() { # Last awk command takes non-commented lines and if they have 2 fields, take the right field (the domain) and leave # the left (IP address), otherwise grab the single field. - < ${source} awk -F '#' '{print $1}' | \ + < "${source}" awk -F '#' '{print $1}' | \ awk -F '/' '{print $1}' | \ awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' | \ - sed -nr -e 's/\.{2,}/./g' -e '/\./p' > ${destination} + sed -nr -e 's/\.{2,}/./g' -e '/\./p' > "${destination}" return 0 fi From f9250d91a52a0bf2c62223ed51b96acf4a80b692 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 6 Feb 2019 19:13:31 +0100 Subject: [PATCH 20/86] Explictly escape the escape. Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 129d0ed4..c742a699 100755 --- a/gravity.sh +++ b/gravity.sh @@ -133,7 +133,7 @@ gravity_store_in_database() { # Store domains in gravity database table ${table} # Use printf as .mode and .import need to be on separate lines # see https://unix.stackexchange.com/a/445615/83260 - output=$( { printf ".mode csv\n.import \"%s\" ${table}\n" "${inputfile}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".mode csv\\n.import \"%s\" ${table}\\n" "${inputfile}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then From 61840edb301005178101ed821da01b05a22e1c0d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 22 Feb 2019 22:46:19 +0100 Subject: [PATCH 21/86] Fill comment field with NULL (default) instead of empty string and use a variable for the timestamp instead of periodically calling date Signed-off-by: DL6ER --- gravity.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index c742a699..6ace8108 100755 --- a/gravity.sh +++ b/gravity.sh @@ -118,12 +118,14 @@ gravity_store_in_database() { local tmpFile tmpFile="$(mktemp -p "/tmp" --suffix=".gravity")" + local timestamp + timestamp="$(date --utc +'%s')" if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then # Apply format for white-, blacklist, and regex tables # Read file line by line grep -v '^ *#' < "${source}" | while IFS= read -r domain do - echo "\"${domain}\",1,$(date --utc +'%s')," >> "${tmpFile}" + echo "\"${domain}\",1,${timestamp}" >> "${tmpFile}" done inputfile="${tmpFile}" else From bbc392caeb631e46f840d1e87d6bebe07442545a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 22 Feb 2019 22:49:02 +0100 Subject: [PATCH 22/86] More review comments Signed-off-by: DL6ER --- gravity.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 6ace8108..b09ddc0c 100755 --- a/gravity.sh +++ b/gravity.sh @@ -120,6 +120,7 @@ gravity_store_in_database() { tmpFile="$(mktemp -p "/tmp" --suffix=".gravity")" local timestamp timestamp="$(date --utc +'%s')" + local inputfile if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then # Apply format for white-, blacklist, and regex tables # Read file line by line @@ -135,7 +136,7 @@ gravity_store_in_database() { # Store domains in gravity database table ${table} # Use printf as .mode and .import need to be on separate lines # see https://unix.stackexchange.com/a/445615/83260 - output=$( { printf ".mode csv\\n.import \"%s\" ${table}\\n" "${inputfile}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".mode csv\\n.import \"%s\" %s\\n" "${inputfile}" "${table}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -144,7 +145,7 @@ gravity_store_in_database() { fi # Delete tmpfile - #rm "$tmpFile" > /dev/null 2>&1 || true + rm "$tmpFile" > /dev/null 2>&1 || true # Only create template file if asked to if [ "${template}" != "-" ]; then From 54ae9d83b0a4ff09c2a40db079df9952cfd122af Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Sat, 23 Feb 2019 13:42:15 -0800 Subject: [PATCH 23/86] Stickler-lints Signed-off-by: Dan Schaper --- gravity.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 6fe9be06..015764ac 100755 --- a/gravity.sh +++ b/gravity.sh @@ -340,11 +340,11 @@ gravity_ParseFileIntoDomains() { # 3) Remove lines containing "#" or "/" # 4) Remove leading tabs, spaces, etc. # 5) Delete lines not matching domain names - < ${source} tr -d '\r' | \ + < "${source}" tr -d '\r' | \ tr '[:upper:]' '[:lower:]' | \ sed -r '/(\/|#).*$/d' | \ sed -r 's/^.*\s+//g' | \ - sed -r '/([^\.]+\.)+[^\.]{2,}/!d' > ${destination} + sed -r '/([^\.]+\.)+[^\.]{2,}/!d' > "${destination}" return 0 fi From de4456330169a3328c6e22ab4860a4f42a916c22 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Apr 2019 18:51:38 +0200 Subject: [PATCH 24/86] Remove addn-hosts from 01-pihole.conf Signed-off-by: DL6ER --- advanced/01-pihole.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/advanced/01-pihole.conf b/advanced/01-pihole.conf index 502293bf..40a117fe 100644 --- a/advanced/01-pihole.conf +++ b/advanced/01-pihole.conf @@ -18,8 +18,6 @@ # WITHIN /etc/dnsmasq.d/yourname.conf # ############################################################################### -addn-hosts=/etc/pihole/gravity.list -addn-hosts=/etc/pihole/black.list addn-hosts=/etc/pihole/local.list domain-needed From 6594a0a6e75ed60c5b15f422a9b319f14e312330 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Apr 2019 18:53:20 +0200 Subject: [PATCH 25/86] Fix detection of blocked query in pihole -t Signed-off-by: DL6ER --- pihole | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pihole b/pihole index 84a5623f..6b572fa9 100755 --- a/pihole +++ b/pihole @@ -313,7 +313,7 @@ tailFunc() { # Colour everything else as gray tail -f /var/log/pihole.log | sed -E \ -e "s,($(date +'%b %d ')| dnsmasq[.*[0-9]]),,g" \ - -e "s,(.*(gravity.list|black.list|regex.list| config ).* is (0.0.0.0|::|NXDOMAIN|${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \ + -e "s,(.*(gravity |black |regex | config ).* is (0.0.0.0|::|NXDOMAIN|${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \ -e "s,.*(query\\[A|DHCP).*,${COL_NC}&${COL_NC}," \ -e "s,.*,${COL_GRAY}&${COL_NC}," exit 0 From c13ebd3a9d9cab33e0d6e8d389b5e69e8115d14d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Apr 2019 18:54:55 +0200 Subject: [PATCH 26/86] Replace Pi-hole ID string with another line we intend to keep Signed-off-by: DL6ER --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 110399f8..ffc1bdf6 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1224,7 +1224,7 @@ version_check_dnsmasq() { # Local, named variables local dnsmasq_conf="/etc/dnsmasq.conf" local dnsmasq_conf_orig="/etc/dnsmasq.conf.orig" - local dnsmasq_pihole_id_string="addn-hosts=/etc/pihole/gravity.list" + local dnsmasq_pihole_id_string="# Dnsmasq config for Pi-hole's FTLDNS" local dnsmasq_original_config="${PI_HOLE_LOCAL_REPO}/advanced/dnsmasq.conf.original" local dnsmasq_pihole_01_snippet="${PI_HOLE_LOCAL_REPO}/advanced/01-pihole.conf" local dnsmasq_pihole_01_location="/etc/dnsmasq.d/01-pihole.conf" @@ -1232,7 +1232,7 @@ version_check_dnsmasq() { # If the dnsmasq config file exists if [[ -f "${dnsmasq_conf}" ]]; then printf " %b Existing dnsmasq.conf found..." "${INFO}" - # If gravity.list is found within this file, we presume it's from older versions on Pi-hole, + # If a specific string is found within this file, we presume it's from older versions on Pi-hole, if grep -q ${dnsmasq_pihole_id_string} ${dnsmasq_conf}; then printf " it is from a previous Pi-hole install.\\n" printf " %b Backing up dnsmasq.conf to dnsmasq.conf.orig..." "${INFO}" From d7ee44960e25b3c3bb89c2e81578c1a8cff14087 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Apr 2019 18:56:44 +0200 Subject: [PATCH 27/86] Remove moving around of the gravity and blacklist files as we do not use them any longer Signed-off-by: DL6ER --- pihole | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/pihole b/pihole index 6b572fa9..71c286c6 100755 --- a/pihole +++ b/pihole @@ -10,10 +10,8 @@ # Please see LICENSE file for your rights under this license. readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" -readonly gravitylist="/etc/pihole/gravity.list" -readonly blacklist="/etc/pihole/black.list" -# setupVars is not readonly here because in some funcitons (checkout), +# setupVars is not readonly here because in some functions (checkout), # it might get set again when the installer is sourced. This causes an # error due to modifying a readonly variable. setupVars="/etc/pihole/setupVars.conf" @@ -148,14 +146,6 @@ Time: echo -e " ${INFO} Blocking already disabled, nothing to do" exit 0 fi - if [[ -e "${gravitylist}" ]]; then - mv "${gravitylist}" "${gravitylist}.bck" - echo "" > "${gravitylist}" - fi - if [[ -e "${blacklist}" ]]; then - mv "${blacklist}" "${blacklist}.bck" - echo "" > "${blacklist}" - fi if [[ $# > 1 ]]; then local error=false if [[ "${2}" == *"s" ]]; then @@ -204,12 +194,6 @@ Time: echo -e " ${INFO} Enabling blocking" local str="Pi-hole Enabled" - if [[ -e "${gravitylist}.bck" ]]; then - mv "${gravitylist}.bck" "${gravitylist}" - fi - if [[ -e "${blacklist}.bck" ]]; then - mv "${blacklist}.bck" "${blacklist}" - fi sed -i "/BLOCKING_ENABLED=/d" "${setupVars}" echo "BLOCKING_ENABLED=true" >> "${setupVars}" fi From ee20164207508ca1e2dbb08e05b0858e074dd370 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Apr 2019 19:01:31 +0200 Subject: [PATCH 28/86] Remove detect_pihole_blocking_status() as we can modify the database at any time Signed-off-by: DL6ER --- gravity.sh | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/gravity.sh b/gravity.sh index b09ddc0c..b287653d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -160,20 +160,6 @@ gravity_store_in_database() { fi } -# Determine if Pi-hole blocking is disabled -# If this is the case, we want to update -# gravity.list.bck and black.list.bck instead of -# gravity.list and black.list -detect_pihole_blocking_status() { - if [[ "${BLOCKING_ENABLED}" == false ]]; then - echo -e " ${INFO} Pi-hole blocking is disabled" - adList="${adList}.bck" - blackList="${blackList}.bck" - else - echo -e " ${INFO} Pi-hole blocking is enabled" - fi -} - # Determine if DNS resolution is available before proceeding gravity_CheckDNSResolutionAvailable() { local lookupDomain="pi.hole" @@ -235,7 +221,7 @@ gravity_GetBlocklistUrls() { echo -e " ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..." if [[ -f "${adListDefault}" ]] && [[ -f "${adListFile}" ]]; then - # Remove superceded $adListDefault file + # Remove superseded $adListDefault file rm "${adListDefault}" 2> /dev/null || \ echo -e " ${CROSS} Unable to remove ${adListDefault}" fi @@ -758,8 +744,6 @@ if [[ "${forceDelete:-}" == true ]]; then echo -e "${OVER} ${TICK} ${str}" fi -detect_pihole_blocking_status - # Determine which functions to run if [[ "${skipDownload}" == false ]]; then # Gravity needs to download blocklists From e57d0fb93c24e6d7f336f449dc9bd1fc7627062d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Apr 2019 19:05:08 +0200 Subject: [PATCH 29/86] Remove the empty templates gravity.list, white.list, black.list. The old source files whitelist.txt, blacklist.txt, and regex.list still exist and need to be removed in a follow-up commit Signed-off-by: DL6ER --- gravity.sh | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/gravity.sh b/gravity.sh index b287653d..6313b12c 100755 --- a/gravity.sh +++ b/gravity.sh @@ -32,10 +32,7 @@ whitelistFile="${piholeDir}/whitelist.txt" blacklistFile="${piholeDir}/blacklist.txt" regexFile="${piholeDir}/regex.list" -adList="${piholeDir}/gravity.list" -blackList="${piholeDir}/black.list" localList="${piholeDir}/local.list" -whiteList="${piholeDir}/white.list" VPNList="/etc/openvpn/ipp.txt" piholeGitDir="/etc/.pihole" @@ -146,18 +143,6 @@ gravity_store_in_database() { # Delete tmpfile rm "$tmpFile" > /dev/null 2>&1 || true - - # Only create template file if asked to - if [ "${template}" != "-" ]; then - # Empty $template if it already exists, otherwise, create it - output=$( { : > "${template}"; } 2>&1 ) - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to create empty ${template}\\n ${output}" - gravity_Cleanup "error" - fi - fi } # Determine if DNS resolution is available before proceeding @@ -569,7 +554,7 @@ gravity_Whitelist() { echo -ne " ${INFO} ${str}..." # Store whitelisted files in gravity database - gravity_store_in_database "whitelist" "${whitelistFile}" "${whiteList}" + gravity_store_in_database "whitelist" "${whitelistFile}" echo -e "${OVER} ${INFO} ${str}" } @@ -589,7 +574,7 @@ gravity_ShowBlockCount() { fi # Store regex files in gravity database - gravity_store_in_database "regex" "${regexFile}" "-" + gravity_store_in_database "regex" "${regexFile}" } # Parse list of domains into hosts format @@ -639,7 +624,7 @@ gravity_ParseBlacklistDomains() { local output status # Store gravity domains in gravity database - gravity_store_in_database "gravity" "${piholeDir}/${preEventHorizon}" "${adList}" + gravity_store_in_database "gravity" "${piholeDir}/${preEventHorizon}" } # Create user-added blacklist entries @@ -649,7 +634,7 @@ gravity_ParseUserDomains() { fi # Fill database table - gravity_store_in_database "blacklist" "${blacklistFile}" "${blackList}" + gravity_store_in_database "blacklist" "${blacklistFile}" } # Trap Ctrl-C From 61cfd2f9f9ec4294e041937748f08368b80e88d5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Apr 2019 19:48:06 +0200 Subject: [PATCH 30/86] Add adlists table Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 3a2230c8..5442987f 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,6 +1,7 @@ CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); CREATE TABLE regex (filter TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); +CREATE TABLE adlists (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); CREATE TABLE info (property TEXT NOT NULL, value TEXT NOT NULL); From c1277705d897ce5ca15aa4d76b1ba253b42f7e6a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Apr 2019 19:55:05 +0200 Subject: [PATCH 31/86] Copy existing whitelist.txt, blacklist.txt, regex.list, and adlists.list to the database. We remove the files afterwards as the content lives in the database now Signed-off-by: DL6ER --- gravity.sh | 62 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/gravity.sh b/gravity.sh index 6313b12c..4fb601b4 100755 --- a/gravity.sh +++ b/gravity.sh @@ -25,12 +25,11 @@ PIHOLE_GROUP="pihole" piholeDir="/etc/${basename}" -adListFile="${piholeDir}/adlists.list" -adListDefault="${piholeDir}/adlists.default" - +# Legacy (pre v5.0) list file locations whitelistFile="${piholeDir}/whitelist.txt" blacklistFile="${piholeDir}/blacklist.txt" regexFile="${piholeDir}/regex.list" +adListFile="${piholeDir}/adlists.list" localList="${piholeDir}/local.list" VPNList="/etc/openvpn/ipp.txt" @@ -93,18 +92,20 @@ generate_gravity_database() { } # Import domains from file and store them in the specified database table -gravity_store_in_database() { +database_table_from_file() { # Define locals local table="${1}" local source="${2}" - local template="${3}" # Create database file if not present if [ ! -e "${gravityDBfile}" ]; then + echo -e " ${INFO} Creating new gravity database" generate_gravity_database fi - # Empty domains + echo -e " ${INFO} Pi-hole upgrade: Moving content of ${source} into database" + + # Truncate table output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 ) status="$?" @@ -118,19 +119,19 @@ gravity_store_in_database() { local timestamp timestamp="$(date --utc +'%s')" local inputfile - if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then - # Apply format for white-, blacklist, and regex tables + if [[ "${table}" == "gravity" ]]; then + # No need to modify the input data for the gravity table + inputfile="${source}" + else + # Apply format for white-, blacklist, regex, and adlists tables # Read file line by line grep -v '^ *#' < "${source}" | while IFS= read -r domain do echo "\"${domain}\",1,${timestamp}" >> "${tmpFile}" done inputfile="${tmpFile}" - else - # No need to modify the input data for the gravity table - inputfile="${source}" fi - # Store domains in gravity database table ${table} + # Store domains in database table specified by ${table} # Use printf as .mode and .import need to be on separate lines # see https://unix.stackexchange.com/a/445615/83260 output=$( { printf ".mode csv\\n.import \"%s\" %s\\n" "${inputfile}" "${table}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) @@ -145,6 +146,31 @@ gravity_store_in_database() { rm "$tmpFile" > /dev/null 2>&1 || true } +migrate_to_database() { +ls -lh + # Migrate pre-v5.0 list files to database-based Pi-hole versions + if [[ -e "${whitelistFile}" ]]; then + # Store whitelisted domains in database + database_table_from_file "whitelist" "${whitelistFile}" + rm "${whitelistFile}" + fi + if [[ -e "${blacklistFile}" ]]; then + # Store blacklisted domains in database + database_table_from_file "blacklist" "${blacklistFile}" + rm "${blacklistFile}" + fi + if [[ -e "${regexFile}" ]]; then + # Store regex domains in database + database_table_from_file "regex" "${regexFile}" + rm "${regexFile}" + fi + if [[ -e "${adListFile}" ]]; then + # Store adlists domains in database + database_table_from_file "adlists" "${adListFile}" + rm "${adListFile}" + fi +} + # Determine if DNS resolution is available before proceeding gravity_CheckDNSResolutionAvailable() { local lookupDomain="pi.hole" @@ -553,9 +579,6 @@ gravity_Whitelist() { str="Number of whitelisted domains: ${num}" echo -ne " ${INFO} ${str}..." - # Store whitelisted files in gravity database - gravity_store_in_database "whitelist" "${whitelistFile}" - echo -e "${OVER} ${INFO} ${str}" } @@ -574,7 +597,7 @@ gravity_ShowBlockCount() { fi # Store regex files in gravity database - gravity_store_in_database "regex" "${regexFile}" + database_table_from_file "regex" "${regexFile}" } # Parse list of domains into hosts format @@ -624,7 +647,7 @@ gravity_ParseBlacklistDomains() { local output status # Store gravity domains in gravity database - gravity_store_in_database "gravity" "${piholeDir}/${preEventHorizon}" + database_table_from_file "gravity" "${piholeDir}/${preEventHorizon}" } # Create user-added blacklist entries @@ -634,7 +657,7 @@ gravity_ParseUserDomains() { fi # Fill database table - gravity_store_in_database "blacklist" "${blacklistFile}" + database_table_from_file "blacklist" "${blacklistFile}" } # Trap Ctrl-C @@ -721,6 +744,9 @@ done # Trap Ctrl-C gravity_Trap +# Move possibly existing legacy files to the gravity database +migrate_to_database + if [[ "${forceDelete:-}" == true ]]; then str="Deleting existing list cache" echo -ne "${INFO} ${str}..." From 2664ac0efc6dab8720170d55b09c2ea50d857f18 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 25 Apr 2019 10:18:37 +0200 Subject: [PATCH 32/86] Improve error reporting Signed-off-by: DL6ER --- gravity.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index 4fb601b4..870027a1 100755 --- a/gravity.sh +++ b/gravity.sh @@ -143,7 +143,12 @@ database_table_from_file() { fi # Delete tmpfile - rm "$tmpFile" > /dev/null 2>&1 || true + rm "${tmpFile}" > /dev/null 2>&1 || \ + echo -e " ${CROSS} Unable to remove ${tmpFile}" + + # Delete source file + rm "${source}" 2> /dev/null || \ + echo -e " ${CROSS} Unable to remove ${source}" } migrate_to_database() { @@ -152,22 +157,18 @@ ls -lh if [[ -e "${whitelistFile}" ]]; then # Store whitelisted domains in database database_table_from_file "whitelist" "${whitelistFile}" - rm "${whitelistFile}" fi if [[ -e "${blacklistFile}" ]]; then # Store blacklisted domains in database database_table_from_file "blacklist" "${blacklistFile}" - rm "${blacklistFile}" fi if [[ -e "${regexFile}" ]]; then # Store regex domains in database database_table_from_file "regex" "${regexFile}" - rm "${regexFile}" fi if [[ -e "${adListFile}" ]]; then # Store adlists domains in database database_table_from_file "adlists" "${adListFile}" - rm "${adListFile}" fi } From 465a39427b1a0f34b83ede17a9dbfb000a19a3bb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 25 Apr 2019 10:46:18 +0200 Subject: [PATCH 33/86] Source adlists from gravity database Signed-off-by: DL6ER --- gravity.sh | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/gravity.sh b/gravity.sh index 870027a1..0333a25d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -151,9 +151,8 @@ database_table_from_file() { echo -e " ${CROSS} Unable to remove ${source}" } +# Migrate pre-v5.0 list files to database-based Pi-hole versions migrate_to_database() { -ls -lh - # Migrate pre-v5.0 list files to database-based Pi-hole versions if [[ -e "${whitelistFile}" ]]; then # Store whitelisted domains in database database_table_from_file "whitelist" "${whitelistFile}" @@ -232,15 +231,9 @@ gravity_CheckDNSResolutionAvailable() { gravity_GetBlocklistUrls() { echo -e " ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..." - if [[ -f "${adListDefault}" ]] && [[ -f "${adListFile}" ]]; then - # Remove superseded $adListDefault file - rm "${adListDefault}" 2> /dev/null || \ - echo -e " ${CROSS} Unable to remove ${adListDefault}" - fi - - # Retrieve source URLs from $adListFile - # Logic: Remove comments and empty lines - mapfile -t sources <<< "$(grep -v -E "^(#|$)" "${adListFile}" 2> /dev/null)" + # Retrieve source URLs from gravity database + # We source only enabled adlists, sqlite3 stores boolean values as 0 (false) or 1 (true) + mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM adlists WHERE enabled = 1;" 2> /dev/null)" # Parse source domains from $sources mapfile -t sourceDomains <<< "$( From d94bbfca7f728258ed3f0574b9eb9bf783114652 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 25 Apr 2019 11:18:54 +0200 Subject: [PATCH 34/86] Finish migrating gravity.db to using our new gravity database. This removes the --skip-download, --blacklist-only, --whitelist-only, and --wildcard-only options as it is not necessary to run gravity when modifying them from now on (gravity really only builds the gravity database and the local.list file). Signed-off-by: DL6ER --- gravity.sh | 171 +++++++++++++++-------------------------------------- 1 file changed, 49 insertions(+), 122 deletions(-) diff --git a/gravity.sh b/gravity.sh index 0333a25d..09e7b19c 100755 --- a/gravity.sh +++ b/gravity.sh @@ -44,8 +44,6 @@ matterAndLight="${basename}.0.matterandlight.txt" parsedMatter="${basename}.1.parsedmatter.txt" preEventHorizon="list.preEventHorizon" -skipDownload="false" - resolver="pihole-FTL" haveSourceUrls=true @@ -103,8 +101,6 @@ database_table_from_file() { generate_gravity_database fi - echo -e " ${INFO} Pi-hole upgrade: Moving content of ${source} into database" - # Truncate table output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 ) status="$?" @@ -153,22 +149,26 @@ database_table_from_file() { # Migrate pre-v5.0 list files to database-based Pi-hole versions migrate_to_database() { - if [[ -e "${whitelistFile}" ]]; then - # Store whitelisted domains in database - database_table_from_file "whitelist" "${whitelistFile}" + if [[ -e "${adListFile}" ]]; then + # Store adlists domains in database + echo -e " ${INFO} Pi-hole upgrade: Moving content of ${adListFile} into database" + database_table_from_file "adlists" "${adListFile}" fi if [[ -e "${blacklistFile}" ]]; then # Store blacklisted domains in database + echo -e " ${INFO} Pi-hole upgrade: Moving content of ${blacklistFile} into database" database_table_from_file "blacklist" "${blacklistFile}" fi + if [[ -e "${whitelistFile}" ]]; then + # Store whitelisted domains in database + echo -e " ${INFO} Pi-hole upgrade: Moving content of ${whitelistFile} into database" + database_table_from_file "whitelist" "${whitelistFile}" + fi if [[ -e "${regexFile}" ]]; then # Store regex domains in database + echo -e " ${INFO} Pi-hole upgrade: Moving content of ${regexFile} into database" database_table_from_file "regex" "${regexFile}" fi - if [[ -e "${adListFile}" ]]; then - # Store adlists domains in database - database_table_from_file "adlists" "${adListFile}" - fi } # Determine if DNS resolution is available before proceeding @@ -282,11 +282,9 @@ gravity_SetDownloadOptions() { *) cmd_ext="";; esac - if [[ "${skipDownload}" == false ]]; then - echo -e " ${INFO} Target: ${domain} (${url##*/})" - gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}" - echo "" - fi + echo -e " ${INFO} Target: ${domain} (${url##*/})" + gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}" + echo "" done gravity_Blackbody=true } @@ -543,55 +541,35 @@ gravity_SortAndFilterConsolidatedList() { if [[ "${haveSourceUrls}" == true ]]; then echo -e "${OVER} ${TICK} ${str}" fi - echo -e " ${INFO} Number of domains being pulled in by gravity: ${COL_BLUE}${num}${COL_NC}" + echo -e " ${INFO} Gravity pulled in ${COL_BLUE}${num}${COL_NC} domains" str="Removing duplicate domains" - if [[ "${haveSourceUrls}" == true ]]; then - echo -ne " ${INFO} ${str}..." - fi - + echo -ne " ${INFO} ${str}..." sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}" + echo -e "${OVER} ${TICK} ${str}" - if [[ "${haveSourceUrls}" == true ]]; then - echo -e "${OVER} ${TICK} ${str}" - # Format $preEventHorizon line total as currency - num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") - echo -e " ${INFO} Number of unique domains trapped in the Event Horizon: ${COL_BLUE}${num}${COL_NC}" - fi + # Format $preEventHorizon line total as currency + num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") + str="Storing ${COL_BLUE}${num}${COL_NC} unique blocking domains in database" + echo -ne " ${INFO} ${str}..." + database_table_from_file "gravity" "${piholeDir}/${preEventHorizon}" + echo -e "${OVER} ${TICK} ${str}" } -# Whitelist user-defined domains -gravity_Whitelist() { - local num str - - if [[ ! -f "${whitelistFile}" ]]; then - echo -e " ${INFO} Nothing to whitelist!" - return 0 - fi - - num=$(wc -l < "${whitelistFile}") - str="Number of whitelisted domains: ${num}" - echo -ne " ${INFO} ${str}..." - - echo -e "${OVER} ${INFO} ${str}" +# Report number of entries in a table +gravity_Table_Count() { + local table="${1}" + local str="${2}" + local num + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table} WHERE enabled = 1;")" + echo -e " ${INFO} Number of ${str}: ${num}" } # Output count of blacklisted domains and regex filters -gravity_ShowBlockCount() { - local num - - if [[ -f "${blacklistFile}" ]]; then - num=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")") - echo -e " ${INFO} Number of blacklisted domains: ${num}" - fi - - if [[ -f "${regexFile}" ]]; then - num=$(grep -cv "^#" "${regexFile}") - echo -e " ${INFO} Number of regex filters: ${num}" - fi - - # Store regex files in gravity database - database_table_from_file "regex" "${regexFile}" +gravity_ShowCount() { + gravity_Table_Count "blacklist" "blacklisted domains" + gravity_Table_Count "whitelist" "whitelisted domains" + gravity_Table_Count "regex" "regex filters" } # Parse list of domains into hosts format @@ -611,7 +589,7 @@ gravity_ParseDomainsIntoHosts() { } # Create "localhost" entries into hosts format -gravity_ParseLocalDomains() { +gravity_generateLocalList() { local hostname if [[ -s "/etc/hostname" ]]; then @@ -636,24 +614,6 @@ gravity_ParseLocalDomains() { fi } -# Create primary blacklist entries -gravity_ParseBlacklistDomains() { - local output status - - # Store gravity domains in gravity database - database_table_from_file "gravity" "${piholeDir}/${preEventHorizon}" -} - -# Create user-added blacklist entries -gravity_ParseUserDomains() { - if [[ ! -f "${blacklistFile}" ]]; then - return 0 - fi - - # Fill database table - database_table_from_file "blacklist" "${blacklistFile}" -} - # Trap Ctrl-C gravity_Trap() { trap '{ echo -e "\\n\\n ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT @@ -689,7 +649,7 @@ gravity_Cleanup() { str="Optimizing domains database" echo -ne " ${INFO} ${str}..." # Run VACUUM command on database to optimize it - output=$( { sqlite3 "${gravityDBfile}" <<< "VACUUM;"; } 2>&1 ) + output=$( { sqlite3 "${gravityDBfile}" "VACUUM;"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -728,10 +688,6 @@ for var in "$@"; do "-f" | "--force" ) forceDelete=true;; "-o" | "--optimize" ) optimize_database=true;; "-h" | "--help" ) helpFunc;; - "-sd" | "--skip-download" ) skipDownload=true;; - "-b" | "--blacklist-only" ) listType="blacklist";; - "-w" | "--whitelist-only" ) listType="whitelist";; - "-wild" | "--wildcard-only" ) listType="wildcard"; dnsRestartType="restart";; esac done @@ -749,54 +705,25 @@ if [[ "${forceDelete:-}" == true ]]; then echo -e "${OVER} ${TICK} ${str}" fi -# Determine which functions to run -if [[ "${skipDownload}" == false ]]; then - # Gravity needs to download blocklists - gravity_CheckDNSResolutionAvailable - gravity_GetBlocklistUrls - if [[ "${haveSourceUrls}" == true ]]; then - gravity_SetDownloadOptions - fi - gravity_ConsolidateDownloadedBlocklists - gravity_SortAndFilterConsolidatedList -else - # Gravity needs to modify Blacklist/Whitelist/Wildcards - echo -e " ${INFO} Using cached Event Horizon list..." - numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") - echo -e " ${INFO} ${COL_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon" +# Gravity downloads blocklists next +gravity_CheckDNSResolutionAvailable +gravity_GetBlocklistUrls +if [[ "${haveSourceUrls}" == true ]]; then +gravity_SetDownloadOptions fi +# Build preEventHorizon +gravity_ConsolidateDownloadedBlocklists +gravity_SortAndFilterConsolidatedList -# Perform when downloading blocklists, or modifying the whitelist -if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "whitelist" ]]; then - gravity_Whitelist -fi - -convert_wildcard_to_regex -gravity_ShowBlockCount - -# Perform when downloading blocklists, or modifying the white/blacklist (not wildcards) -if [[ "${skipDownload}" == false ]] || [[ "${listType}" == *"list" ]]; then - str="Parsing domains" - echo -ne " ${INFO} ${str}..." - - gravity_ParseUserDomains - - # Perform when downloading blocklists - if [[ ! "${listType:-}" == "blacklist" ]]; then - gravity_ParseLocalDomains - gravity_ParseBlacklistDomains - fi - - echo -e "${OVER} ${TICK} ${str}" - - gravity_Cleanup -fi +# Create local.list +gravity_generateLocalList +gravity_ShowCount +gravity_Cleanup echo "" # Determine if DNS has been restarted by this instance of gravity if [[ -z "${dnsWasOffline:-}" ]]; then - # Use "force-reload" when restarting dnsmasq for everything but Wildcards - "${PIHOLE_COMMAND}" restartdns "${dnsRestartType:-force-reload}" + "${PIHOLE_COMMAND}" restartdns reload fi "${PIHOLE_COMMAND}" status From e5d1cb5a2ef11ab4642bd3b8f80cb2edcd9cb2b5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 25 Apr 2019 12:10:42 +0200 Subject: [PATCH 35/86] Rewrite list.sh to operate on database-based lists Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 247 +++++++++++++++++---------------------- 1 file changed, 104 insertions(+), 143 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index c1d95aae..1a2094de 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -11,10 +11,8 @@ # Globals basename=pihole piholeDir=/etc/"${basename}" -whitelist="${piholeDir}"/whitelist.txt -blacklist="${piholeDir}"/blacklist.txt +gravityDBfile="${piholeDir}/gravity.db" -readonly regexlist="/etc/pihole/regex.list" reload=false addmode=true verbose=true @@ -22,35 +20,34 @@ wildcard=false domList=() -listMain="" -listAlt="" +listType="" colfile="/opt/pihole/COL_TABLE" source ${colfile} helpFunc() { - if [[ "${listMain}" == "${whitelist}" ]]; then + if [[ "${listType}" == "whitelist" ]]; then param="w" - type="white" - elif [[ "${listMain}" == "${regexlist}" && "${wildcard}" == true ]]; then + type="whitelist" + elif [[ "${listType}" == "regex" && "${wildcard}" == true ]]; then param="-wild" - type="wildcard black" - elif [[ "${listMain}" == "${regexlist}" ]]; then + type="wildcard blacklist" + elif [[ "${listType}" == "regex" ]]; then param="-regex" - type="regex black" + type="regex filter" else param="b" - type="black" + type="blacklist" fi echo "Usage: pihole -${param} [options] Example: 'pihole -${param} site.com', or 'pihole -${param} site1.com site2.com' -${type^}list one or more domains +${type^} one or more domains Options: - -d, --delmode Remove domain(s) from the ${type}list - -nr, --noreload Update ${type}list without refreshing dnsmasq + -d, --delmode Remove domain(s) from the ${type} + -nr, --noreload Update ${type} without reloading the DNS server -q, --quiet Make output less verbose -h, --help Show this help dialog -l, --list Display all your ${type}listed domains @@ -73,7 +70,7 @@ HandleOther() { # Check validity of domain (don't check for regex entries) if [[ "${#domain}" -le 253 ]]; then - if [[ "${listMain}" == "${regexlist}" && "${wildcard}" == false ]]; then + if [[ "${listType}" == "regex" && "${wildcard}" == false ]]; then validDomain="${domain}" else validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check @@ -88,169 +85,134 @@ HandleOther() { fi } -PoplistFile() { - # Check whitelist file exists, and if not, create it - if [[ ! -f "${whitelist}" ]]; then - touch "${whitelist}" - fi - - # Check blacklist file exists, and if not, create it - if [[ ! -f "${blacklist}" ]]; then - touch "${blacklist}" - fi - +ProcessDomainList() { for dom in "${domList[@]}"; do # Logic: If addmode then add to desired list and remove from the other; if delmode then remove from desired list but do not add to the other if ${addmode}; then - AddDomain "${dom}" "${listMain}" + AddDomain "${dom}" "${listType}" RemoveDomain "${dom}" "${listAlt}" else - RemoveDomain "${dom}" "${listMain}" + RemoveDomain "${dom}" "${listType}" fi done } AddDomain() { + local domain list listname sqlitekey num + domain="$1" list="$2" - domain=$(EscapeRegexp "$1") - [[ "${list}" == "${whitelist}" ]] && listname="whitelist" - [[ "${list}" == "${blacklist}" ]] && listname="blacklist" - - if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then - [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only" - [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only" - bool=true - # Is the domain in the list we want to add it to? - grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false - - if [[ "${bool}" == false ]]; then - # Domain not found in the whitelist file, add it! - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding ${1} to ${listname}..." - fi - reload=true - # Add it to the list we want to add it to - echo "$1" >> "${list}" - else - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} already exists in ${listname}, no need to add!" - fi + if [[ "${list}" == "regex" ]]; then + listname="regex filters" + sqlitekey="filter" + if [[ "${wildcard}" == true ]]; then + domain="(^|\\.)${domain//\./\\.}$" fi - elif [[ "${list}" == "${regexlist}" ]]; then - [[ -z "${type}" ]] && type="--wildcard-only" - bool=true - domain="${1}" + else + # Whitelist / Blacklist + listname="${list}list" + sqlitekey="domain" + fi - [[ "${wildcard}" == true ]] && domain="(^|\\.)${domain//\./\\.}$" + # Is the domain in the list we want to add it to? + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")" - # Is the domain in the list? - # Search only for exactly matching lines - grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false - - if [[ "${bool}" == false ]]; then - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding ${domain} to regex list..." - fi - reload="restart" - echo "$domain" >> "${regexlist}" - else - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${domain} already exists in regex list, no need to add!" - fi + if [[ "${num}" -eq 0 ]]; then + # Domain not found in the file, add it! + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} Adding ${1} to ${listname}..." + fi + reload=true + # Add it to the list we want to add it to + local timestamp + timestamp="$(date --utc +'%s')" + sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (${sqlitekey},enabled,date_added) VALUES (\"${domain}\",1,${timestamp});" + else + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} ${1} already exists in ${listname}, no need to add!" fi fi } RemoveDomain() { + local domain list listname sqlitekey num + domain="$1" list="$2" - domain=$(EscapeRegexp "$1") - [[ "${list}" == "${whitelist}" ]] && listname="whitelist" - [[ "${list}" == "${blacklist}" ]] && listname="blacklist" - - if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then - bool=true - [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only" - [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only" - # Is it in the list? Logic follows that if its whitelisted it should not be blacklisted and vice versa - grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false - if [[ "${bool}" == true ]]; then - # Remove it from the other one - echo -e " ${INFO} Removing $1 from ${listname}..." - # /I flag: search case-insensitive - sed -i "/${domain}/Id" "${list}" - reload=true - else - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} does not exist in ${listname}, no need to remove!" - fi + if [[ "${list}" == "regex" ]]; then + listname="regex filters" + sqlitekey="filter" + if [[ "${wildcard}" == true ]]; then + domain="(^|\\.)${domain//\./\\.}$" fi - elif [[ "${list}" == "${regexlist}" ]]; then - [[ -z "${type}" ]] && type="--wildcard-only" - domain="${1}" + else + # Whitelist / Blacklist + listname="${list}list" + sqlitekey="domain" + fi - [[ "${wildcard}" == true ]] && domain="(^|\\.)${domain//\./\\.}$" + # Is the domain in the list we want to remove it from? + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")" - bool=true - # Is it in the list? - grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false - if [[ "${bool}" == true ]]; then - # Remove it from the other one - echo -e " ${INFO} Removing $domain from regex list..." - local lineNumber - lineNumber=$(grep -Fnx "$domain" "${list}" | cut -f1 -d:) - sed -i "${lineNumber}d" "${list}" - reload=true - else - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${domain} does not exist in regex list, no need to remove!" - fi + if [[ "${num}" -ne 0 ]]; then + # Domain found in the file, remove it! + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} Removing ${1} from ${listname}..." + fi + reload=true + # Remove it from the current list + local timestamp + timestamp="$(date --utc +'%s')" + sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE ${sqlitekey} = \"${domain}\";" + else + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} ${1} does not exist in ${listname}, no need to remove!" fi fi } -# Update Gravity -Reload() { - echo "" - pihole -g --skip-download "${type:-}" -} - Displaylist() { - if [[ -f ${listMain} ]]; then - if [[ "${listMain}" == "${whitelist}" ]]; then - string="gravity resistant domains" - else - string="domains caught in the sinkhole" - fi - verbose=false - echo -e "Displaying $string:\n" - count=1 - while IFS= read -r RD || [ -n "${RD}" ]; do - echo " ${count}: ${RD}" - count=$((count+1)) - done < "${listMain}" + local domain list listname count status + + if [[ "${listType}" == "regex" ]]; then + listname="regex filters list" else - echo -e " ${COL_LIGHT_RED}${listMain} does not exist!${COL_NC}" + # Whitelist / Blacklist + listname="${listType}" + fi + data="$(sqlite3 "${gravityDBfile}" "SELECT * FROM ${listType};" 2> /dev/null)" + + if [[ -z $data ]]; then + echo -e "Not showing empty ${listname}" + else + echo -e "Displaying ${listname}:" + count=1 + while IFS= read -r line + do + domain="$(cut -d'|' -f1 <<< "${line}")" + enabled="$(cut -d'|' -f2 <<< "${line}")" + if [[ "${enabled}" -eq 1 ]]; then + status="enabled" + else + status="disabled" + fi + echo " ${count}: ${domain} (${status})" + count=$((count+1)) + done <<< "${data}" + exit 0; fi - exit 0; } NukeList() { - if [[ -f "${listMain}" ]]; then - # Back up original list - cp "${listMain}" "${listMain}.bck~" - # Empty out file - echo "" > "${listMain}" - fi + sqlite3 "${gravityDBfile}" "DELETE FROM ${listType};" } for var in "$@"; do case "${var}" in - "-w" | "whitelist" ) listMain="${whitelist}"; listAlt="${blacklist}";; - "-b" | "blacklist" ) listMain="${blacklist}"; listAlt="${whitelist}";; - "--wild" | "wildcard" ) listMain="${regexlist}"; wildcard=true;; - "--regex" | "regex" ) listMain="${regexlist}";; + "-w" | "whitelist" ) listType="whitelist"; listAlt="blacklist";; + "-b" | "blacklist" ) listType="blacklist"; listAlt="whitelist";; + "--wild" | "wildcard" ) listType="regex"; wildcard=true;; + "--regex" | "regex" ) listType="regex";; "-nr"| "--noreload" ) reload=false;; "-d" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; @@ -267,9 +229,8 @@ if [[ $# = 0 ]]; then helpFunc fi -PoplistFile +ProcessDomainList if [[ "${reload}" != false ]]; then - # Ensure that "restart" is used for Wildcard updates - Reload "${reload}" + pihole restartdns reload fi From 788cd783212828229fb8708f38841b4a89472267 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 25 Apr 2019 12:30:38 +0200 Subject: [PATCH 36/86] Reduce code duplication and add special handling of | character as it might appear in regex filter string Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 70 +++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 1a2094de..727919fe 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -21,9 +21,11 @@ wildcard=false domList=() listType="" +listname="" +sqlitekey="" -colfile="/opt/pihole/COL_TABLE" -source ${colfile} +# shellcheck source=/opt/pihole/COL_TABLE +source "/opt/pihole/COL_TABLE" helpFunc() { @@ -61,7 +63,7 @@ EscapeRegexp() { # string in our regular expressions # This sed is intentionally executed in three steps to ease maintainability # The first sed removes any amount of leading dots - echo $* | sed 's/^\.*//' | sed "s/[]\.|$(){}?+*^]/\\\\&/g" | sed "s/\\//\\\\\//g" + echo "$@" | sed 's/^\.*//' | sed "s/[]\.|$(){}?+*^]/\\\\&/g" | sed "s/\\//\\\\\//g" } HandleOther() { @@ -86,11 +88,28 @@ HandleOther() { } ProcessDomainList() { + if [[ "${listType}" == "regex" ]]; then + # Regex filter list + listname="regex filters" + sqlitekey="filter" + else + # Whitelist / Blacklist + listname="${listType}" + sqlitekey="domain" + fi + for dom in "${domList[@]}"; do - # Logic: If addmode then add to desired list and remove from the other; if delmode then remove from desired list but do not add to the other + # Format domain into regex filter if requested + if [[ "${wildcard}" == true ]]; then + dom="(^|\\.)${dom//\./\\.}$" + fi + + # Logic: If addmode then add to desired list and remove from the other; if delmode then remove from desired list but do not add to the othe if ${addmode}; then AddDomain "${dom}" "${listType}" - RemoveDomain "${dom}" "${listAlt}" + if [[ ! "${listType}" == "regex" ]]; then + RemoveDomain "${dom}" "${listAlt}" + fi else RemoveDomain "${dom}" "${listType}" fi @@ -98,22 +117,10 @@ ProcessDomainList() { } AddDomain() { - local domain list listname sqlitekey num + local domain list num domain="$1" list="$2" - if [[ "${list}" == "regex" ]]; then - listname="regex filters" - sqlitekey="filter" - if [[ "${wildcard}" == true ]]; then - domain="(^|\\.)${domain//\./\\.}$" - fi - else - # Whitelist / Blacklist - listname="${list}list" - sqlitekey="domain" - fi - # Is the domain in the list we want to add it to? num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")" @@ -135,22 +142,10 @@ AddDomain() { } RemoveDomain() { - local domain list listname sqlitekey num + local domain list num domain="$1" list="$2" - if [[ "${list}" == "regex" ]]; then - listname="regex filters" - sqlitekey="filter" - if [[ "${wildcard}" == true ]]; then - domain="(^|\\.)${domain//\./\\.}$" - fi - else - # Whitelist / Blacklist - listname="${list}list" - sqlitekey="domain" - fi - # Is the domain in the list we want to remove it from? num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")" @@ -172,7 +167,7 @@ RemoveDomain() { } Displaylist() { - local domain list listname count status + local list listname count num_pipes domain enabled status if [[ "${listType}" == "regex" ]]; then listname="regex filters list" @@ -189,13 +184,20 @@ Displaylist() { count=1 while IFS= read -r line do - domain="$(cut -d'|' -f1 <<< "${line}")" - enabled="$(cut -d'|' -f2 <<< "${line}")" + # Count number of pipes seen in this line + # This is necessary because we can only detect the pipe separating the fields + # from the end backwards as the domain (which is the first field) may contain + # pipe symbols as they are perfectly valid regex filter control characters + num_pipes="$(grep -c "^" <<< "$(grep -o "|" <<< "${line}")")" + + domain="$(cut -d'|' -f"-$((num_pipes-2))" <<< "${line}")" + enabled="$(cut -d'|' -f$((num_pipes-1)) <<< "${line}")" if [[ "${enabled}" -eq 1 ]]; then status="enabled" else status="disabled" fi + echo " ${count}: ${domain} (${status})" count=$((count+1)) done <<< "${data}" From 43bced7997772d5604d1302e1c8fbc6369ec45e1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 25 Apr 2019 12:45:08 +0200 Subject: [PATCH 37/86] Add more comments Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 727919fe..5fd24f6b 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -24,8 +24,8 @@ listType="" listname="" sqlitekey="" -# shellcheck source=/opt/pihole/COL_TABLE -source "/opt/pihole/COL_TABLE" +colfile="/opt/pihole/COL_TABLE" +source ${colfile} helpFunc() { @@ -63,7 +63,7 @@ EscapeRegexp() { # string in our regular expressions # This sed is intentionally executed in three steps to ease maintainability # The first sed removes any amount of leading dots - echo "$@" | sed 's/^\.*//' | sed "s/[]\.|$(){}?+*^]/\\\\&/g" | sed "s/\\//\\\\\//g" + echo $* | sed 's/^\.*//' | sed "s/[]\.|$(){}?+*^]/\\\\&/g" | sed "s/\\//\\\\\//g" } HandleOther() { @@ -190,8 +190,11 @@ Displaylist() { # pipe symbols as they are perfectly valid regex filter control characters num_pipes="$(grep -c "^" <<< "$(grep -o "|" <<< "${line}")")" + # Extract domain and enabled status based on the obtained number of pipe characters domain="$(cut -d'|' -f"-$((num_pipes-2))" <<< "${line}")" - enabled="$(cut -d'|' -f$((num_pipes-1)) <<< "${line}")" + enabled="$(cut -d'|' -f"$((num_pipes-1))" <<< "${line}")" + + # Translate boolean status into human readable string if [[ "${enabled}" -eq 1 ]]; then status="enabled" else From cce66a13b69ece62037b9dd2a01cf4aab18a3abe Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 25 Apr 2019 14:54:41 +0200 Subject: [PATCH 38/86] Fix output in list.sh when adding a domain that is already present in the target list Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 5fd24f6b..1ad99468 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -161,7 +161,7 @@ RemoveDomain() { sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE ${sqlitekey} = \"${domain}\";" else if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} does not exist in ${listname}, no need to remove!" + echo -e " ${INFO} ${1} does not exist in ${list}, no need to remove!" fi fi } From a2a7e8000795d2b58071d7393b86cf2bc3e894fc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 25 Apr 2019 15:01:13 +0200 Subject: [PATCH 39/86] Add --web for sending a confirmation string when executed from the web interface. This is necessary as the "Reloading DNS service" message does not always propagate through to the web interface when reloading pihole-FTL takes a while. Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 1ad99468..969e7e44 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -17,6 +17,7 @@ reload=false addmode=true verbose=true wildcard=false +web=false domList=() @@ -224,6 +225,7 @@ for var in "$@"; do "-h" | "--help" ) helpFunc;; "-l" | "--list" ) Displaylist;; "--nuke" ) NukeList;; + "--web" ) web=true;; * ) HandleOther "${var}";; esac done @@ -236,6 +238,11 @@ fi ProcessDomainList +# Used on web interface +if $web; then +echo "DONE" +fi + if [[ "${reload}" != false ]]; then pihole restartdns reload fi From 0efc46260f0726edf0e2986df536c0f4e0c93fdc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 16:16:30 +0200 Subject: [PATCH 40/86] Use both old and new strings for detecting whether this is a Pi-hole dnsmasq config file Signed-off-by: DL6ER --- automated install/basic-install.sh | 6 ++++-- gravity.sh | 8 +++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 6bcfb564..06facdf7 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1224,7 +1224,8 @@ version_check_dnsmasq() { # Local, named variables local dnsmasq_conf="/etc/dnsmasq.conf" local dnsmasq_conf_orig="/etc/dnsmasq.conf.orig" - local dnsmasq_pihole_id_string="# Dnsmasq config for Pi-hole's FTLDNS" + local dnsmasq_pihole_id_string="addn-hosts=/etc/pihole/gravity.list" + local dnsmasq_pihole_id_string2="# Dnsmasq config for Pi-hole's FTLDNS" local dnsmasq_original_config="${PI_HOLE_LOCAL_REPO}/advanced/dnsmasq.conf.original" local dnsmasq_pihole_01_snippet="${PI_HOLE_LOCAL_REPO}/advanced/01-pihole.conf" local dnsmasq_pihole_01_location="/etc/dnsmasq.d/01-pihole.conf" @@ -1233,7 +1234,8 @@ version_check_dnsmasq() { if [[ -f "${dnsmasq_conf}" ]]; then printf " %b Existing dnsmasq.conf found..." "${INFO}" # If a specific string is found within this file, we presume it's from older versions on Pi-hole, - if grep -q ${dnsmasq_pihole_id_string} ${dnsmasq_conf}; then + if grep -q "${dnsmasq_pihole_id_string}" "${dnsmasq_conf}" || + grep -q "${dnsmasq_pihole_id_string2}" "${dnsmasq_conf}"; then printf " it is from a previous Pi-hole install.\\n" printf " %b Backing up dnsmasq.conf to dnsmasq.conf.orig..." "${INFO}" # so backup the original file diff --git a/gravity.sh b/gravity.sh index 09e7b19c..75057eec 100755 --- a/gravity.sh +++ b/gravity.sh @@ -20,8 +20,9 @@ source "${regexconverter}" basename="pihole" PIHOLE_COMMAND="/usr/local/bin/${basename}" -PIHOLE_USER="pihole" -PIHOLE_GROUP="pihole" +DATABASE_USER="pihole" +DATABASE_GROUP="www-data" +DATABASE_PERMISSIONS="0660" piholeDir="/etc/${basename}" @@ -86,7 +87,8 @@ fi # Generate new sqlite3 file from schema template generate_gravity_database() { sqlite3 "${gravityDBfile}" < "${gravityDBschema}" - chown $PIHOLE_USER:$PIHOLE_GROUP "${gravityDBfile}" + chown ${DATABASE_USER}:${DATABASE_GROUP} "${piholeDir}" "${gravityDBfile}" + chmod ${DATABASE_PERMISSIONS} "${piholeDir}" "${gravityDBfile}" } # Import domains from file and store them in the specified database table From 08ee9526adde9fa3bd18cffa8b58ef148c2ccc52 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 17:06:01 +0200 Subject: [PATCH 41/86] Review comments Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 76 +++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 969e7e44..af44013e 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -105,7 +105,8 @@ ProcessDomainList() { dom="(^|\\.)${dom//\./\\.}$" fi - # Logic: If addmode then add to desired list and remove from the other; if delmode then remove from desired list but do not add to the othe + # Logic: If addmode then add to desired list and remove from the other; + # if delmode then remove from desired list but do not add to the other if ${addmode}; then AddDomain "${dom}" "${listType}" if [[ ! "${listType}" == "regex" ]]; then @@ -119,63 +120,64 @@ ProcessDomainList() { AddDomain() { local domain list num - domain="$1" + # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input + domain="$(printf "%q" "$1")" list="$2" # Is the domain in the list we want to add it to? num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")" - if [[ "${num}" -eq 0 ]]; then - # Domain not found in the file, add it! - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding ${1} to ${listname}..." - fi - reload=true - # Add it to the list we want to add it to - local timestamp - timestamp="$(date --utc +'%s')" - sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (${sqlitekey},enabled,date_added) VALUES (\"${domain}\",1,${timestamp});" - else - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} already exists in ${listname}, no need to add!" - fi + if [[ "${num}" -ne 0 ]]; then + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} ${1} already exists in ${listname}, no need to add!" + fi + return fi + + # Domain not found in the table, add it! + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} Adding ${1} to ${listname}..." + fi + reload=true + # Add it to the list we want to add it to + local timestamp + timestamp="$(date --utc +'%s')" + # Insert only domain and date_added here. The enabled fields will be filled + # with its default value is true. + sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (${sqlitekey},date_added) VALUES (\"${domain}\",${timestamp});" } RemoveDomain() { local domain list num - domain="$1" + # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input + domain="$(printf "%q" "$1")" list="$2" # Is the domain in the list we want to remove it from? num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")" - if [[ "${num}" -ne 0 ]]; then - # Domain found in the file, remove it! - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Removing ${1} from ${listname}..." - fi - reload=true - # Remove it from the current list - local timestamp - timestamp="$(date --utc +'%s')" - sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE ${sqlitekey} = \"${domain}\";" - else - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} does not exist in ${list}, no need to remove!" - fi + if [[ "${num}" -eq 0 ]]; then + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} ${1} does not exist in ${list}, no need to remove!" + fi + return fi + + # Domain found in the table, remove it! + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} Removing ${1} from ${listname}..." + fi + reload=true + # Remove it from the current list + local timestamp + timestamp="$(date --utc +'%s')" + sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE ${sqlitekey} = \"${domain}\";" } Displaylist() { local list listname count num_pipes domain enabled status - if [[ "${listType}" == "regex" ]]; then - listname="regex filters list" - else - # Whitelist / Blacklist - listname="${listType}" - fi + listname="${listType}" data="$(sqlite3 "${gravityDBfile}" "SELECT * FROM ${listType};" 2> /dev/null)" if [[ -z $data ]]; then From 6b1d2523e8bea9310e5095208f0d4732cef14cb9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 17:07:16 +0200 Subject: [PATCH 42/86] Also print datetime a domain was added when listing the domains on the CLI Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index af44013e..d11ab96c 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -175,7 +175,7 @@ RemoveDomain() { } Displaylist() { - local list listname count num_pipes domain enabled status + local list listname count num_pipes domain enabled status nicedate listname="${listType}" data="$(sqlite3 "${gravityDBfile}" "SELECT * FROM ${listType};" 2> /dev/null)" @@ -196,7 +196,9 @@ Displaylist() { # Extract domain and enabled status based on the obtained number of pipe characters domain="$(cut -d'|' -f"-$((num_pipes-2))" <<< "${line}")" enabled="$(cut -d'|' -f"$((num_pipes-1))" <<< "${line}")" + dateadded="$(cut -d'|' -f"$((num_pipes))" <<< "${line}")" + echo "${dateadded}" # Translate boolean status into human readable string if [[ "${enabled}" -eq 1 ]]; then status="enabled" @@ -204,7 +206,10 @@ Displaylist() { status="disabled" fi - echo " ${count}: ${domain} (${status})" + # Get nice representation of numerical date stored in database + nicedate=$(date --rfc-2822 -d "@${dateadded}") + + echo " ${count}: ${domain} (${status}, added ${nicedate})" count=$((count+1)) done <<< "${data}" exit 0; From a891d6439533f64a42baacdb409df4725ac7461e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 17:13:36 +0200 Subject: [PATCH 43/86] We should always exit after displaying a list, even if it is empty Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index d11ab96c..cd9dce52 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -212,8 +212,8 @@ Displaylist() { echo " ${count}: ${domain} (${status}, added ${nicedate})" count=$((count+1)) done <<< "${data}" - exit 0; fi + exit 0; } NukeList() { From 5bb816367422c5c6394543ff88ed8b22f31cf60f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 17:17:21 +0200 Subject: [PATCH 44/86] Change type of whitelist, blacklist, regex, and adlists time column to INTEGER and insert a DEFAULT clause that returns the current Unix time as such an integer Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 5442987f..cd62c94e 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,7 +1,7 @@ -CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); -CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); -CREATE TABLE regex (filter TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); -CREATE TABLE adlists (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); +CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); +CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); +CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); +CREATE TABLE adlists (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); CREATE TABLE info (property TEXT NOT NULL, value TEXT NOT NULL); From 53e3ff2b2421a491badb24be578717e3bc11e485 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 17:32:05 +0200 Subject: [PATCH 45/86] Only add non-empty lines when moving file content into database tables Signed-off-by: DL6ER --- gravity.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 75057eec..1d94424f 100755 --- a/gravity.sh +++ b/gravity.sh @@ -125,7 +125,10 @@ database_table_from_file() { # Read file line by line grep -v '^ *#' < "${source}" | while IFS= read -r domain do - echo "\"${domain}\",1,${timestamp}" >> "${tmpFile}" + # Only add non-empty lines + if [[ ! -z "${domain}" ]]; then + echo "\"${domain}\",1,${timestamp}" >> "${tmpFile}" + fi done inputfile="${tmpFile}" fi From bd0215524ba0191b5364a596e769027a22be88fd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 17:36:00 +0200 Subject: [PATCH 46/86] Add vw_adlists view which returns only enabled adlists Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 6 +++++- gravity.sh | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index cd62c94e..ebe97955 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -20,6 +20,10 @@ CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain FROM whitelist a WHERE a.enabled == 1; -CREATE VIEW vw_regex AS SELECT DISTINCT a.filter +CREATE VIEW vw_regex AS SELECT DISTINCT a.domain FROM regex a WHERE a.enabled == 1; + +CREATE VIEW vw_adlists AS SELECT DISTINCT a.address +FROM adlists a +WHERE a.enabled == 1; diff --git a/gravity.sh b/gravity.sh index 1d94424f..a0652ced 100755 --- a/gravity.sh +++ b/gravity.sh @@ -238,7 +238,7 @@ gravity_GetBlocklistUrls() { # Retrieve source URLs from gravity database # We source only enabled adlists, sqlite3 stores boolean values as 0 (false) or 1 (true) - mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM adlists WHERE enabled = 1;" 2> /dev/null)" + mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)" # Parse source domains from $sources mapfile -t sourceDomains <<< "$( From 3dcb9722d35fb42d8a63903d5808a493e0b8507c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 17:39:35 +0200 Subject: [PATCH 47/86] Remove global haveSourceUrls variable and just skip downloading and processing adlists if there aren't any to be downloaded Signed-off-by: DL6ER --- gravity.sh | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/gravity.sh b/gravity.sh index a0652ced..3543ac15 100755 --- a/gravity.sh +++ b/gravity.sh @@ -47,8 +47,6 @@ preEventHorizon="list.preEventHorizon" resolver="pihole-FTL" -haveSourceUrls=true - # Source setupVars from install script setupVars="${piholeDir}/setupVars.conf" if [[ -f "${setupVars}" ]];then @@ -255,11 +253,12 @@ gravity_GetBlocklistUrls() { if [[ -n "${sources[*]}" ]] && [[ -n "${sourceDomains[*]}" ]]; then echo -e "${OVER} ${TICK} ${str}" + return 0 else echo -e "${OVER} ${CROSS} ${str}" echo -e " ${INFO} No source list found, or it is empty" echo "" - haveSourceUrls=false + return 1 fi } @@ -503,9 +502,7 @@ gravity_ConsolidateDownloadedBlocklists() { local str lastLine str="Consolidating blocklists" - if [[ "${haveSourceUrls}" == true ]]; then - echo -ne " ${INFO} ${str}..." - fi + echo -ne " ${INFO} ${str}..." # Empty $matterAndLight if it already exists, otherwise, create it : > "${piholeDir}/${matterAndLight}" @@ -524,9 +521,8 @@ gravity_ConsolidateDownloadedBlocklists() { fi fi done - if [[ "${haveSourceUrls}" == true ]]; then - echo -e "${OVER} ${TICK} ${str}" - fi + echo -e "${OVER} ${TICK} ${str}" + } # Parse consolidated list into (filtered, unique) domains-only format @@ -534,18 +530,15 @@ gravity_SortAndFilterConsolidatedList() { local str num str="Extracting domains from blocklists" - if [[ "${haveSourceUrls}" == true ]]; then - echo -ne " ${INFO} ${str}..." - fi + echo -ne " ${INFO} ${str}..." # Parse into file gravity_ParseFileIntoDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${parsedMatter}" # Format $parsedMatter line total as currency num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${parsedMatter}")") - if [[ "${haveSourceUrls}" == true ]]; then - echo -e "${OVER} ${TICK} ${str}" - fi + + echo -e "${OVER} ${TICK} ${str}" echo -e " ${INFO} Gravity pulled in ${COL_BLUE}${num}${COL_NC} domains" str="Removing duplicate domains" @@ -713,12 +706,12 @@ fi # Gravity downloads blocklists next gravity_CheckDNSResolutionAvailable gravity_GetBlocklistUrls -if [[ "${haveSourceUrls}" == true ]]; then -gravity_SetDownloadOptions +if gravity_GetBlocklistUrls; then + gravity_SetDownloadOptions + # Build preEventHorizon + gravity_ConsolidateDownloadedBlocklists + gravity_SortAndFilterConsolidatedList fi -# Build preEventHorizon -gravity_ConsolidateDownloadedBlocklists -gravity_SortAndFilterConsolidatedList # Create local.list gravity_generateLocalList From e04dc9d2bb20400912067d1ff39c6dcc2873f76c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 17:44:42 +0200 Subject: [PATCH 48/86] Remove sqlitekey variable as the first column of the regex table is now called domain as well Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index cd9dce52..dcd6f580 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -23,7 +23,6 @@ domList=() listType="" listname="" -sqlitekey="" colfile="/opt/pihole/COL_TABLE" source ${colfile} @@ -92,11 +91,9 @@ ProcessDomainList() { if [[ "${listType}" == "regex" ]]; then # Regex filter list listname="regex filters" - sqlitekey="filter" else # Whitelist / Blacklist listname="${listType}" - sqlitekey="domain" fi for dom in "${domList[@]}"; do @@ -125,7 +122,7 @@ AddDomain() { list="$2" # Is the domain in the list we want to add it to? - num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")" + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = \"${domain}\";")" if [[ "${num}" -ne 0 ]]; then if [[ "${verbose}" == true ]]; then @@ -144,7 +141,7 @@ AddDomain() { timestamp="$(date --utc +'%s')" # Insert only domain and date_added here. The enabled fields will be filled # with its default value is true. - sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (${sqlitekey},date_added) VALUES (\"${domain}\",${timestamp});" + sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain,date_added) VALUES (\"${domain}\",${timestamp});" } RemoveDomain() { @@ -154,7 +151,7 @@ RemoveDomain() { list="$2" # Is the domain in the list we want to remove it from? - num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")" + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = \"${domain}\";")" if [[ "${num}" -eq 0 ]]; then if [[ "${verbose}" == true ]]; then @@ -171,7 +168,7 @@ RemoveDomain() { # Remove it from the current list local timestamp timestamp="$(date --utc +'%s')" - sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE ${sqlitekey} = \"${domain}\";" + sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE domain = \"${domain}\";" } Displaylist() { From 8524aecfed5eb4e3984fe938d2049c47976741c7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 17:56:11 +0200 Subject: [PATCH 49/86] Don't set gravity database permissions Signed-off-by: DL6ER --- gravity.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index 3543ac15..98db04b8 100755 --- a/gravity.sh +++ b/gravity.sh @@ -20,9 +20,6 @@ source "${regexconverter}" basename="pihole" PIHOLE_COMMAND="/usr/local/bin/${basename}" -DATABASE_USER="pihole" -DATABASE_GROUP="www-data" -DATABASE_PERMISSIONS="0660" piholeDir="/etc/${basename}" @@ -85,8 +82,6 @@ fi # Generate new sqlite3 file from schema template generate_gravity_database() { sqlite3 "${gravityDBfile}" < "${gravityDBschema}" - chown ${DATABASE_USER}:${DATABASE_GROUP} "${piholeDir}" "${gravityDBfile}" - chmod ${DATABASE_PERMISSIONS} "${piholeDir}" "${gravityDBfile}" } # Import domains from file and store them in the specified database table From 3fe43ce1d923f825b73a4f9c367b3a7f1bf344c9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 18:15:53 +0200 Subject: [PATCH 50/86] Rely on default value (database provided) for date_added when adding a new domain Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index dcd6f580..053def02 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -133,15 +133,12 @@ AddDomain() { # Domain not found in the table, add it! if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding ${1} to ${listname}..." + echo -e " ${INFO} Adding ${1} to the ${listname}..." fi reload=true - # Add it to the list we want to add it to - local timestamp - timestamp="$(date --utc +'%s')" - # Insert only domain and date_added here. The enabled fields will be filled - # with its default value is true. - sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain,date_added) VALUES (\"${domain}\",${timestamp});" + # Insert only the domain here. The enabled and date_added fields will be filled + # with their default values (enabled = true, date_added = current timestamp) + sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain) VALUES (\"${domain}\");" } RemoveDomain() { @@ -162,12 +159,10 @@ RemoveDomain() { # Domain found in the table, remove it! if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Removing ${1} from ${listname}..." + echo -e " ${INFO} Removing ${1} from the ${listname}..." fi reload=true # Remove it from the current list - local timestamp - timestamp="$(date --utc +'%s')" sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE domain = \"${domain}\";" } @@ -195,7 +190,6 @@ Displaylist() { enabled="$(cut -d'|' -f"$((num_pipes-1))" <<< "${line}")" dateadded="$(cut -d'|' -f"$((num_pipes))" <<< "${line}")" - echo "${dateadded}" # Translate boolean status into human readable string if [[ "${enabled}" -eq 1 ]]; then status="enabled" From b4ae142149f60150d583ec49876dfdef7dd3d259 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 20:54:01 +0200 Subject: [PATCH 51/86] Remove printf escaping (we will realize it differently) and ensure we're using single quotes for strings (although double quotes are possible, too) Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 053def02..374f888b 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -118,11 +118,11 @@ ProcessDomainList() { AddDomain() { local domain list num # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input - domain="$(printf "%q" "$1")" + domain="$1" list="$2" # Is the domain in the list we want to add it to? - num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = \"${domain}\";")" + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = '${domain}';")" if [[ "${num}" -ne 0 ]]; then if [[ "${verbose}" == true ]]; then @@ -138,17 +138,17 @@ AddDomain() { reload=true # Insert only the domain here. The enabled and date_added fields will be filled # with their default values (enabled = true, date_added = current timestamp) - sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain) VALUES (\"${domain}\");" + sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain) VALUES ('${domain}');" } RemoveDomain() { local domain list num # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input - domain="$(printf "%q" "$1")" + domain="$1" list="$2" # Is the domain in the list we want to remove it from? - num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = \"${domain}\";")" + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = '${domain}';")" if [[ "${num}" -eq 0 ]]; then if [[ "${verbose}" == true ]]; then @@ -163,7 +163,7 @@ RemoveDomain() { fi reload=true # Remove it from the current list - sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE domain = \"${domain}\";" + sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE domain = '${domain}';" } Displaylist() { From 126f7101ffaa56b42891d9db1bed12e270222bd0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 28 Apr 2019 14:42:52 +0200 Subject: [PATCH 52/86] Add (optional) date_modified field and use vw_whitelist view in vw_blacklist Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 6 +++--- advanced/Templates/gravity.db.sql | 17 ++++++++--------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 374f888b..73cb2878 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -186,9 +186,9 @@ Displaylist() { num_pipes="$(grep -c "^" <<< "$(grep -o "|" <<< "${line}")")" # Extract domain and enabled status based on the obtained number of pipe characters - domain="$(cut -d'|' -f"-$((num_pipes-2))" <<< "${line}")" - enabled="$(cut -d'|' -f"$((num_pipes-1))" <<< "${line}")" - dateadded="$(cut -d'|' -f"$((num_pipes))" <<< "${line}")" + domain="$(cut -d'|' -f"-$((num_pipes-3))" <<< "${line}")" + enabled="$(cut -d'|' -f"$((num_pipes-2))" <<< "${line}")" + dateadded="$(cut -d'|' -f"$((num_pipes-1))" <<< "${line}")" # Translate boolean status into human readable string if [[ "${enabled}" -eq 1 ]]; then diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index ebe97955..31fbfd9f 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,7 +1,7 @@ -CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); -CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); -CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); -CREATE TABLE adlists (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); +CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT); +CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT); +CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT); +CREATE TABLE adlists (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); CREATE TABLE info (property TEXT NOT NULL, value TEXT NOT NULL); @@ -11,15 +11,14 @@ CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain FROM gravity a WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1); -CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain -FROM blacklist a -WHERE a.enabled == 1 AND - a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1); - CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain FROM whitelist a WHERE a.enabled == 1; +CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain +FROM blacklist a +WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist; + CREATE VIEW vw_regex AS SELECT DISTINCT a.domain FROM regex a WHERE a.enabled == 1; From efeba594ae791000df0461b00e82e02251892038 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 28 Apr 2019 21:39:06 +0200 Subject: [PATCH 53/86] Add support for manipulating adlists from the CLI Signed-off-by: DL6ER --- advanced/Scripts/webpage.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index c6852896..6ac351bc 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -17,6 +17,8 @@ readonly FTLconf="/etc/pihole/pihole-FTL.conf" # 03 -> wildcards readonly dhcpstaticconfig="/etc/dnsmasq.d/04-pihole-static-dhcp.conf" +readonly gravityDBfile="/etc/pihole/gravity.db" + coltable="/opt/pihole/COL_TABLE" if [[ -f ${coltable} ]]; then source ${coltable} @@ -385,19 +387,17 @@ SetWebUILayout() { } CustomizeAdLists() { - list="/etc/pihole/adlists.list" + local address + address="${args[3]}" if [[ "${args[2]}" == "enable" ]]; then - sed -i "\\@${args[3]}@s/^#http/http/g" "${list}" + sqlite3 "${gravityDBfile}" "UPDATE adlists SET enabled = 1 WHERE address = '${address}'" elif [[ "${args[2]}" == "disable" ]]; then - sed -i "\\@${args[3]}@s/^http/#http/g" "${list}" + sqlite3 "${gravityDBfile}" "UPDATE adlists SET enabled = 0 WHERE address = '${address}'" elif [[ "${args[2]}" == "add" ]]; then - if [[ $(grep -c "^${args[3]}$" "${list}") -eq 0 ]] ; then - echo "${args[3]}" >> ${list} - fi + sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlists (address) VALUES ('${address}')" elif [[ "${args[2]}" == "del" ]]; then - var=$(echo "${args[3]}" | sed 's/\//\\\//g') - sed -i "/${var}/Id" "${list}" + sqlite3 "${gravityDBfile}" "DELETE FROM adlists WHERE address = '${address}'" else echo "Not permitted" return 1 From 4946b5907bf4d45fda807558e2e0234ed70db784 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 28 Apr 2019 22:15:47 +0200 Subject: [PATCH 54/86] Move migrated files to backup directory instead of deleting them. This has the benefit that gravity.db can be recreated at any time by: 1. deleting gravity.db, 2. copying all files from the migration_backup directory back to /etc/pihole, 3. running pihole -g again. Signed-off-by: DL6ER --- gravity.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index a3456fb4..6e238e68 100755 --- a/gravity.sh +++ b/gravity.sh @@ -89,6 +89,8 @@ database_table_from_file() { # Define locals local table="${1}" local source="${2}" + local backup_path="${piholeDir}/migration_backup" + local backup_file="${backup_path}/$(basename "${2}")" # Create database file if not present if [ ! -e "${gravityDBfile}" ]; then @@ -140,8 +142,9 @@ database_table_from_file() { rm "${tmpFile}" > /dev/null 2>&1 || \ echo -e " ${CROSS} Unable to remove ${tmpFile}" - # Delete source file - rm "${source}" 2> /dev/null || \ + # Move source file to backup directory, create directory if not existing + mkdir -p "${backup_path}" + mv "${source}" "${backup_file}" 2> /dev/null || \ echo -e " ${CROSS} Unable to remove ${source}" } @@ -702,7 +705,6 @@ fi # Gravity downloads blocklists next gravity_CheckDNSResolutionAvailable -gravity_GetBlocklistUrls if gravity_GetBlocklistUrls; then gravity_SetDownloadOptions # Build preEventHorizon From a932209143af5eb9e3acd4991a5bb203375ba461 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 29 Apr 2019 17:19:02 +0200 Subject: [PATCH 55/86] Set default for date_modified to be the current time Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 8 ++++---- gravity.sh | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 31fbfd9f..c1066f33 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,7 +1,7 @@ -CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT); -CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT); -CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT); -CREATE TABLE adlists (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT); +CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); +CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); +CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); +CREATE TABLE adlists (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); CREATE TABLE info (property TEXT NOT NULL, value TEXT NOT NULL); diff --git a/gravity.sh b/gravity.sh index 6e238e68..331b632d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -122,7 +122,7 @@ database_table_from_file() { do # Only add non-empty lines if [[ ! -z "${domain}" ]]; then - echo "\"${domain}\",1,${timestamp}" >> "${tmpFile}" + echo "\"${domain}\",1,${timestamp},${timestamp}" >> "${tmpFile}" fi done inputfile="${tmpFile}" @@ -145,7 +145,7 @@ database_table_from_file() { # Move source file to backup directory, create directory if not existing mkdir -p "${backup_path}" mv "${source}" "${backup_file}" 2> /dev/null || \ - echo -e " ${CROSS} Unable to remove ${source}" + echo -e " ${CROSS} Unable to backup ${source} to ${backup_path}" } # Migrate pre-v5.0 list files to database-based Pi-hole versions From 2180531a84516f6720985f034fc8d0cdb589e6a7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 29 Apr 2019 19:43:35 +0200 Subject: [PATCH 56/86] Add trigger for automatic updates of the date_modified column and improve formating in the gravity.db template Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 37 ++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index c1066f33..dc0f23ed 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -8,21 +8,38 @@ CREATE TABLE info (property TEXT NOT NULL, value TEXT NOT NULL); INSERT INTO info VALUES("version","1"); CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain -FROM gravity a -WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1); + FROM gravity a + WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1); CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain -FROM whitelist a -WHERE a.enabled == 1; + FROM whitelist a + WHERE a.enabled == 1; +CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist + BEGIN + UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain -FROM blacklist a -WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist; + FROM blacklist a + WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist; +CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist + BEGIN + UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; CREATE VIEW vw_regex AS SELECT DISTINCT a.domain -FROM regex a -WHERE a.enabled == 1; + FROM regex a + WHERE a.enabled == 1; +CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex + BEGIN + UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; CREATE VIEW vw_adlists AS SELECT DISTINCT a.address -FROM adlists a -WHERE a.enabled == 1; + FROM adlists a + WHERE a.enabled == 1; +CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists + BEGIN + UPDATE adlists SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address; + END; + From 5c575e73c7e3d9efd84a86740d7c09b0316089ff Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Tue, 30 Apr 2019 22:41:12 +0200 Subject: [PATCH 57/86] Adds world readable attribute to files created by Pi-hole to circumvent #2724 Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- advanced/Scripts/list.sh | 5 ++++- advanced/Scripts/piholeCheckout.sh | 3 +++ advanced/Scripts/piholeLogFlush.sh | 4 +++- advanced/Scripts/updatecheck.sh | 3 +++ advanced/Scripts/webpage.sh | 3 +++ automated install/basic-install.sh | 35 +++++++++++++++++++++++++----- automated install/uninstall.sh | 2 +- gravity.sh | 8 ++++++- pihole | 4 ++++ 9 files changed, 58 insertions(+), 9 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index c1d95aae..e0455e0c 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -92,11 +92,13 @@ PoplistFile() { # Check whitelist file exists, and if not, create it if [[ ! -f "${whitelist}" ]]; then touch "${whitelist}" + chmod a+r "${whitelist}" fi # Check blacklist file exists, and if not, create it if [[ ! -f "${blacklist}" ]]; then touch "${blacklist}" + chmod a+r "${blacklist}" fi for dom in "${domList[@]}"; do @@ -239,9 +241,10 @@ Displaylist() { NukeList() { if [[ -f "${listMain}" ]]; then # Back up original list - cp "${listMain}" "${listMain}.bck~" + cp -p "${listMain}" "${listMain}.bck~" # Empty out file echo "" > "${listMain}" + chmod a+r "${listMain}" fi } diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index 1bfe5e21..a982289f 100644 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -90,6 +90,7 @@ checkout() { local path path="development/${binary}" echo "development" > /etc/pihole/ftlbranch + chmod a+r /etc/pihole/ftlbranch elif [[ "${1}" == "master" ]] ; then # Shortcut to check out master branches echo -e " ${INFO} Shortcut \"master\" detected - checking out master branches..." @@ -104,6 +105,7 @@ checkout() { local path path="master/${binary}" echo "master" > /etc/pihole/ftlbranch + chmod a+r /etc/pihole/ftlbranch elif [[ "${1}" == "core" ]] ; then str="Fetching branches from ${piholeGitUrl}" echo -ne " ${INFO} $str" @@ -166,6 +168,7 @@ checkout() { if check_download_exists "$path"; then echo " ${TICK} Branch ${2} exists" echo "${2}" > /etc/pihole/ftlbranch + chmod a+r /etc/pihole/ftlbranch FTLinstall "${binary}" restart_service pihole-FTL enable_service pihole-FTL diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 561fbce7..1e4b7abe 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -39,8 +39,9 @@ if [[ "$@" == *"once"* ]]; then # Note that moving the file is not an option, as # dnsmasq would happily continue writing into the # moved file (it will have the same file handler) - cp /var/log/pihole.log /var/log/pihole.log.1 + cp -p /var/log/pihole.log /var/log/pihole.log.1 echo " " > /var/log/pihole.log + chmod a+r /var/log/pihole.log fi else # Manual flushing @@ -53,6 +54,7 @@ else echo " " > /var/log/pihole.log if [ -f /var/log/pihole.log.1 ]; then echo " " > /var/log/pihole.log.1 + chmod a+r /var/log/pihole.log.1 fi fi # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history) diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 257c1929..55fce328 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -51,6 +51,7 @@ if [[ "$2" == "remote" ]]; then GITHUB_CORE_VERSION="$(json_extract tag_name "$(curl -s 'https://api.github.com/repos/pi-hole/pi-hole/releases/latest' 2> /dev/null)")" echo -n "${GITHUB_CORE_VERSION}" > "${GITHUB_VERSION_FILE}" + chmod a+r "${GITHUB_VERSION_FILE}" if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then GITHUB_WEB_VERSION="$(json_extract tag_name "$(curl -s 'https://api.github.com/repos/pi-hole/AdminLTE/releases/latest' 2> /dev/null)")" @@ -66,6 +67,7 @@ else CORE_BRANCH="$(get_local_branch /etc/.pihole)" echo -n "${CORE_BRANCH}" > "${LOCAL_BRANCH_FILE}" + chmod a+r "${LOCAL_BRANCH_FILE}" if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then WEB_BRANCH="$(get_local_branch /var/www/html/admin)" @@ -79,6 +81,7 @@ else CORE_VERSION="$(get_local_version /etc/.pihole)" echo -n "${CORE_VERSION}" > "${LOCAL_VERSION_FILE}" + chmod a+r "${LOCAL_VERSION_FILE}" if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then WEB_VERSION="$(get_local_version /var/www/html/admin)" diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index c6852896..eba539a6 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -322,6 +322,7 @@ dhcp-option=option:router,${DHCP_ROUTER} dhcp-leasefile=/etc/pihole/dhcp.leases #quiet-dhcp " > "${dhcpconfig}" + chmod a+r "${dhcpconfig}" if [[ "${PIHOLE_DOMAIN}" != "none" ]]; then echo "domain=${PIHOLE_DOMAIN}" >> "${dhcpconfig}" @@ -541,11 +542,13 @@ addAudit() do echo "${var}" >> /etc/pihole/auditlog.list done + chmod a+r /etc/pihole/auditlog.list } clearAudit() { echo -n "" > /etc/pihole/auditlog.list + chmod a+r /etc/pihole/auditlog.list } SetPrivacyLevel() { diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 94377647..2d85d854 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -194,7 +194,7 @@ if is_command apt-get ; then exit # exit the installer else printf " %b Enabling universe package repository for Ubuntu Bionic\\n" "${INFO}" - cp ${APT_SOURCES} ${APT_SOURCES}.backup # Backup current repo list + cp -p ${APT_SOURCES} ${APT_SOURCES}.backup # Backup current repo list printf " %b Backed up current configuration to %s\\n" "${TICK}" "${APT_SOURCES}.backup" add-apt-repository universe printf " %b Enabled %s\\n" "${TICK}" "'universe' repository" @@ -847,7 +847,7 @@ setIFCFG() { # Put the IP in variables without the CIDR notation printf -v CIDR "%s" "${IPV4_ADDRESS##*/}" # Backup existing interface configuration: - cp "${IFCFG_FILE}" "${IFCFG_FILE}".pihole.orig + cp -p "${IFCFG_FILE}" "${IFCFG_FILE}".pihole.orig # Build Interface configuration file using the GLOBAL variables we have { echo "# Configured via Pi-hole installer" @@ -1241,7 +1241,8 @@ version_check_dnsmasq() { printf "%b %b Backing up dnsmasq.conf to dnsmasq.conf.orig...\\n" "${OVER}" "${TICK}" printf " %b Restoring default dnsmasq.conf..." "${INFO}" # and replace it with the default - cp ${dnsmasq_original_config} ${dnsmasq_conf} + cp -p ${dnsmasq_original_config} ${dnsmasq_conf} + chmod a+r ${dnsmasq_conf} printf "%b %b Restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" # Otherwise, else @@ -1252,7 +1253,7 @@ version_check_dnsmasq() { # If a file cannot be found, printf " %b No dnsmasq.conf found... restoring default dnsmasq.conf..." "${INFO}" # restore the default one - cp ${dnsmasq_original_config} ${dnsmasq_conf} + cp -p ${dnsmasq_original_config} ${dnsmasq_conf} printf "%b %b No dnsmasq.conf found... restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" fi @@ -1260,9 +1261,11 @@ version_check_dnsmasq() { # Check to see if dnsmasq directory exists (it may not due to being a fresh install and dnsmasq no longer being a dependency) if [[ ! -d "/etc/dnsmasq.d" ]];then mkdir "/etc/dnsmasq.d" + chmod 755 "/etc/dnsmasq.d" fi # Copy the new Pi-hole DNS config file into the dnsmasq.d directory cp ${dnsmasq_pihole_01_snippet} ${dnsmasq_pihole_01_location} + chmod a+r ${dnsmasq_pihole_01_location} printf "%b %b Copying 01-pihole.conf to /etc/dnsmasq.d/01-pihole.conf\\n" "${OVER}" "${TICK}" # Replace our placeholder values with the GLOBAL DNS variables that we populated earlier # First, swap in the interface to listen on @@ -1382,6 +1385,7 @@ installConfigs() { mkdir /etc/lighttpd # and set the owners chown "${USER}":root /etc/lighttpd + chmod 755 /etc/lighttpd # Otherwise, if the config file already exists elif [[ -f "/etc/lighttpd/lighttpd.conf" ]]; then # back up the original @@ -1389,8 +1393,10 @@ installConfigs() { fi # and copy in the config file Pi-hole needs cp ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} /etc/lighttpd/lighttpd.conf + chmod a+r /etc/lighttpd/lighttpd.conf # Make sure the external.conf file exists, as lighttpd v1.4.50 crashes without it touch /etc/lighttpd/external.conf + chmod a+r /etc/lighttpd/external.conf # if there is a custom block page in the html/pihole directory, replace 404 handler in lighttpd config if [[ -f "${PI_HOLE_BLOCKPAGE_DIR}/custom.php" ]]; then sed -i 's/^\(server\.error-handler-404\s*=\s*\).*$/\1"pihole\/custom\.php"/' /etc/lighttpd/lighttpd.conf @@ -1422,15 +1428,26 @@ install_manpage() { if [[ ! -d "/usr/local/share/man/man8" ]]; then # if not present, create man8 directory mkdir /usr/local/share/man/man8 + chown root:staff /usr/local/share/man/man8 + chmod a+r /usr/local/share/man/man8 + chmod a+x /usr/local/share/man/man8 + chmod g+s /usr/local/share/man/man8 fi if [[ ! -d "/usr/local/share/man/man5" ]]; then - # if not present, create man8 directory + # if not present, create man5 directory mkdir /usr/local/share/man/man5 + chown root:staff /usr/local/share/man/man5 + chmod a+r /usr/local/share/man/man5 + chmod a+x /usr/local/share/man/man5 + chmod g+s /usr/local/share/man/man5 fi # Testing complete, copy the files & update the man db cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole.8 /usr/local/share/man/man8/pihole.8 + chmod a+r /usr/local/share/man/man8/pihole.8 cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.8 /usr/local/share/man/man8/pihole-FTL.8 + chmod a+r /usr/local/share/man/man8/pihole-FTL.8 cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.conf.5 /usr/local/share/man/man5/pihole-FTL.conf.5 + chmod a+r /usr/local/share/man/man5/pihole-FTL.conf.5 if mandb -q &>/dev/null; then # Updated successfully printf "%b %b man pages installed and database updated\\n" "${OVER}" "${TICK}" @@ -1711,6 +1728,9 @@ installCron() { printf "\\n %b %s..." "${INFO}" "${str}" # Copy the cron file over from the local repo cp ${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole.cron /etc/cron.d/pihole + # File must not be world or group writeable and must be owned by root + chmod 644 /etc/cron.d/pihole + chown root:root /etc/cron.d/pihole # Randomize gravity update time sed -i "s/59 1 /$((1 + RANDOM % 58)) $((3 + RANDOM % 2))/" /etc/cron.d/pihole # Randomize update checker time @@ -1818,6 +1838,7 @@ finalExports() { echo "INSTALL_WEB_INTERFACE=${INSTALL_WEB_INTERFACE}" echo "LIGHTTPD_ENABLED=${LIGHTTPD_ENABLED}" }>> "${setupVars}" + chmod 744 "${setupVars}" # Set the privacy level sed -i '/PRIVACYLEVEL/d' "${PI_HOLE_CONFIG_DIR}/pihole-FTL.conf" @@ -1841,6 +1862,7 @@ installLogrotate() { printf "\\n %b %s..." "${INFO}" "${str}" # Copy the file over from the local repo cp ${PI_HOLE_LOCAL_REPO}/advanced/Templates/logrotate /etc/pihole/logrotate + chmod a+r /etc/pihole/logrotate # Different operating systems have different user / group # settings for logrotate that makes it impossible to create # a static logrotate file that will work with e.g. @@ -2431,6 +2453,7 @@ copy_to_install_log() { # Copy the contents of file descriptor 3 into the install log # Since we use color codes such as '\e[1;33m', they should be removed sed 's/\[[0-9;]\{1,5\}m//g' < /proc/$$/fd/3 > "${installLogLoc}" + chmod a+r "${installLogLoc}" } main() { @@ -2516,6 +2539,8 @@ main() { welcomeDialogs # Create directory for Pi-hole storage mkdir -p /etc/pihole/ + chmod a+r /ect/pihole/ + chmod a+x /etc/pihole/ # Determine available interfaces get_available_interfaces # Find interfaces and let the user choose one diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index d0a6dcf0..732fc246 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -153,7 +153,7 @@ removeNoPurge() { # Restore Resolved if [[ -e /etc/systemd/resolved.conf.orig ]]; then - ${SUDO} cp /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf + ${SUDO} cp -p /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf systemctl reload-or-restart systemd-resolved fi diff --git a/gravity.sh b/gravity.sh index 015764ac..6ae0e337 100755 --- a/gravity.sh +++ b/gravity.sh @@ -485,6 +485,7 @@ gravity_SortAndFilterConsolidatedList() { fi sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}" + chmod a+r "${piholeDir}/${preEventHorizon}" if [[ "${haveSourceUrls}" == true ]]; then echo -e "${OVER} ${TICK} ${str}" @@ -509,6 +510,7 @@ gravity_Whitelist() { # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in $whitelistFile comm -23 "${piholeDir}/${preEventHorizon}" <(sort "${whitelistFile}") > "${piholeDir}/${whitelistMatter}" + chmod a+r "${piholeDir}/${whitelistMatter}" echo -e "${OVER} ${INFO} ${str}" } @@ -561,6 +563,7 @@ gravity_ParseLocalDomains() { # Empty $localList if it already exists, otherwise, create it : > "${localList}" + chmod a+r "${localList}" gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}" @@ -581,8 +584,9 @@ gravity_ParseBlacklistDomains() { mv "${piholeDir}/${whitelistMatter}" "${piholeDir}/${accretionDisc}" else # There was no whitelist file, so use preEventHorizon instead of whitelistMatter. - cp "${piholeDir}/${preEventHorizon}" "${piholeDir}/${accretionDisc}" + cp -p "${piholeDir}/${preEventHorizon}" "${piholeDir}/${accretionDisc}" fi + chmod a+r "${piholeDir}/${accretionDisc}" # Move the file over as /etc/pihole/gravity.list so dnsmasq can use it output=$( { mv "${piholeDir}/${accretionDisc}" "${adList}"; } 2>&1 ) @@ -592,6 +596,7 @@ gravity_ParseBlacklistDomains() { echo -e "\\n ${CROSS} Unable to move ${accretionDisc} from ${piholeDir}\\n ${output}" gravity_Cleanup "error" fi + chmod a+r "${adList}" } # Create user-added blacklist entries @@ -602,6 +607,7 @@ gravity_ParseUserDomains() { # Copy the file over as /etc/pihole/black.list so dnsmasq can use it cp "${blacklistFile}" "${blackList}" 2> /dev/null || \ echo -e "\\n ${CROSS} Unable to move ${blacklistFile##*/} to ${piholeDir}" + chmod a+r "${blackList}" } # Trap Ctrl-C diff --git a/pihole b/pihole index 84a5623f..59a99af1 100755 --- a/pihole +++ b/pihole @@ -151,10 +151,12 @@ Time: if [[ -e "${gravitylist}" ]]; then mv "${gravitylist}" "${gravitylist}.bck" echo "" > "${gravitylist}" + chmod a+r "${gravitylist}" fi if [[ -e "${blacklist}" ]]; then mv "${blacklist}" "${blacklist}.bck" echo "" > "${blacklist}" + chmod a+r "${blacklist}" fi if [[ $# > 1 ]]; then local error=false @@ -206,9 +208,11 @@ Time: if [[ -e "${gravitylist}.bck" ]]; then mv "${gravitylist}.bck" "${gravitylist}" + chmod a+r "${gravitylist}" fi if [[ -e "${blacklist}.bck" ]]; then mv "${blacklist}.bck" "${blacklist}" + chmod a+r "${blacklist}" fi sed -i "/BLOCKING_ENABLED=/d" "${setupVars}" echo "BLOCKING_ENABLED=true" >> "${setupVars}" From 8a92fb24c4d4e3caff4286ea8698becdfe986a58 Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Wed, 1 May 2019 11:20:26 +0200 Subject: [PATCH 58/86] Incorporates changes requested by @Mcat12 Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- advanced/Scripts/list.sh | 6 +++--- advanced/Scripts/piholeCheckout.sh | 6 +++--- advanced/Scripts/piholeLogFlush.sh | 4 ++-- advanced/Scripts/updatecheck.sh | 6 +++--- advanced/Scripts/webpage.sh | 6 +++--- automated install/basic-install.sh | 33 ++++++++++++------------------ gravity.sh | 12 +++++------ pihole | 8 ++++---- 8 files changed, 37 insertions(+), 44 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index e0455e0c..b4fffbaf 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -92,13 +92,13 @@ PoplistFile() { # Check whitelist file exists, and if not, create it if [[ ! -f "${whitelist}" ]]; then touch "${whitelist}" - chmod a+r "${whitelist}" + chmod 644 "${whitelist}" fi # Check blacklist file exists, and if not, create it if [[ ! -f "${blacklist}" ]]; then touch "${blacklist}" - chmod a+r "${blacklist}" + chmod 644 "${blacklist}" fi for dom in "${domList[@]}"; do @@ -244,7 +244,7 @@ NukeList() { cp -p "${listMain}" "${listMain}.bck~" # Empty out file echo "" > "${listMain}" - chmod a+r "${listMain}" + chmod 644 "${listMain}" fi } diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index a982289f..13fa3402 100644 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -90,7 +90,7 @@ checkout() { local path path="development/${binary}" echo "development" > /etc/pihole/ftlbranch - chmod a+r /etc/pihole/ftlbranch + chmod 644 /etc/pihole/ftlbranch elif [[ "${1}" == "master" ]] ; then # Shortcut to check out master branches echo -e " ${INFO} Shortcut \"master\" detected - checking out master branches..." @@ -105,7 +105,7 @@ checkout() { local path path="master/${binary}" echo "master" > /etc/pihole/ftlbranch - chmod a+r /etc/pihole/ftlbranch + chmod 644 /etc/pihole/ftlbranch elif [[ "${1}" == "core" ]] ; then str="Fetching branches from ${piholeGitUrl}" echo -ne " ${INFO} $str" @@ -168,7 +168,7 @@ checkout() { if check_download_exists "$path"; then echo " ${TICK} Branch ${2} exists" echo "${2}" > /etc/pihole/ftlbranch - chmod a+r /etc/pihole/ftlbranch + chmod 644 /etc/pihole/ftlbranch FTLinstall "${binary}" restart_service pihole-FTL enable_service pihole-FTL diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 1e4b7abe..51e94d7c 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -41,7 +41,7 @@ if [[ "$@" == *"once"* ]]; then # moved file (it will have the same file handler) cp -p /var/log/pihole.log /var/log/pihole.log.1 echo " " > /var/log/pihole.log - chmod a+r /var/log/pihole.log + chmod 644 /var/log/pihole.log fi else # Manual flushing @@ -54,7 +54,7 @@ else echo " " > /var/log/pihole.log if [ -f /var/log/pihole.log.1 ]; then echo " " > /var/log/pihole.log.1 - chmod a+r /var/log/pihole.log.1 + chmod 644 /var/log/pihole.log.1 fi fi # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history) diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 55fce328..972ab585 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -51,7 +51,7 @@ if [[ "$2" == "remote" ]]; then GITHUB_CORE_VERSION="$(json_extract tag_name "$(curl -s 'https://api.github.com/repos/pi-hole/pi-hole/releases/latest' 2> /dev/null)")" echo -n "${GITHUB_CORE_VERSION}" > "${GITHUB_VERSION_FILE}" - chmod a+r "${GITHUB_VERSION_FILE}" + chmod 644 "${GITHUB_VERSION_FILE}" if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then GITHUB_WEB_VERSION="$(json_extract tag_name "$(curl -s 'https://api.github.com/repos/pi-hole/AdminLTE/releases/latest' 2> /dev/null)")" @@ -67,7 +67,7 @@ else CORE_BRANCH="$(get_local_branch /etc/.pihole)" echo -n "${CORE_BRANCH}" > "${LOCAL_BRANCH_FILE}" - chmod a+r "${LOCAL_BRANCH_FILE}" + chmod 644 "${LOCAL_BRANCH_FILE}" if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then WEB_BRANCH="$(get_local_branch /var/www/html/admin)" @@ -81,7 +81,7 @@ else CORE_VERSION="$(get_local_version /etc/.pihole)" echo -n "${CORE_VERSION}" > "${LOCAL_VERSION_FILE}" - chmod a+r "${LOCAL_VERSION_FILE}" + chmod 644 "${LOCAL_VERSION_FILE}" if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then WEB_VERSION="$(get_local_version /var/www/html/admin)" diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index eba539a6..467eebd6 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -322,7 +322,7 @@ dhcp-option=option:router,${DHCP_ROUTER} dhcp-leasefile=/etc/pihole/dhcp.leases #quiet-dhcp " > "${dhcpconfig}" - chmod a+r "${dhcpconfig}" + chmod 644 "${dhcpconfig}" if [[ "${PIHOLE_DOMAIN}" != "none" ]]; then echo "domain=${PIHOLE_DOMAIN}" >> "${dhcpconfig}" @@ -542,13 +542,13 @@ addAudit() do echo "${var}" >> /etc/pihole/auditlog.list done - chmod a+r /etc/pihole/auditlog.list + chmod 644 /etc/pihole/auditlog.list } clearAudit() { echo -n "" > /etc/pihole/auditlog.list - chmod a+r /etc/pihole/auditlog.list + chmod 644 /etc/pihole/auditlog.list } SetPrivacyLevel() { diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 2d85d854..c1f69efb 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1242,7 +1242,7 @@ version_check_dnsmasq() { printf " %b Restoring default dnsmasq.conf..." "${INFO}" # and replace it with the default cp -p ${dnsmasq_original_config} ${dnsmasq_conf} - chmod a+r ${dnsmasq_conf} + chmod 644 ${dnsmasq_conf} printf "%b %b Restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" # Otherwise, else @@ -1265,7 +1265,7 @@ version_check_dnsmasq() { fi # Copy the new Pi-hole DNS config file into the dnsmasq.d directory cp ${dnsmasq_pihole_01_snippet} ${dnsmasq_pihole_01_location} - chmod a+r ${dnsmasq_pihole_01_location} + chmod 644 ${dnsmasq_pihole_01_location} printf "%b %b Copying 01-pihole.conf to /etc/dnsmasq.d/01-pihole.conf\\n" "${OVER}" "${TICK}" # Replace our placeholder values with the GLOBAL DNS variables that we populated earlier # First, swap in the interface to listen on @@ -1393,10 +1393,10 @@ installConfigs() { fi # and copy in the config file Pi-hole needs cp ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} /etc/lighttpd/lighttpd.conf - chmod a+r /etc/lighttpd/lighttpd.conf + chmod 644 /etc/lighttpd/lighttpd.conf # Make sure the external.conf file exists, as lighttpd v1.4.50 crashes without it touch /etc/lighttpd/external.conf - chmod a+r /etc/lighttpd/external.conf + chmod 644 /etc/lighttpd/external.conf # if there is a custom block page in the html/pihole directory, replace 404 handler in lighttpd config if [[ -f "${PI_HOLE_BLOCKPAGE_DIR}/custom.php" ]]; then sed -i 's/^\(server\.error-handler-404\s*=\s*\).*$/\1"pihole\/custom\.php"/' /etc/lighttpd/lighttpd.conf @@ -1428,26 +1428,20 @@ install_manpage() { if [[ ! -d "/usr/local/share/man/man8" ]]; then # if not present, create man8 directory mkdir /usr/local/share/man/man8 - chown root:staff /usr/local/share/man/man8 - chmod a+r /usr/local/share/man/man8 - chmod a+x /usr/local/share/man/man8 - chmod g+s /usr/local/share/man/man8 + chmod 755 /usr/local/share/man/man8 fi if [[ ! -d "/usr/local/share/man/man5" ]]; then # if not present, create man5 directory mkdir /usr/local/share/man/man5 - chown root:staff /usr/local/share/man/man5 - chmod a+r /usr/local/share/man/man5 - chmod a+x /usr/local/share/man/man5 - chmod g+s /usr/local/share/man/man5 + chmod 755 /usr/local/share/man/man5 fi # Testing complete, copy the files & update the man db cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole.8 /usr/local/share/man/man8/pihole.8 - chmod a+r /usr/local/share/man/man8/pihole.8 + chmod 644 /usr/local/share/man/man8/pihole.8 cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.8 /usr/local/share/man/man8/pihole-FTL.8 - chmod a+r /usr/local/share/man/man8/pihole-FTL.8 + chmod 644 /usr/local/share/man/man8/pihole-FTL.8 cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.conf.5 /usr/local/share/man/man5/pihole-FTL.conf.5 - chmod a+r /usr/local/share/man/man5/pihole-FTL.conf.5 + chmod 644 /usr/local/share/man/man5/pihole-FTL.conf.5 if mandb -q &>/dev/null; then # Updated successfully printf "%b %b man pages installed and database updated\\n" "${OVER}" "${TICK}" @@ -1838,7 +1832,7 @@ finalExports() { echo "INSTALL_WEB_INTERFACE=${INSTALL_WEB_INTERFACE}" echo "LIGHTTPD_ENABLED=${LIGHTTPD_ENABLED}" }>> "${setupVars}" - chmod 744 "${setupVars}" + chmod 644 "${setupVars}" # Set the privacy level sed -i '/PRIVACYLEVEL/d' "${PI_HOLE_CONFIG_DIR}/pihole-FTL.conf" @@ -1862,7 +1856,7 @@ installLogrotate() { printf "\\n %b %s..." "${INFO}" "${str}" # Copy the file over from the local repo cp ${PI_HOLE_LOCAL_REPO}/advanced/Templates/logrotate /etc/pihole/logrotate - chmod a+r /etc/pihole/logrotate + chmod 644 /etc/pihole/logrotate # Different operating systems have different user / group # settings for logrotate that makes it impossible to create # a static logrotate file that will work with e.g. @@ -2453,7 +2447,7 @@ copy_to_install_log() { # Copy the contents of file descriptor 3 into the install log # Since we use color codes such as '\e[1;33m', they should be removed sed 's/\[[0-9;]\{1,5\}m//g' < /proc/$$/fd/3 > "${installLogLoc}" - chmod a+r "${installLogLoc}" + chmod 644 "${installLogLoc}" } main() { @@ -2539,8 +2533,7 @@ main() { welcomeDialogs # Create directory for Pi-hole storage mkdir -p /etc/pihole/ - chmod a+r /ect/pihole/ - chmod a+x /etc/pihole/ + chmod 755 /ect/pihole/ # Determine available interfaces get_available_interfaces # Find interfaces and let the user choose one diff --git a/gravity.sh b/gravity.sh index 6ae0e337..a2f2c782 100755 --- a/gravity.sh +++ b/gravity.sh @@ -485,7 +485,7 @@ gravity_SortAndFilterConsolidatedList() { fi sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}" - chmod a+r "${piholeDir}/${preEventHorizon}" + chmod 644 "${piholeDir}/${preEventHorizon}" if [[ "${haveSourceUrls}" == true ]]; then echo -e "${OVER} ${TICK} ${str}" @@ -510,7 +510,7 @@ gravity_Whitelist() { # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in $whitelistFile comm -23 "${piholeDir}/${preEventHorizon}" <(sort "${whitelistFile}") > "${piholeDir}/${whitelistMatter}" - chmod a+r "${piholeDir}/${whitelistMatter}" + chmod 644 "${piholeDir}/${whitelistMatter}" echo -e "${OVER} ${INFO} ${str}" } @@ -563,7 +563,7 @@ gravity_ParseLocalDomains() { # Empty $localList if it already exists, otherwise, create it : > "${localList}" - chmod a+r "${localList}" + chmod 644 "${localList}" gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}" @@ -586,7 +586,7 @@ gravity_ParseBlacklistDomains() { # There was no whitelist file, so use preEventHorizon instead of whitelistMatter. cp -p "${piholeDir}/${preEventHorizon}" "${piholeDir}/${accretionDisc}" fi - chmod a+r "${piholeDir}/${accretionDisc}" + chmod 644 "${piholeDir}/${accretionDisc}" # Move the file over as /etc/pihole/gravity.list so dnsmasq can use it output=$( { mv "${piholeDir}/${accretionDisc}" "${adList}"; } 2>&1 ) @@ -596,7 +596,7 @@ gravity_ParseBlacklistDomains() { echo -e "\\n ${CROSS} Unable to move ${accretionDisc} from ${piholeDir}\\n ${output}" gravity_Cleanup "error" fi - chmod a+r "${adList}" + chmod 644 "${adList}" } # Create user-added blacklist entries @@ -607,7 +607,7 @@ gravity_ParseUserDomains() { # Copy the file over as /etc/pihole/black.list so dnsmasq can use it cp "${blacklistFile}" "${blackList}" 2> /dev/null || \ echo -e "\\n ${CROSS} Unable to move ${blacklistFile##*/} to ${piholeDir}" - chmod a+r "${blackList}" + chmod 644 "${blackList}" } # Trap Ctrl-C diff --git a/pihole b/pihole index 59a99af1..541434a0 100755 --- a/pihole +++ b/pihole @@ -151,12 +151,12 @@ Time: if [[ -e "${gravitylist}" ]]; then mv "${gravitylist}" "${gravitylist}.bck" echo "" > "${gravitylist}" - chmod a+r "${gravitylist}" + chmod 644 "${gravitylist}" fi if [[ -e "${blacklist}" ]]; then mv "${blacklist}" "${blacklist}.bck" echo "" > "${blacklist}" - chmod a+r "${blacklist}" + chmod 644 "${blacklist}" fi if [[ $# > 1 ]]; then local error=false @@ -208,11 +208,11 @@ Time: if [[ -e "${gravitylist}.bck" ]]; then mv "${gravitylist}.bck" "${gravitylist}" - chmod a+r "${gravitylist}" + chmod 644 "${gravitylist}" fi if [[ -e "${blacklist}.bck" ]]; then mv "${blacklist}.bck" "${blacklist}" - chmod a+r "${blacklist}" + chmod 644 "${blacklist}" fi sed -i "/BLOCKING_ENABLED=/d" "${setupVars}" echo "BLOCKING_ENABLED=true" >> "${setupVars}" From 487718512dabdbdb4e3858eeed98c711f3de6fa7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 16:37:23 +0200 Subject: [PATCH 59/86] Remove DISTINCT attribute from views. They are not needed as the corresponding data is obtained from fields with UNIQUE attribute so duplications are already prevented at domain insertion. The elimination of the duplication check on imort through the views slightly reduces the load during import (I observed about 8 percent time saved on my NanoPi NEO). Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index dc0f23ed..dac0a50c 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -7,11 +7,11 @@ CREATE TABLE info (property TEXT NOT NULL, value TEXT NOT NULL); INSERT INTO info VALUES("version","1"); -CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain +CREATE VIEW vw_gravity AS SELECT a.domain FROM gravity a WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1); -CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain +CREATE VIEW vw_whitelist AS SELECT a.domain FROM whitelist a WHERE a.enabled == 1; CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist @@ -19,7 +19,7 @@ CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; END; -CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain +CREATE VIEW vw_blacklist AS SELECT a.domain FROM blacklist a WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist; CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist @@ -27,7 +27,7 @@ CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; END; -CREATE VIEW vw_regex AS SELECT DISTINCT a.domain +CREATE VIEW vw_regex AS SELECT a.domain FROM regex a WHERE a.enabled == 1; CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex @@ -35,7 +35,7 @@ CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; END; -CREATE VIEW vw_adlists AS SELECT DISTINCT a.address +CREATE VIEW vw_adlists AS SELECT a.address FROM adlists a WHERE a.enabled == 1; CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists From d5781fb1105d0c0ca1eda09c8e92f19ac135ff0c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 16:38:41 +0200 Subject: [PATCH 60/86] Fix extra space in front on END statement in CREATE TRIGGER commands Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index dac0a50c..0960018b 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -17,7 +17,7 @@ CREATE VIEW vw_whitelist AS SELECT a.domain CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist BEGIN UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; - END; + END; CREATE VIEW vw_blacklist AS SELECT a.domain FROM blacklist a @@ -25,7 +25,7 @@ CREATE VIEW vw_blacklist AS SELECT a.domain CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist BEGIN UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; - END; + END; CREATE VIEW vw_regex AS SELECT a.domain FROM regex a @@ -33,7 +33,7 @@ CREATE VIEW vw_regex AS SELECT a.domain CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex BEGIN UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; - END; + END; CREATE VIEW vw_adlists AS SELECT a.address FROM adlists a @@ -41,5 +41,5 @@ CREATE VIEW vw_adlists AS SELECT a.address CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists BEGIN UPDATE adlists SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address; - END; + END; From bc9b62363895b8b6483e0f518ba46607b75f9909 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 16:56:16 +0200 Subject: [PATCH 61/86] Modify query.sh to use the gravity database Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) mode change 100644 => 100755 advanced/Scripts/query.sh diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh old mode 100644 new mode 100755 index 69a3c7a4..8eb7c404 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -11,7 +11,7 @@ # Globals piholeDir="/etc/pihole" -adListsList="$piholeDir/adlists.list" +gravityDBfile="${piholeDir}/gravity.db" wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf" options="$*" adlist="" @@ -73,11 +73,6 @@ Options: exit 0 fi -if [[ ! -e "$adListsList" ]]; then - echo -e "${COL_LIGHT_RED}The file $adListsList was not found${COL_NC}" - exit 1 -fi - # Handle valid options if [[ "${options}" == *"-bp"* ]]; then exact="exact"; blockpage=true @@ -185,12 +180,9 @@ if [[ -z "${exact}" ]]; then fi # Get adlist file content as array -if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then - for adlistUrl in $(< "${adListsList}"); do - if [[ "${adlistUrl:0:4}" =~ (http|www.) ]]; then - adlists+=("${adlistUrl}") - fi - done +if [[ -n "${blockpage}" ]]; then + # Retrieve source URLs from gravity database + mapfile -t adlists <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)" fi # Print "Exact matches for" title From b372f808ddf1ff0fed03b1f314c0a85e487766cd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 17:06:14 +0200 Subject: [PATCH 62/86] Fix lint errors Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 2 +- gravity.sh | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 8eb7c404..a4ac895b 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -180,7 +180,7 @@ if [[ -z "${exact}" ]]; then fi # Get adlist file content as array -if [[ -n "${blockpage}" ]]; then +if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then # Retrieve source URLs from gravity database mapfile -t adlists <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)" fi diff --git a/gravity.sh b/gravity.sh index 331b632d..9f4065e7 100755 --- a/gravity.sh +++ b/gravity.sh @@ -87,10 +87,11 @@ generate_gravity_database() { # Import domains from file and store them in the specified database table database_table_from_file() { # Define locals - local table="${1}" - local source="${2}" - local backup_path="${piholeDir}/migration_backup" - local backup_file="${backup_path}/$(basename "${2}")" + local table source backup_path backup_file + table="${1}" + source="${2}" + backup_path="${piholeDir}/migration_backup" + backup_file="${backup_path}/$(basename "${2}")" # Create database file if not present if [ ! -e "${gravityDBfile}" ]; then From 983117d788bd32ef1521a5c2da3088a2039e23bf Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 17:31:34 +0200 Subject: [PATCH 63/86] Add id (primary key, autoincrement) field to lists tables Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 28 ++++++++++++++++++++++++---- gravity.sh | 6 +++++- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 0960018b..65bd7448 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,7 +1,27 @@ -CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); -CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); -CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); -CREATE TABLE adlists (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); +CREATE TABLE whitelist (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT); +CREATE TABLE blacklist (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT); +CREATE TABLE regex (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT); +CREATE TABLE adlists (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + address TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); CREATE TABLE info (property TEXT NOT NULL, value TEXT NOT NULL); diff --git a/gravity.sh b/gravity.sh index 9f4065e7..c1eb0720 100755 --- a/gravity.sh +++ b/gravity.sh @@ -118,12 +118,16 @@ database_table_from_file() { inputfile="${source}" else # Apply format for white-, blacklist, regex, and adlists tables + local rowid + declare -i rowid + rowid=1 # Read file line by line grep -v '^ *#' < "${source}" | while IFS= read -r domain do # Only add non-empty lines if [[ ! -z "${domain}" ]]; then - echo "\"${domain}\",1,${timestamp},${timestamp}" >> "${tmpFile}" + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp}" >> "${tmpFile}" + rowid+=1 fi done inputfile="${tmpFile}" From 5e26e8245ba4f849577bb1ba232389ba00e59247 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 17:50:36 +0200 Subject: [PATCH 64/86] Order results of queries to list views by id key Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 65bd7448..d1651619 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -33,7 +33,9 @@ CREATE VIEW vw_gravity AS SELECT a.domain CREATE VIEW vw_whitelist AS SELECT a.domain FROM whitelist a - WHERE a.enabled == 1; + WHERE a.enabled == 1 + ORDER BY a.id; + CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist BEGIN UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; @@ -41,7 +43,9 @@ CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist CREATE VIEW vw_blacklist AS SELECT a.domain FROM blacklist a - WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist; + WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist + ORDER BY a.id; + CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist BEGIN UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; @@ -49,7 +53,9 @@ CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist CREATE VIEW vw_regex AS SELECT a.domain FROM regex a - WHERE a.enabled == 1; + WHERE a.enabled == 1 + ORDER BY a.id; + CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex BEGIN UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; @@ -57,7 +63,9 @@ CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex CREATE VIEW vw_adlists AS SELECT a.address FROM adlists a - WHERE a.enabled == 1; + WHERE a.enabled == 1 + ORDER BY a.id; + CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists BEGIN UPDATE adlists SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address; From 540c57f755c0e6255c2416738478491494f692b4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 17:51:25 +0200 Subject: [PATCH 65/86] Add comment for automigrated list entries Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index c1eb0720..afd01aad 100755 --- a/gravity.sh +++ b/gravity.sh @@ -126,7 +126,7 @@ database_table_from_file() { do # Only add non-empty lines if [[ ! -z "${domain}" ]]; then - echo "${rowid},\"${domain}\",1,${timestamp},${timestamp}" >> "${tmpFile}" + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" rowid+=1 fi done From 6c1e3a17f7534bb1ed7ea448401d11adc4b66406 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 17:54:29 +0200 Subject: [PATCH 66/86] Remove NOT NULL constraint on list ids as this is implied by PRIMARY KEY (even though most, if not all, examples on the web still use both attributes at the same time) Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index d1651619..4e78e026 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,22 +1,22 @@ -CREATE TABLE whitelist (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, +CREATE TABLE whitelist (id INTEGER PRIMARY KEY AUTOINCREMENT, domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); -CREATE TABLE blacklist (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, +CREATE TABLE blacklist (id INTEGER PRIMARY KEY AUTOINCREMENT, domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); -CREATE TABLE regex (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, +CREATE TABLE regex (id INTEGER PRIMARY KEY AUTOINCREMENT, domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); -CREATE TABLE adlists (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, +CREATE TABLE adlists (id INTEGER PRIMARY KEY AUTOINCREMENT, address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), From 106f079afc24dd4f43699ca0a9f494f5e72dc850 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 17:57:23 +0200 Subject: [PATCH 67/86] Improve indentation (single tab) in SQLite template Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 71 ++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 26 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 4e78e026..a8d48608 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,29 +1,48 @@ -CREATE TABLE whitelist (id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT); -CREATE TABLE blacklist (id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT); -CREATE TABLE regex (id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT); -CREATE TABLE adlists (id INTEGER PRIMARY KEY AUTOINCREMENT, - address TEXT UNIQUE NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT); -CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); -CREATE TABLE info (property TEXT NOT NULL, value TEXT NOT NULL); +CREATE TABLE whitelist +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); +CREATE TABLE blacklist +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); +CREATE TABLE regex +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); +CREATE TABLE adlists +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + address TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); +CREATE TABLE gravity +( + domain TEXT UNIQUE NOT NULL +); +CREATE TABLE info +( + property TEXT NOT NULL, + value TEXT NOT NULL +); INSERT INTO info VALUES("version","1"); From 2f698904e406fc978cb00bf4759aebbc22b14479 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 18:03:35 +0200 Subject: [PATCH 68/86] Use PRIMARY KEY instead of NOT NULL also in gravity and info tables Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index a8d48608..fd6d2b28 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -36,11 +36,11 @@ CREATE TABLE adlists ); CREATE TABLE gravity ( - domain TEXT UNIQUE NOT NULL + domain TEXT UNIQUE PRIMARY KEY ); CREATE TABLE info ( - property TEXT NOT NULL, + property TEXT PRIMARY KEY, value TEXT NOT NULL ); From 008e88b84be1f818fe6518d954ddc8b4a7aca63c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 18:04:31 +0200 Subject: [PATCH 69/86] Only migrate legacy list files when we create gravity.db in the same gravity run Signed-off-by: DL6ER --- gravity.sh | 51 ++++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/gravity.sh b/gravity.sh index afd01aad..02f41f86 100755 --- a/gravity.sh +++ b/gravity.sh @@ -93,12 +93,6 @@ database_table_from_file() { backup_path="${piholeDir}/migration_backup" backup_file="${backup_path}/$(basename "${2}")" - # Create database file if not present - if [ ! -e "${gravityDBfile}" ]; then - echo -e " ${INFO} Creating new gravity database" - generate_gravity_database - fi - # Truncate table output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 ) status="$?" @@ -155,25 +149,32 @@ database_table_from_file() { # Migrate pre-v5.0 list files to database-based Pi-hole versions migrate_to_database() { - if [[ -e "${adListFile}" ]]; then - # Store adlists domains in database - echo -e " ${INFO} Pi-hole upgrade: Moving content of ${adListFile} into database" - database_table_from_file "adlists" "${adListFile}" - fi - if [[ -e "${blacklistFile}" ]]; then - # Store blacklisted domains in database - echo -e " ${INFO} Pi-hole upgrade: Moving content of ${blacklistFile} into database" - database_table_from_file "blacklist" "${blacklistFile}" - fi - if [[ -e "${whitelistFile}" ]]; then - # Store whitelisted domains in database - echo -e " ${INFO} Pi-hole upgrade: Moving content of ${whitelistFile} into database" - database_table_from_file "whitelist" "${whitelistFile}" - fi - if [[ -e "${regexFile}" ]]; then - # Store regex domains in database - echo -e " ${INFO} Pi-hole upgrade: Moving content of ${regexFile} into database" - database_table_from_file "regex" "${regexFile}" + # Create database file if not present + if [ ! -e "${gravityDBfile}" ]; then + echo -e " ${INFO} Creating new gravity database" + generate_gravity_database + + # Migrate list files to new database + if [[ -e "${adListFile}" ]]; then + # Store adlists domains in database + echo -e " ${INFO} Migrating content of ${adListFile} into new database" + database_table_from_file "adlists" "${adListFile}" + fi + if [[ -e "${blacklistFile}" ]]; then + # Store blacklisted domains in database + echo -e " ${INFO} Migrating content of ${blacklistFile} into new database" + database_table_from_file "blacklist" "${blacklistFile}" + fi + if [[ -e "${whitelistFile}" ]]; then + # Store whitelisted domains in database + echo -e " ${INFO} Migrating content of ${whitelistFile} into new database" + database_table_from_file "whitelist" "${whitelistFile}" + fi + if [[ -e "${regexFile}" ]]; then + # Store regex domains in database + echo -e " ${INFO} Migrating content of ${regexFile} into new database" + database_table_from_file "regex" "${regexFile}" + fi fi } From 0cfecd31d3501c5a760207bd1780c855a90be2f9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 21:12:22 +0200 Subject: [PATCH 70/86] Invert logic to reduce nesting Signed-off-by: DL6ER --- gravity.sh | 52 +++++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/gravity.sh b/gravity.sh index 02f41f86..76d42f32 100755 --- a/gravity.sh +++ b/gravity.sh @@ -149,32 +149,34 @@ database_table_from_file() { # Migrate pre-v5.0 list files to database-based Pi-hole versions migrate_to_database() { - # Create database file if not present - if [ ! -e "${gravityDBfile}" ]; then - echo -e " ${INFO} Creating new gravity database" - generate_gravity_database + # Create database file only if not present + if [ -e "${gravityDBfile}" ]; then + return 0 + fi - # Migrate list files to new database - if [[ -e "${adListFile}" ]]; then - # Store adlists domains in database - echo -e " ${INFO} Migrating content of ${adListFile} into new database" - database_table_from_file "adlists" "${adListFile}" - fi - if [[ -e "${blacklistFile}" ]]; then - # Store blacklisted domains in database - echo -e " ${INFO} Migrating content of ${blacklistFile} into new database" - database_table_from_file "blacklist" "${blacklistFile}" - fi - if [[ -e "${whitelistFile}" ]]; then - # Store whitelisted domains in database - echo -e " ${INFO} Migrating content of ${whitelistFile} into new database" - database_table_from_file "whitelist" "${whitelistFile}" - fi - if [[ -e "${regexFile}" ]]; then - # Store regex domains in database - echo -e " ${INFO} Migrating content of ${regexFile} into new database" - database_table_from_file "regex" "${regexFile}" - fi + echo -e " ${INFO} Creating new gravity database" + generate_gravity_database + + # Migrate list files to new database + if [[ -e "${adListFile}" ]]; then + # Store adlists domains in database + echo -e " ${INFO} Migrating content of ${adListFile} into new database" + database_table_from_file "adlists" "${adListFile}" + fi + if [[ -e "${blacklistFile}" ]]; then + # Store blacklisted domains in database + echo -e " ${INFO} Migrating content of ${blacklistFile} into new database" + database_table_from_file "blacklist" "${blacklistFile}" + fi + if [[ -e "${whitelistFile}" ]]; then + # Store whitelisted domains in database + echo -e " ${INFO} Migrating content of ${whitelistFile} into new database" + database_table_from_file "whitelist" "${whitelistFile}" + fi + if [[ -e "${regexFile}" ]]; then + # Store regex domains in database + echo -e " ${INFO} Migrating content of ${regexFile} into new database" + database_table_from_file "regex" "${regexFile}" fi } From d46ff1d2d79dbd2cb6f2ee2f919e793a6371757b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 22:08:30 +0200 Subject: [PATCH 71/86] Remove UNIQUE on gravity table as the only column is already PRIMARY KEY. Primary keys must contain UNIQUE values, and cannot contain NULL values Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index fd6d2b28..372a4a29 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -36,7 +36,7 @@ CREATE TABLE adlists ); CREATE TABLE gravity ( - domain TEXT UNIQUE PRIMARY KEY + domain TEXT PRIMARY KEY ); CREATE TABLE info ( From 0bc112ce52e717467835f052353c6c8283e8db20 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 2 May 2019 17:11:39 +0200 Subject: [PATCH 72/86] Query only those columns we actually use when showing domains on the CLI. Show date of last modification instead of addition date for domains. Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 73cb2878..fa81348b 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -170,7 +170,7 @@ Displaylist() { local list listname count num_pipes domain enabled status nicedate listname="${listType}" - data="$(sqlite3 "${gravityDBfile}" "SELECT * FROM ${listType};" 2> /dev/null)" + data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM ${listType};" 2> /dev/null)" if [[ -z $data ]]; then echo -e "Not showing empty ${listname}" @@ -186,9 +186,9 @@ Displaylist() { num_pipes="$(grep -c "^" <<< "$(grep -o "|" <<< "${line}")")" # Extract domain and enabled status based on the obtained number of pipe characters - domain="$(cut -d'|' -f"-$((num_pipes-3))" <<< "${line}")" - enabled="$(cut -d'|' -f"$((num_pipes-2))" <<< "${line}")" - dateadded="$(cut -d'|' -f"$((num_pipes-1))" <<< "${line}")" + domain="$(cut -d'|' -f"-$((num_pipes-1))" <<< "${line}")" + enabled="$(cut -d'|' -f"$((num_pipes))" <<< "${line}")" + datemod="$(cut -d'|' -f"$((num_pipes+1))" <<< "${line}")" # Translate boolean status into human readable string if [[ "${enabled}" -eq 1 ]]; then @@ -198,9 +198,9 @@ Displaylist() { fi # Get nice representation of numerical date stored in database - nicedate=$(date --rfc-2822 -d "@${dateadded}") + nicedate=$(date --rfc-2822 -d "@${datemod}") - echo " ${count}: ${domain} (${status}, added ${nicedate})" + echo " ${count}: ${domain} (${status}, last modified ${nicedate})" count=$((count+1)) done <<< "${data}" fi From 2c3a27e9c7e03ac0464808ae0faf346568f135bc Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Fri, 3 May 2019 12:27:56 +0200 Subject: [PATCH 73/86] Replaces cp chmod with install and add read permission to repository files after checkout and pull Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- automated install/basic-install.sh | 56 +++++++++++++----------------- 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index c1f69efb..0d574f20 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -414,6 +414,8 @@ make_repo() { fi # Clone the repo and return the return code from this command git clone -q --depth 1 "${remoteRepo}" "${directory}" &> /dev/null || return $? + # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git) + chmod -R a+r "${directory}" # Show a colored message showing it's status printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" # Always return 0? Not sure this is correct @@ -447,6 +449,8 @@ update_repo() { git pull --quiet &> /dev/null || return $? # Show a completion message printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" + # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git) + chmod -R a+r "${directory}" # Move back into the original directory cd "${curdir}" &> /dev/null || return 1 return 0 @@ -494,6 +498,8 @@ resetRepo() { printf " %b %s..." "${INFO}" "${str}" # Use git to remove the local changes git reset --hard &> /dev/null || return $? + # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git) + chmod -R a+r "${directory}" # And show the status printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" # Returning success anyway? @@ -1241,8 +1247,7 @@ version_check_dnsmasq() { printf "%b %b Backing up dnsmasq.conf to dnsmasq.conf.orig...\\n" "${OVER}" "${TICK}" printf " %b Restoring default dnsmasq.conf..." "${INFO}" # and replace it with the default - cp -p ${dnsmasq_original_config} ${dnsmasq_conf} - chmod 644 ${dnsmasq_conf} + install -D -m 644 -T ${dnsmasq_original_config} ${dnsmasq_conf} printf "%b %b Restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" # Otherwise, else @@ -1253,19 +1258,17 @@ version_check_dnsmasq() { # If a file cannot be found, printf " %b No dnsmasq.conf found... restoring default dnsmasq.conf..." "${INFO}" # restore the default one - cp -p ${dnsmasq_original_config} ${dnsmasq_conf} + install -D -m 644 -T ${dnsmasq_original_config} ${dnsmasq_conf} printf "%b %b No dnsmasq.conf found... restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" fi printf " %b Copying 01-pihole.conf to /etc/dnsmasq.d/01-pihole.conf..." "${INFO}" # Check to see if dnsmasq directory exists (it may not due to being a fresh install and dnsmasq no longer being a dependency) if [[ ! -d "/etc/dnsmasq.d" ]];then - mkdir "/etc/dnsmasq.d" - chmod 755 "/etc/dnsmasq.d" + install -d -m 755 "/etc/dnsmasq.d" fi # Copy the new Pi-hole DNS config file into the dnsmasq.d directory - cp ${dnsmasq_pihole_01_snippet} ${dnsmasq_pihole_01_location} - chmod 644 ${dnsmasq_pihole_01_location} + install -D -m 644 -T ${dnsmasq_pihole_01_snippet} ${dnsmasq_pihole_01_location} printf "%b %b Copying 01-pihole.conf to /etc/dnsmasq.d/01-pihole.conf\\n" "${OVER}" "${TICK}" # Replace our placeholder values with the GLOBAL DNS variables that we populated earlier # First, swap in the interface to listen on @@ -1381,19 +1384,15 @@ installConfigs() { if [[ "${INSTALL_WEB_SERVER}" == true ]]; then # and if the Web server conf directory does not exist, if [[ ! -d "/etc/lighttpd" ]]; then - # make it - mkdir /etc/lighttpd - # and set the owners - chown "${USER}":root /etc/lighttpd - chmod 755 /etc/lighttpd + # make it and set the owners + install -d -m 755 -o "${USER}" -g root /etc/lighttpd # Otherwise, if the config file already exists elif [[ -f "/etc/lighttpd/lighttpd.conf" ]]; then # back up the original mv /etc/lighttpd/lighttpd.conf /etc/lighttpd/lighttpd.conf.orig fi # and copy in the config file Pi-hole needs - cp ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} /etc/lighttpd/lighttpd.conf - chmod 644 /etc/lighttpd/lighttpd.conf + install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} /etc/lighttpd/lighttpd.conf # Make sure the external.conf file exists, as lighttpd v1.4.50 crashes without it touch /etc/lighttpd/external.conf chmod 644 /etc/lighttpd/external.conf @@ -1427,21 +1426,16 @@ install_manpage() { fi if [[ ! -d "/usr/local/share/man/man8" ]]; then # if not present, create man8 directory - mkdir /usr/local/share/man/man8 - chmod 755 /usr/local/share/man/man8 + install -d -m 755 /usr/local/share/man/man8 fi if [[ ! -d "/usr/local/share/man/man5" ]]; then # if not present, create man5 directory - mkdir /usr/local/share/man/man5 - chmod 755 /usr/local/share/man/man5 + install -d -m 755 /usr/local/share/man/man5 fi # Testing complete, copy the files & update the man db - cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole.8 /usr/local/share/man/man8/pihole.8 - chmod 644 /usr/local/share/man/man8/pihole.8 - cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.8 /usr/local/share/man/man8/pihole-FTL.8 - chmod 644 /usr/local/share/man/man8/pihole-FTL.8 - cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.conf.5 /usr/local/share/man/man5/pihole-FTL.conf.5 - chmod 644 /usr/local/share/man/man5/pihole-FTL.conf.5 + install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/manpages/pihole.8 /usr/local/share/man/man8/pihole.8 + install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.8 /usr/local/share/man/man8/pihole-FTL.8 + install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.conf.5 /usr/local/share/man/man5/pihole-FTL.conf.5 if mandb -q &>/dev/null; then # Updated successfully printf "%b %b man pages installed and database updated\\n" "${OVER}" "${TICK}" @@ -1671,7 +1665,7 @@ installPiholeWeb() { # Install the directory install -d -m 0755 ${PI_HOLE_BLOCKPAGE_DIR} # and the blockpage - install -D ${PI_HOLE_LOCAL_REPO}/advanced/{index,blockingpage}.* ${PI_HOLE_BLOCKPAGE_DIR}/ + install -D -m 644 ${PI_HOLE_LOCAL_REPO}/advanced/{index,blockingpage}.* ${PI_HOLE_BLOCKPAGE_DIR}/ # Remove superseded file if [[ -e "${PI_HOLE_BLOCKPAGE_DIR}/index.js" ]]; then @@ -1721,10 +1715,8 @@ installCron() { local str="Installing latest Cron script" printf "\\n %b %s..." "${INFO}" "${str}" # Copy the cron file over from the local repo - cp ${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole.cron /etc/cron.d/pihole # File must not be world or group writeable and must be owned by root - chmod 644 /etc/cron.d/pihole - chown root:root /etc/cron.d/pihole + install -D -m 644 -T -o root -g root ${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole.cron /etc/cron.d/pihole # Randomize gravity update time sed -i "s/59 1 /$((1 + RANDOM % 58)) $((3 + RANDOM % 2))/" /etc/cron.d/pihole # Randomize update checker time @@ -1855,8 +1847,7 @@ installLogrotate() { local str="Installing latest logrotate script" printf "\\n %b %s..." "${INFO}" "${str}" # Copy the file over from the local repo - cp ${PI_HOLE_LOCAL_REPO}/advanced/Templates/logrotate /etc/pihole/logrotate - chmod 644 /etc/pihole/logrotate + install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/Templates/logrotate /etc/pihole/logrotate # Different operating systems have different user / group # settings for logrotate that makes it impossible to create # a static logrotate file that will work with e.g. @@ -2120,6 +2111,8 @@ checkout_pull_branch() { printf " %b %s" "${INFO}" "$str" git checkout "${branch}" --quiet || return 1 printf "%b %b %s\\n" "${OVER}" "${TICK}" "$str" + # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git) + chmod -R a+r "${directory}" git_pull=$(git pull || return 1) @@ -2532,8 +2525,7 @@ main() { # Display welcome dialogs welcomeDialogs # Create directory for Pi-hole storage - mkdir -p /etc/pihole/ - chmod 755 /ect/pihole/ + install -d -m 755 /etc/pihole/ # Determine available interfaces get_available_interfaces # Find interfaces and let the user choose one From 69c06ba6fe18e07f3b5611e8d849a996ff1931f1 Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Sun, 5 May 2019 00:32:10 +0200 Subject: [PATCH 74/86] Sets permissions for dnsmaq.conf Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- automated install/basic-install.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 0d574f20..208bfe7c 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1366,6 +1366,7 @@ installConfigs() { # Format: Name;Primary IPv4;Secondary IPv4;Primary IPv6;Secondary IPv6 # Some values may be empty (for example: DNS servers without IPv6 support) echo "${DNS_SERVERS}" > "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" + chmod 644 "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" # Install empty file if it does not exist if [[ ! -r "${PI_HOLE_CONFIG_DIR}/pihole-FTL.conf" ]]; then @@ -1692,7 +1693,7 @@ installPiholeWeb() { local str="Installing sudoer file" printf "\\n %b %s..." "${INFO}" "${str}" # Make the .d directory if it doesn't exist - mkdir -p /etc/sudoers.d/ + install -d -m 755 /etc/sudoers.d/ # and copy in the pihole sudoers file install -m 0640 ${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole.sudo /etc/sudoers.d/pihole # Add lighttpd user (OS dependent) to sudoers file @@ -2259,6 +2260,7 @@ disable_dnsmasq() { fi # Create /etc/dnsmasq.conf echo "conf-dir=/etc/dnsmasq.d" > "${conffile}" + chmod 644 "${conffile}" } get_binary_name() { From a25f331e83c94320b634b7954d071a0688ff4730 Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Sun, 5 May 2019 09:20:49 +0200 Subject: [PATCH 75/86] Adds chmod 644 for /var/www and /var/www/html Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- automated install/basic-install.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 208bfe7c..79a6c94a 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -867,6 +867,8 @@ setIFCFG() { echo "DNS2=$PIHOLE_DNS_2" echo "USERCTL=no" }> "${IFCFG_FILE}" + chmod 644 "${IFCFG_FILE}" + chown root:root "${IFCFG_FILE}" # Use ip to immediately set the new address ip addr replace dev "${PIHOLE_INTERFACE}" "${IPV4_ADDRESS}" # If NetworkMangler command line interface exists and ready to mangle, @@ -1896,6 +1898,8 @@ installPihole() { # make the Web directory if necessary install -d -m 0755 ${webroot} fi + chmod a+rx /var/www + chmod a+rx /var/www/html if [[ "${INSTALL_WEB_SERVER}" == true ]]; then # Set the owner and permissions From e19adccd9c623eeba20e46a6347108e917af133e Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Sun, 5 May 2019 22:34:13 +0200 Subject: [PATCH 76/86] Moves chmod for /var/www/html into INSTALL_WEB_SERVER block Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- automated install/basic-install.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 79a6c94a..da1a6f9b 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1898,13 +1898,14 @@ installPihole() { # make the Web directory if necessary install -d -m 0755 ${webroot} fi - chmod a+rx /var/www - chmod a+rx /var/www/html if [[ "${INSTALL_WEB_SERVER}" == true ]]; then # Set the owner and permissions chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} ${webroot} chmod 0775 ${webroot} + # Repair permissions if /var/www/html is not world readable + chmod a+rx /var/www + chmod a+rx /var/www/html # Give pihole access to the Web server group usermod -a -G ${LIGHTTPD_GROUP} pihole # If the lighttpd command is executable, From 4d23b3267f1ff16a4392c481f7e84473dd42649c Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Tue, 7 May 2019 10:46:16 +0200 Subject: [PATCH 77/86] Adds X to chmod for git operations Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- automated install/basic-install.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index da1a6f9b..585896c1 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -415,7 +415,7 @@ make_repo() { # Clone the repo and return the return code from this command git clone -q --depth 1 "${remoteRepo}" "${directory}" &> /dev/null || return $? # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git) - chmod -R a+r "${directory}" + chmod -R a+rX "${directory}" # Show a colored message showing it's status printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" # Always return 0? Not sure this is correct @@ -450,7 +450,7 @@ update_repo() { # Show a completion message printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git) - chmod -R a+r "${directory}" + chmod -R a+rX "${directory}" # Move back into the original directory cd "${curdir}" &> /dev/null || return 1 return 0 @@ -499,7 +499,7 @@ resetRepo() { # Use git to remove the local changes git reset --hard &> /dev/null || return $? # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git) - chmod -R a+r "${directory}" + chmod -R a+rX "${directory}" # And show the status printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" # Returning success anyway? @@ -2118,7 +2118,7 @@ checkout_pull_branch() { git checkout "${branch}" --quiet || return 1 printf "%b %b %s\\n" "${OVER}" "${TICK}" "$str" # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git) - chmod -R a+r "${directory}" + chmod -R a+rX "${directory}" git_pull=$(git pull || return 1) From 2e6198077d8f505415b8f494357eb40cdec12277 Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Thu, 9 May 2019 12:23:44 +0200 Subject: [PATCH 78/86] chmod 644 adlists.list Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- automated install/basic-install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 585896c1..00c5c95a 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1193,6 +1193,7 @@ chooseBlocklists() { do appendToListsFile "${choice}" done + chmod 644 "${adlistFile}" } # Accept a string parameter, it must be one of the default lists From a3d2a1062fe12f4da89c21274b8228c133dd5bfa Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Fri, 10 May 2019 11:13:19 +0200 Subject: [PATCH 79/86] Makes blocklists world readable Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- gravity.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gravity.sh b/gravity.sh index a2f2c782..445274ce 100755 --- a/gravity.sh +++ b/gravity.sh @@ -328,6 +328,7 @@ gravity_DownloadBlocklistFromUrl() { # Parse source files into domains format gravity_ParseFileIntoDomains() { local source="${1}" destination="${2}" firstLine abpFilter + chmod 644 "${source}" # Determine if we are parsing a consolidated list if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then @@ -345,6 +346,7 @@ gravity_ParseFileIntoDomains() { sed -r '/(\/|#).*$/d' | \ sed -r 's/^.*\s+//g' | \ sed -r '/([^\.]+\.)+[^\.]{2,}/!d' > "${destination}" + chmod 644 "${destination}" return 0 fi @@ -375,6 +377,7 @@ gravity_ParseFileIntoDomains() { if($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" } if($0) { print $0 } }' "${source}" > "${destination}" + chmod 644 "${destination}" # Determine if there are Adblock exception rules # https://adblockplus.org/filters @@ -391,6 +394,7 @@ gravity_ParseFileIntoDomains() { # Remove exceptions comm -23 "${destination}" <(sort "${destination}.exceptionsFile.tmp") > "${source}" + chmod 644 "${source}" mv "${source}" "${destination}" fi @@ -427,6 +431,7 @@ gravity_ParseFileIntoDomains() { gravity_Cleanup "error" fi fi + chmod 644 "${destination}" } # Create (unfiltered) "Matter and Light" consolidated list @@ -440,6 +445,7 @@ gravity_ConsolidateDownloadedBlocklists() { # Empty $matterAndLight if it already exists, otherwise, create it : > "${piholeDir}/${matterAndLight}" + chmod 644 "${piholeDir}/${matterAndLight}" # Loop through each *.domains file for i in "${activeDomains[@]}"; do From e076db5a4e1e15cb9d8e8707bf2a5da7be125489 Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Fri, 10 May 2019 11:19:28 +0200 Subject: [PATCH 80/86] Changes ownership of macvendor.db to pihole Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- automated install/basic-install.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 00c5c95a..a00aee16 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2216,6 +2216,8 @@ FTLinstall() { # Before stopping FTL, we download the macvendor database curl -sSL "https://ftl.pi-hole.net/macvendor.db" -o "${PI_HOLE_CONFIG_DIR}/macvendor.db" || true + chmod 644 "${PI_HOLE_CONFIG_DIR}/macvendor.db" + chown pihole:pihole "${PI_HOLE_CONFIG_DIR}/macvendor.db" # Stop pihole-FTL service if available stop_service pihole-FTL &> /dev/null From 225285cb977ced4047de542978749c35a3ee43bf Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Sat, 11 May 2019 11:14:07 +0200 Subject: [PATCH 81/86] Removes chmod for ${source} and superfluous chmod for ${destination} Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- gravity.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index 445274ce..8fdfa5d7 100755 --- a/gravity.sh +++ b/gravity.sh @@ -328,7 +328,6 @@ gravity_DownloadBlocklistFromUrl() { # Parse source files into domains format gravity_ParseFileIntoDomains() { local source="${1}" destination="${2}" firstLine abpFilter - chmod 644 "${source}" # Determine if we are parsing a consolidated list if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then @@ -377,7 +376,6 @@ gravity_ParseFileIntoDomains() { if($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" } if($0) { print $0 } }' "${source}" > "${destination}" - chmod 644 "${destination}" # Determine if there are Adblock exception rules # https://adblockplus.org/filters @@ -394,7 +392,6 @@ gravity_ParseFileIntoDomains() { # Remove exceptions comm -23 "${destination}" <(sort "${destination}.exceptionsFile.tmp") > "${source}" - chmod 644 "${source}" mv "${source}" "${destination}" fi From 3f90261520aae689e213637bc27d0754ab0ce97b Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Sun, 12 May 2019 11:25:48 +0200 Subject: [PATCH 82/86] Moves chmod after every creation of ${destination} instead of end of function Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- gravity.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 8fdfa5d7..2a22b792 100755 --- a/gravity.sh +++ b/gravity.sh @@ -376,6 +376,7 @@ gravity_ParseFileIntoDomains() { if($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" } if($0) { print $0 } }' "${source}" > "${destination}" + chmod 644 "${destination}" # Determine if there are Adblock exception rules # https://adblockplus.org/filters @@ -393,6 +394,7 @@ gravity_ParseFileIntoDomains() { # Remove exceptions comm -23 "${destination}" <(sort "${destination}.exceptionsFile.tmp") > "${source}" mv "${source}" "${destination}" + chmod 644 "${destination}" fi echo -e "${OVER} ${TICK} Format: Adblock" @@ -416,11 +418,13 @@ gravity_ParseFileIntoDomains() { # Print if nonempty length { print } ' "${source}" 2> /dev/null > "${destination}" + chmod 644 "${destination}" echo -e "${OVER} ${TICK} Format: URL" else # Default: Keep hosts/domains file in same format as it was downloaded output=$( { mv "${source}" "${destination}"; } 2>&1 ) + chmod 644 "${destination}" if [[ ! -e "${destination}" ]]; then echo -e "\\n ${CROSS} Unable to move tmp file to ${piholeDir} @@ -428,7 +432,6 @@ gravity_ParseFileIntoDomains() { gravity_Cleanup "error" fi fi - chmod 644 "${destination}" } # Create (unfiltered) "Matter and Light" consolidated list From efbd42f7b5e7a9532aeefa13ff1965cb62d67981 Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Sun, 12 May 2019 11:27:25 +0200 Subject: [PATCH 83/86] Moves creation of pihole user before FTL installation Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- automated install/basic-install.sh | 5 ++--- test/test_automated_install.py | 9 +++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a00aee16..9520c44c 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1890,9 +1890,6 @@ accountForRefactor() { # Install base files and web interface installPihole() { - # Create the pihole user - create_pihole_user - # If the user wants to install the Web interface, if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then if [[ ! -d "${webroot}" ]]; then @@ -2591,6 +2588,8 @@ main() { else LIGHTTPD_ENABLED=false fi + # Create the pihole user + create_pihole_user # Check if FTL is installed - do this early on as FTL is a hard dependency for Pi-hole if ! FTLdetect; then printf " %b FTL Engine not installed\\n" "${CROSS}" diff --git a/test/test_automated_install.py b/test/test_automated_install.py index 853048d1..282c627d 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -398,6 +398,7 @@ def test_FTL_detect_aarch64_no_errors(Pihole): ) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh + create_pihole_user FTLdetect ''') expected_stdout = info_box + ' FTL Checks...' @@ -418,6 +419,7 @@ def test_FTL_detect_armv6l_no_errors(Pihole): mock_command('ldd', {'/bin/ls': ('/lib/ld-linux-armhf.so.3', '0')}, Pihole) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh + create_pihole_user FTLdetect ''') expected_stdout = info_box + ' FTL Checks...' @@ -439,6 +441,7 @@ def test_FTL_detect_armv7l_no_errors(Pihole): mock_command('ldd', {'/bin/ls': ('/lib/ld-linux-armhf.so.3', '0')}, Pihole) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh + create_pihole_user FTLdetect ''') expected_stdout = info_box + ' FTL Checks...' @@ -455,6 +458,7 @@ def test_FTL_detect_x86_64_no_errors(Pihole): ''' detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh + create_pihole_user FTLdetect ''') expected_stdout = info_box + ' FTL Checks...' @@ -471,6 +475,7 @@ def test_FTL_detect_unknown_no_errors(Pihole): mock_command('uname', {'-m': ('mips', '0')}, Pihole) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh + create_pihole_user FTLdetect ''') expected_stdout = 'Not able to detect architecture (unknown: mips)' @@ -484,6 +489,7 @@ def test_FTL_download_aarch64_no_errors(Pihole): download_binary = Pihole.run(''' source /opt/pihole/basic-install.sh binary="pihole-FTL-aarch64-linux-gnu" + create_pihole_user FTLinstall ''') expected_stdout = tick_box + ' Downloading and Installing FTL' @@ -498,6 +504,7 @@ def test_FTL_download_unknown_fails_no_errors(Pihole): download_binary = Pihole.run(''' source /opt/pihole/basic-install.sh binary="pihole-FTL-mips" + create_pihole_user FTLinstall ''') expected_stdout = cross_box + ' Downloading and Installing FTL' @@ -514,6 +521,7 @@ def test_FTL_download_binary_unset_no_errors(Pihole): ''' download_binary = Pihole.run(''' source /opt/pihole/basic-install.sh + create_pihole_user FTLinstall ''') expected_stdout = cross_box + ' Downloading and Installing FTL' @@ -530,6 +538,7 @@ def test_FTL_binary_installed_and_responsive_no_errors(Pihole): ''' installed_binary = Pihole.run(''' source /opt/pihole/basic-install.sh + create_pihole_user FTLdetect pihole-FTL version ''') From ab2f8a0c09ecdb1ce1499f48267c05021e2bb29b Mon Sep 17 00:00:00 2001 From: David Haguenauer Date: Fri, 10 May 2019 10:13:23 -0400 Subject: [PATCH 84/86] Quote variables in basic-install.sh This greatly reduces the number of warnings emitted by ShellCheck, and in turn should make it more likely that errors are caught in the future. Signed-off-by: David Haguenauer --- automated install/basic-install.sh | 116 ++++++++++++++--------------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 18c47485..35d3305c 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -118,7 +118,7 @@ done # If the color table file exists, if [[ -f "${coltable}" ]]; then # source it - source ${coltable} + source "${coltable}" # Otherwise, else # Set these values so the installer can still run in color @@ -182,14 +182,14 @@ if is_command apt-get ; then # A variable to store the command used to update the package cache UPDATE_PKG_CACHE="${PKG_MANAGER} update" # An array for something... - PKG_INSTALL=(${PKG_MANAGER} --yes --no-install-recommends install) + PKG_INSTALL=("${PKG_MANAGER}" --yes --no-install-recommends install) # grep -c will return 1 retVal on 0 matches, block this throwing the set -e with an OR TRUE PKG_COUNT="${PKG_MANAGER} -s -o Debug::NoLocking=true upgrade | grep -c ^Inst || true" # Some distros vary slightly so these fixes for dependencies may apply # on Ubuntu 18.04.1 LTS we need to add the universe repository to gain access to dialog and dhcpcd5 APT_SOURCES="/etc/apt/sources.list" if awk 'BEGIN{a=1;b=0}/bionic main/{a=0}/bionic.*universe/{b=1}END{exit a + b}' ${APT_SOURCES}; then - if ! whiptail --defaultno --title "Dependencies Require Update to Allowed Repositories" --yesno "Would you like to enable 'universe' repository?\\n\\nThis repository is required by the following packages:\\n\\n- dhcpcd5\\n- dialog" ${r} ${c}; then + if ! whiptail --defaultno --title "Dependencies Require Update to Allowed Repositories" --yesno "Would you like to enable 'universe' repository?\\n\\nThis repository is required by the following packages:\\n\\n- dhcpcd5\\n- dialog" "${r}" "${c}"; then printf " %b Aborting installation: dependencies could not be installed.\\n" "${CROSS}" exit # exit the installer else @@ -201,7 +201,7 @@ if is_command apt-get ; then fi fi # Debian 7 doesn't have iproute2 so if the dry run install is successful, - if ${PKG_MANAGER} install --dry-run iproute2 > /dev/null 2>&1; then + if "${PKG_MANAGER}" install --dry-run iproute2 > /dev/null 2>&1; then # we can install it iproute_pkg="iproute2" # Otherwise, @@ -222,7 +222,7 @@ if is_command apt-get ; then # Check if installed php is v 7.0, or newer to determine packages to install if [[ "$phpInsNewer" != true ]]; then # Prefer the php metapackage if it's there - if ${PKG_MANAGER} install --dry-run php > /dev/null 2>&1; then + if "${PKG_MANAGER}" install --dry-run php > /dev/null 2>&1; then phpVer="php" # fall back on the php5 packages else @@ -233,19 +233,19 @@ if is_command apt-get ; then phpVer="php$phpInsMajor.$phpInsMinor" fi # We also need the correct version for `php-sqlite` (which differs across distros) - if ${PKG_MANAGER} install --dry-run ${phpVer}-sqlite3 > /dev/null 2>&1; then + if "${PKG_MANAGER}" install --dry-run "${phpVer}"-sqlite3 > /dev/null 2>&1; then phpSqlite="sqlite3" else phpSqlite="sqlite" fi # Since our install script is so large, we need several other programs to successfully get a machine provisioned # These programs are stored in an array so they can be looped through later - INSTALLER_DEPS=(apt-utils dialog debconf dhcpcd5 git ${iproute_pkg} whiptail) + INSTALLER_DEPS=(apt-utils dialog debconf dhcpcd5 git "${iproute_pkg}" whiptail) # Pi-hole itself has several dependencies that also need to be installed PIHOLE_DEPS=(cron curl dnsutils iputils-ping lsof netcat psmisc sudo unzip wget idn2 sqlite3 libcap2-bin dns-root-data resolvconf libcap2) # The Web dashboard has some that also need to be installed # It's useful to separate the two since our repos are also setup as "Core" code and "Web" code - PIHOLE_WEB_DEPS=(lighttpd ${phpVer}-common ${phpVer}-cgi ${phpVer}-${phpSqlite}) + PIHOLE_WEB_DEPS=(lighttpd "${phpVer}"-common "${phpVer}"-cgi "${phpVer}-${phpSqlite}") # The Web server user, LIGHTTPD_USER="www-data" # group, @@ -281,7 +281,7 @@ elif is_command rpm ; then # Fedora and family update cache on every PKG_INSTALL call, no need for a separate update. UPDATE_PKG_CACHE=":" - PKG_INSTALL=(${PKG_MANAGER} install -y) + PKG_INSTALL=("${PKG_MANAGER}" install -y) PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l" INSTALLER_DEPS=(dialog git iproute newt procps-ng which) PIHOLE_DEPS=(bind-utils cronie curl findutils nmap-ncat sudo unzip wget libidn2 psmisc sqlite libcap) @@ -319,7 +319,7 @@ elif is_command rpm ; then # The default php on CentOS 7.x is 5.4 which is EOL # Check if the version of PHP available via installed repositories is >= to PHP 7 - AVAILABLE_PHP_VERSION=$(${PKG_MANAGER} info php | grep -i version | grep -o '[0-9]\+' | head -1) + AVAILABLE_PHP_VERSION=$("${PKG_MANAGER}" info php | grep -i version | grep -o '[0-9]\+' | head -1) if [[ $AVAILABLE_PHP_VERSION -ge $SUPPORTED_CENTOS_PHP_VERSION ]]; then # Since PHP 7 is available by default, install via default PHP package names : # do nothing as PHP is current @@ -329,7 +329,7 @@ elif is_command rpm ; then rpm -q ${REMI_PKG} &> /dev/null || rc=$? if [[ $rc -ne 0 ]]; then # The PHP version available via default repositories is older than version 7 - if ! whiptail --defaultno --title "PHP 7 Update (recommended)" --yesno "PHP 7.x is recommended for both security and language features.\\nWould you like to install PHP7 via Remi's RPM repository?\\n\\nSee: https://rpms.remirepo.net for more information" ${r} ${c}; then + if ! whiptail --defaultno --title "PHP 7 Update (recommended)" --yesno "PHP 7.x is recommended for both security and language features.\\nWould you like to install PHP7 via Remi's RPM repository?\\n\\nSee: https://rpms.remirepo.net for more information" "${r}" "${c}"; then # User decided to NOT update PHP from REMI, attempt to install the default available PHP version printf " %b User opt-out of PHP 7 upgrade on CentOS. Deprecated PHP may be in use.\\n" "${INFO}" : # continue with unsupported php version @@ -352,7 +352,7 @@ elif is_command rpm ; then fi else # Warn user of unsupported version of Fedora or CentOS - if ! whiptail --defaultno --title "Unsupported RPM based distribution" --yesno "Would you like to continue installation on an unsupported RPM based distribution?\\n\\nPlease ensure the following packages have been installed manually:\\n\\n- lighttpd\\n- lighttpd-fastcgi\\n- PHP version 7+" ${r} ${c}; then + if ! whiptail --defaultno --title "Unsupported RPM based distribution" --yesno "Would you like to continue installation on an unsupported RPM based distribution?\\n\\nPlease ensure the following packages have been installed manually:\\n\\n- lighttpd\\n- lighttpd-fastcgi\\n- PHP version 7+" "${r}" "${c}"; then printf " %b Aborting installation due to unsupported RPM based distribution\\n" "${CROSS}" exit # exit the installer else @@ -543,15 +543,15 @@ get_available_interfaces() { # A function for displaying the dialogs the user sees when first running the installer welcomeDialogs() { # Display the welcome dialog using an appropriately sized window via the calculation conducted earlier in the script - whiptail --msgbox --backtitle "Welcome" --title "Pi-hole automated installer" "\\n\\nThis installer will transform your device into a network-wide ad blocker!" ${r} ${c} + whiptail --msgbox --backtitle "Welcome" --title "Pi-hole automated installer" "\\n\\nThis installer will transform your device into a network-wide ad blocker!" "${r}" "${c}" # Request that users donate if they enjoy the software since we all work on it in our free time - whiptail --msgbox --backtitle "Plea" --title "Free and open source" "\\n\\nThe Pi-hole is free, but powered by your donations: http://pi-hole.net/donate" ${r} ${c} + whiptail --msgbox --backtitle "Plea" --title "Free and open source" "\\n\\nThe Pi-hole is free, but powered by your donations: http://pi-hole.net/donate" "${r}" "${c}" # Explain the need for a static address whiptail --msgbox --backtitle "Initiating network interface" --title "Static IP Needed" "\\n\\nThe Pi-hole is a SERVER so it needs a STATIC IP ADDRESS to function properly. -In the next section, you can choose to use your current network settings (DHCP) or to manually edit them." ${r} ${c} +In the next section, you can choose to use your current network settings (DHCP) or to manually edit them." "${r}" "${c}" } # We need to make sure there is enough space before installing, so there is a function to check this @@ -638,7 +638,7 @@ chooseInterface() { # Feed the available interfaces into this while loop done <<< "${availableInterfaces}" # The whiptail command that will be run, stored in a variable - chooseInterfaceCmd=(whiptail --separate-output --radiolist "Choose An Interface (press space to select)" ${r} ${c} ${interfaceCount}) + chooseInterfaceCmd=(whiptail --separate-output --radiolist "Choose An Interface (press space to select)" "${r}" "${c}" "${interfaceCount}") # Now run the command using the interfaces saved into the array chooseInterfaceOptions=$("${chooseInterfaceCmd[@]}" "${interfacesArray[@]}" 2>&1 >/dev/tty) || \ # If the user chooses Cancel, exit @@ -719,7 +719,7 @@ useIPv6dialog() { # If the IPV6_ADDRESS contains a value if [[ ! -z "${IPV6_ADDRESS}" ]]; then # Display that IPv6 is supported and will be used - whiptail --msgbox --backtitle "IPv6..." --title "IPv6 Supported" "$IPV6_ADDRESS will be used to block ads." ${r} ${c} + whiptail --msgbox --backtitle "IPv6..." --title "IPv6 Supported" "$IPV6_ADDRESS will be used to block ads." "${r}" "${c}" fi } @@ -729,7 +729,7 @@ use4andor6() { local useIPv4 local useIPv6 # Let use select IPv4 and/or IPv6 via a checklist - cmd=(whiptail --separate-output --checklist "Select Protocols (press space to select)" ${r} ${c} 2) + cmd=(whiptail --separate-output --checklist "Select Protocols (press space to select)" "${r}" "${c}" 2) # In an array, show the options available: # IPv4 (on by default) options=(IPv4 "Block ads over IPv4" on @@ -778,11 +778,11 @@ getStaticIPv4Settings() { # This is useful for users that are using DHCP reservations; then we can just use the information gathered via our functions if whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Do you want to use your current network settings as a static address? IP address: ${IPV4_ADDRESS} - Gateway: ${IPv4gw}" ${r} ${c}; then + Gateway: ${IPv4gw}" "${r}" "${c}"; then # If they choose yes, let the user know that the IP address will not be available via DHCP and may cause a conflict. whiptail --msgbox --backtitle "IP information" --title "FYI: IP Conflict" "It is possible your router could still try to assign this IP to a device, which would cause a conflict. But in most cases the router is smart enough to not do that. If you are worried, either manually set the address, or modify the DHCP reservation pool so it does not include the IP you want. -It is also possible to use a DHCP reservation, but if you are going to do that, you might as well set a static address." ${r} ${c} +It is also possible to use a DHCP reservation, but if you are going to do that, you might as well set a static address." "${r}" "${c}" # Nothing else to do since the variables are already set above else # Otherwise, we need to ask the user to input their desired settings. @@ -791,13 +791,13 @@ It is also possible to use a DHCP reservation, but if you are going to do that, until [[ "${ipSettingsCorrect}" = True ]]; do # Ask for the IPv4 address - IPV4_ADDRESS=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 address" --inputbox "Enter your desired IPv4 address" ${r} ${c} "${IPV4_ADDRESS}" 3>&1 1>&2 2>&3) || \ + IPV4_ADDRESS=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 address" --inputbox "Enter your desired IPv4 address" "${r}" "${c}" "${IPV4_ADDRESS}" 3>&1 1>&2 2>&3) || \ # Cancelling IPv4 settings window { ipSettingsCorrect=False; echo -e " ${COL_LIGHT_RED}Cancel was selected, exiting installer${COL_NC}"; exit 1; } printf " %b Your static IPv4 address: %s\\n" "${INFO}" "${IPV4_ADDRESS}" # Ask for the gateway - IPv4gw=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 gateway (router)" --inputbox "Enter your desired IPv4 default gateway" ${r} ${c} "${IPv4gw}" 3>&1 1>&2 2>&3) || \ + IPv4gw=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 gateway (router)" --inputbox "Enter your desired IPv4 default gateway" "${r}" "${c}" "${IPv4gw}" 3>&1 1>&2 2>&3) || \ # Cancelling gateway settings window { ipSettingsCorrect=False; echo -e " ${COL_LIGHT_RED}Cancel was selected, exiting installer${COL_NC}"; exit 1; } printf " %b Your static IPv4 gateway: %s\\n" "${INFO}" "${IPv4gw}" @@ -805,7 +805,7 @@ It is also possible to use a DHCP reservation, but if you are going to do that, # Give the user a chance to review their settings before moving on if whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Are these settings correct? IP address: ${IPV4_ADDRESS} - Gateway: ${IPv4gw}" ${r} ${c}; then + Gateway: ${IPv4gw}" "${r}" "${c}"; then # After that's done, the loop ends and we move on ipSettingsCorrect=True else @@ -933,7 +933,7 @@ valid_ip() { # and set the new one to a dot (period) IFS='.' # Put the IP into an array - ip=(${ip}) + ip=("${ip}") # Restore the IFS to what it was IFS=${OIFS} ## Evaluate each octet by checking if it's less than or equal to 255 (the max for each octet) @@ -943,7 +943,7 @@ valid_ip() { stat=$? fi # Return the exit code - return ${stat} + return "${stat}" } # A function to choose the upstream DNS provider(s) @@ -973,7 +973,7 @@ setDNS() { # Restore the IFS to what it was IFS=${OIFS} # In a whiptail dialog, show the options - DNSchoices=$(whiptail --separate-output --menu "Select Upstream DNS Provider. To use your own, select Custom." ${r} ${c} 7 \ + DNSchoices=$(whiptail --separate-output --menu "Select Upstream DNS Provider. To use your own, select Custom." "${r}" "${c}" 7 \ "${DNSChooseOptions[@]}" 2>&1 >/dev/tty) || \ # exit if Cancel is selected { printf " %bCancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; exit 1; } @@ -1003,7 +1003,7 @@ setDNS() { fi # Dialog for the user to enter custom upstream servers - piholeDNS=$(whiptail --backtitle "Specify Upstream DNS Provider(s)" --inputbox "Enter your desired upstream DNS provider(s), separated by a comma.\\n\\nFor example '8.8.8.8, 8.8.4.4'" ${r} ${c} "${prePopulate}" 3>&1 1>&2 2>&3) || \ + piholeDNS=$(whiptail --backtitle "Specify Upstream DNS Provider(s)" --inputbox "Enter your desired upstream DNS provider(s), separated by a comma.\\n\\nFor example '8.8.8.8, 8.8.4.4'" "${r}" "${c}" "${prePopulate}" 3>&1 1>&2 2>&3) || \ { printf " %bCancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; exit 1; } # Clean user input and replace whitespace with comma. piholeDNS=$(sed 's/[, \t]\+/,/g' <<< "${piholeDNS}") @@ -1036,7 +1036,7 @@ setDNS() { # Otherwise, else # Show the settings - if (whiptail --backtitle "Specify Upstream DNS Provider(s)" --title "Upstream DNS Provider(s)" --yesno "Are these settings correct?\\n DNS Server 1: $PIHOLE_DNS_1\\n DNS Server 2: ${PIHOLE_DNS_2}" ${r} ${c}); then + if (whiptail --backtitle "Specify Upstream DNS Provider(s)" --title "Upstream DNS Provider(s)" --yesno "Are these settings correct?\\n DNS Server 1: $PIHOLE_DNS_1\\n DNS Server 2: ${PIHOLE_DNS_2}" "${r}" "${c}"); then # and break from the loop since the servers are valid DNSSettingsCorrect=True # Otherwise, @@ -1127,7 +1127,7 @@ setAdminFlag() { local WebChoices # Similar to the logging function, ask what the user wants - WebToggleCommand=(whiptail --separate-output --radiolist "Do you wish to install the web admin interface?" ${r} ${c} 6) + WebToggleCommand=(whiptail --separate-output --radiolist "Do you wish to install the web admin interface?" "${r}" "${c}" 6) # with the default being enabled WebChooseOptions=("On (Recommended)" "" on Off "" off) @@ -1248,11 +1248,11 @@ version_check_dnsmasq() { printf " it is from a previous Pi-hole install.\\n" printf " %b Backing up dnsmasq.conf to dnsmasq.conf.orig..." "${INFO}" # so backup the original file - mv -f ${dnsmasq_conf} ${dnsmasq_conf_orig} + mv -f "${dnsmasq_conf}" "${dnsmasq_conf_orig}" printf "%b %b Backing up dnsmasq.conf to dnsmasq.conf.orig...\\n" "${OVER}" "${TICK}" printf " %b Restoring default dnsmasq.conf..." "${INFO}" # and replace it with the default - install -D -m 644 -T ${dnsmasq_original_config} ${dnsmasq_conf} + install -D -m 644 -T "${dnsmasq_original_config}" "${dnsmasq_conf}" printf "%b %b Restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" # Otherwise, else @@ -1263,7 +1263,7 @@ version_check_dnsmasq() { # If a file cannot be found, printf " %b No dnsmasq.conf found... restoring default dnsmasq.conf..." "${INFO}" # restore the default one - install -D -m 644 -T ${dnsmasq_original_config} ${dnsmasq_conf} + install -D -m 644 -T "${dnsmasq_original_config}" "${dnsmasq_conf}" printf "%b %b No dnsmasq.conf found... restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" fi @@ -1273,37 +1273,37 @@ version_check_dnsmasq() { install -d -m 755 "/etc/dnsmasq.d" fi # Copy the new Pi-hole DNS config file into the dnsmasq.d directory - install -D -m 644 -T ${dnsmasq_pihole_01_snippet} ${dnsmasq_pihole_01_location} + install -D -m 644 -T "${dnsmasq_pihole_01_snippet}" "${dnsmasq_pihole_01_location}" printf "%b %b Copying 01-pihole.conf to /etc/dnsmasq.d/01-pihole.conf\\n" "${OVER}" "${TICK}" # Replace our placeholder values with the GLOBAL DNS variables that we populated earlier # First, swap in the interface to listen on - sed -i "s/@INT@/$PIHOLE_INTERFACE/" ${dnsmasq_pihole_01_location} + sed -i "s/@INT@/$PIHOLE_INTERFACE/" "${dnsmasq_pihole_01_location}" if [[ "${PIHOLE_DNS_1}" != "" ]]; then # Then swap in the primary DNS server - sed -i "s/@DNS1@/$PIHOLE_DNS_1/" ${dnsmasq_pihole_01_location} + sed -i "s/@DNS1@/$PIHOLE_DNS_1/" "${dnsmasq_pihole_01_location}" else # - sed -i '/^server=@DNS1@/d' ${dnsmasq_pihole_01_location} + sed -i '/^server=@DNS1@/d' "${dnsmasq_pihole_01_location}" fi if [[ "${PIHOLE_DNS_2}" != "" ]]; then # Then swap in the primary DNS server - sed -i "s/@DNS2@/$PIHOLE_DNS_2/" ${dnsmasq_pihole_01_location} + sed -i "s/@DNS2@/$PIHOLE_DNS_2/" "${dnsmasq_pihole_01_location}" else # - sed -i '/^server=@DNS2@/d' ${dnsmasq_pihole_01_location} + sed -i '/^server=@DNS2@/d' "${dnsmasq_pihole_01_location}" fi # - sed -i 's/^#conf-dir=\/etc\/dnsmasq.d$/conf-dir=\/etc\/dnsmasq.d/' ${dnsmasq_conf} + sed -i 's/^#conf-dir=\/etc\/dnsmasq.d$/conf-dir=\/etc\/dnsmasq.d/' "${dnsmasq_conf}" # If the user does not want to enable logging, if [[ "${QUERY_LOGGING}" == false ]] ; then # Disable it by commenting out the directive in the DNS config file - sed -i 's/^log-queries/#log-queries/' ${dnsmasq_pihole_01_location} + sed -i 's/^log-queries/#log-queries/' "${dnsmasq_pihole_01_location}" # Otherwise, else # enable it by uncommenting the directive in the DNS config file - sed -i 's/^#log-queries/log-queries/' ${dnsmasq_pihole_01_location} + sed -i 's/^#log-queries/log-queries/' "${dnsmasq_pihole_01_location}" fi } @@ -1647,7 +1647,7 @@ install_dependent_packages() { # Install Fedora/CentOS packages for i in "${argArray1[@]}"; do printf " %b Checking for %s..." "${INFO}" "${i}" - if ${PKG_MANAGER} -q list installed "${i}" &> /dev/null; then + if "${PKG_MANAGER}" -q list installed "${i}" &> /dev/null; then printf "%b %b Checking for %s" "${OVER}" "${TICK}" "${i}" else printf "%b %b Checking for %s (will be installed)" "${OVER}" "${INFO}" "${i}" @@ -1765,7 +1765,7 @@ configureFirewall() { # If a firewall is running, if firewall-cmd --state &> /dev/null; then # ask if the user wants to install Pi-hole's default firewall rules - whiptail --title "Firewall in use" --yesno "We have detected a running firewall\\n\\nPi-hole currently requires HTTP and DNS port access.\\n\\n\\n\\nInstall Pi-hole default firewall rules?" ${r} ${c} || \ + whiptail --title "Firewall in use" --yesno "We have detected a running firewall\\n\\nPi-hole currently requires HTTP and DNS port access.\\n\\n\\n\\nInstall Pi-hole default firewall rules?" "${r}" "${c}" || \ { printf " %b Not installing firewall rulesets.\\n" "${INFO}"; return 0; } printf " %b Configuring FirewallD for httpd and pihole-FTL\\n" "${TICK}" # Allow HTTP and DNS traffic @@ -1778,7 +1778,7 @@ configureFirewall() { # If chain Policy is not ACCEPT or last Rule is not ACCEPT # then check and insert our Rules above the DROP/REJECT Rule. if iptables -S INPUT | head -n1 | grep -qv '^-P.*ACCEPT$' || iptables -S INPUT | tail -n1 | grep -qv '^-\(A\|P\).*ACCEPT$'; then - whiptail --title "Firewall in use" --yesno "We have detected a running firewall\\n\\nPi-hole currently requires HTTP and DNS port access.\\n\\n\\n\\nInstall Pi-hole default firewall rules?" ${r} ${c} || \ + whiptail --title "Firewall in use" --yesno "We have detected a running firewall\\n\\nPi-hole currently requires HTTP and DNS port access.\\n\\n\\n\\nInstall Pi-hole default firewall rules?" "${r}" "${c}" || \ { printf " %b Not installing firewall rulesets.\\n" "${INFO}"; return 0; } printf " %b Installing new IPTables firewall rulesets\\n" "${TICK}" # Check chain first, otherwise a new rule will duplicate old ones @@ -1872,21 +1872,21 @@ installLogrotate() { # At some point in the future this list can be pruned, for now we'll need it to ensure updates don't break. # Refactoring of install script has changed the name of a couple of variables. Sort them out here. accountForRefactor() { - sed -i 's/piholeInterface/PIHOLE_INTERFACE/g' ${setupVars} - sed -i 's/IPv4_address/IPV4_ADDRESS/g' ${setupVars} - sed -i 's/IPv4addr/IPV4_ADDRESS/g' ${setupVars} - sed -i 's/IPv6_address/IPV6_ADDRESS/g' ${setupVars} - sed -i 's/piholeIPv6/IPV6_ADDRESS/g' ${setupVars} - sed -i 's/piholeDNS1/PIHOLE_DNS_1/g' ${setupVars} - sed -i 's/piholeDNS2/PIHOLE_DNS_2/g' ${setupVars} - sed -i 's/^INSTALL_WEB=/INSTALL_WEB_INTERFACE=/' ${setupVars} + sed -i 's/piholeInterface/PIHOLE_INTERFACE/g' "${setupVars}" + sed -i 's/IPv4_address/IPV4_ADDRESS/g' "${setupVars}" + sed -i 's/IPv4addr/IPV4_ADDRESS/g' "${setupVars}" + sed -i 's/IPv6_address/IPV6_ADDRESS/g' "${setupVars}" + sed -i 's/piholeIPv6/IPV6_ADDRESS/g' "${setupVars}" + sed -i 's/piholeDNS1/PIHOLE_DNS_1/g' "${setupVars}" + sed -i 's/piholeDNS2/PIHOLE_DNS_2/g' "${setupVars}" + sed -i 's/^INSTALL_WEB=/INSTALL_WEB_INTERFACE=/' "${setupVars}" # Add 'INSTALL_WEB_SERVER', if its not been applied already: https://github.com/pi-hole/pi-hole/pull/2115 if ! grep -q '^INSTALL_WEB_SERVER=' ${setupVars}; then local webserver_installed=false if grep -q '^INSTALL_WEB_INTERFACE=true' ${setupVars}; then webserver_installed=true fi - echo -e "INSTALL_WEB_SERVER=$webserver_installed" >> ${setupVars} + echo -e "INSTALL_WEB_SERVER=$webserver_installed" >> "${setupVars}" fi } @@ -1968,7 +1968,7 @@ checkSelinux() { # If it's enforcing, if [[ "${enforceMode}" == "Enforcing" ]]; then # Explain Pi-hole does not support it yet - whiptail --defaultno --title "SELinux Enforcing Detected" --yesno "SELinux is being ENFORCED on your system! \\n\\nPi-hole currently does not support SELinux, but you may still continue with the installation.\\n\\nNote: Web Admin will not be fully functional unless you set your policies correctly\\n\\nContinue installing Pi-hole?" ${r} ${c} || \ + whiptail --defaultno --title "SELinux Enforcing Detected" --yesno "SELinux is being ENFORCED on your system! \\n\\nPi-hole currently does not support SELinux, but you may still continue with the installation.\\n\\nNote: Web Admin will not be fully functional unless you set your policies correctly\\n\\nContinue installing Pi-hole?" "${r}" "${c}" || \ { printf "\\n %bSELinux Enforcing detected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; exit 1; } printf " %b Continuing installation with SELinux Enforcing\\n" "${INFO}" printf " %b Please refer to official SELinux documentation to create a custom policy\\n" "${INFO}" @@ -2007,7 +2007,7 @@ If you set a new IP address, you should restart the Pi. The install log is in /etc/pihole. -${additional}" ${r} ${c} +${additional}" "${r}" "${c}" } update_dialogs() { @@ -2028,7 +2028,7 @@ update_dialogs() { opt2b="This will reset your Pi-hole and allow you to enter new settings." # Display the information to the user - UpdateCmd=$(whiptail --title "Existing Install Detected!" --menu "\\n\\nWe have detected an existing install.\\n\\nPlease choose from the following options: \\n($strAdd)" ${r} ${c} 2 \ + UpdateCmd=$(whiptail --title "Existing Install Detected!" --menu "\\n\\nWe have detected an existing install.\\n\\nPlease choose from the following options: \\n($strAdd)" "${r}" "${c}" 2 \ "${opt1a}" "${opt1b}" \ "${opt2a}" "${opt2b}" 3>&2 2>&1 1>&3) || \ { printf " %bCancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; exit 1; } @@ -2556,7 +2556,7 @@ main() { installDefaultBlocklists # Source ${setupVars} to use predefined user variables in the functions - source ${setupVars} + source "${setupVars}" # Get the privacy level if it exists (default is 0) if [[ -f "${PI_HOLE_CONFIG_DIR}/pihole-FTL.conf" ]]; then @@ -2613,7 +2613,7 @@ main() { pw=$(tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c 8) # shellcheck disable=SC1091 . /opt/pihole/webpage.sh - echo "WEBPASSWORD=$(HashPassword ${pw})" >> ${setupVars} + echo "WEBPASSWORD=$(HashPassword "${pw}")" >> "${setupVars}" fi fi From 69081a91757590f6a7d1ef157c786be418d2a7e8 Mon Sep 17 00:00:00 2001 From: David Haguenauer Date: Fri, 10 May 2019 10:37:38 -0400 Subject: [PATCH 85/86] Drop indirection from install_dependent_packages Previously, install_dependent_packages would receive an array variable name as its single parameter, and would use variable indirection to access it; this change simplifies that function so that it instead receives the expanded array. Signed-off-by: David Haguenauer --- automated install/basic-install.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 35d3305c..a5a78102 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1616,7 +1616,6 @@ install_dependent_packages() { # Install packages passed in via argument array # No spinner - conflicts with set -e - declare -a argArray1=("${!1}") declare -a installArray # Debian based package install - debconf will download the entire package list @@ -1626,7 +1625,7 @@ install_dependent_packages() { # installed by us, and remove only the installed packages, and not the entire list. if is_command debconf-apt-progress ; then # For each package, - for i in "${argArray1[@]}"; do + for i in "$@"; do printf " %b Checking for %s..." "${INFO}" "${i}" if dpkg-query -W -f='${Status}' "${i}" 2>/dev/null | grep "ok installed" &> /dev/null; then printf "%b %b Checking for %s\\n" "${OVER}" "${TICK}" "${i}" @@ -1645,7 +1644,7 @@ install_dependent_packages() { fi # Install Fedora/CentOS packages - for i in "${argArray1[@]}"; do + for i in "$@"; do printf " %b Checking for %s..." "${INFO}" "${i}" if "${PKG_MANAGER}" -q list installed "${i}" &> /dev/null; then printf "%b %b Checking for %s" "${OVER}" "${TICK}" "${i}" @@ -2525,7 +2524,7 @@ main() { notify_package_updates_available # Install packages used by this installation script - install_dependent_packages INSTALLER_DEPS[@] + install_dependent_packages "${INSTALLER_DEPS[@]}" # Check if SELinux is Enforcing checkSelinux @@ -2576,7 +2575,7 @@ main() { dep_install_list+=("${PIHOLE_WEB_DEPS[@]}") fi - install_dependent_packages dep_install_list[@] + install_dependent_packages "${dep_install_list[@]}" unset dep_install_list # On some systems, lighttpd is not enabled on first install. We need to enable it here if the user From be3a21ae0770371c0614f1923f02ac504c53df0a Mon Sep 17 00:00:00 2001 From: David Haguenauer Date: Mon, 13 May 2019 09:13:23 -0400 Subject: [PATCH 86/86] Put quotes at ends of strings Signed-off-by: David Haguenauer --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a5a78102..bb000808 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -233,7 +233,7 @@ if is_command apt-get ; then phpVer="php$phpInsMajor.$phpInsMinor" fi # We also need the correct version for `php-sqlite` (which differs across distros) - if "${PKG_MANAGER}" install --dry-run "${phpVer}"-sqlite3 > /dev/null 2>&1; then + if "${PKG_MANAGER}" install --dry-run "${phpVer}-sqlite3" > /dev/null 2>&1; then phpSqlite="sqlite3" else phpSqlite="sqlite" @@ -245,7 +245,7 @@ if is_command apt-get ; then PIHOLE_DEPS=(cron curl dnsutils iputils-ping lsof netcat psmisc sudo unzip wget idn2 sqlite3 libcap2-bin dns-root-data resolvconf libcap2) # The Web dashboard has some that also need to be installed # It's useful to separate the two since our repos are also setup as "Core" code and "Web" code - PIHOLE_WEB_DEPS=(lighttpd "${phpVer}"-common "${phpVer}"-cgi "${phpVer}-${phpSqlite}") + PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-${phpSqlite}") # The Web server user, LIGHTTPD_USER="www-data" # group,