@ -36,21 +36,16 @@ blacklistFile="${piholeDir}/blacklist.txt"
regexFile = " ${ piholeDir } /regex.list " regexFile = " ${ piholeDir } /regex.list "
adListFile = " ${ piholeDir } /adlists.list " adListFile = " ${ piholeDir } /adlists.list "
localList = " ${ piholeDir } /local.list "
VPNList = "/etc/openvpn/ipp.txt"
piholeGitDir = "/etc/.pihole" piholeGitDir = "/etc/.pihole"
GRAVITYDB = $( getFTLConfigValue files.gravity) GRAVITYDB = $( getFTLConfigValue files.gravity)
GRAVITY_TMPDIR = $( getFTLConfigValue files.gravity_tmp)
gravityDBschema = " ${ piholeGitDir } /advanced/Templates/gravity.db.sql " gravityDBschema = " ${ piholeGitDir } /advanced/Templates/gravity.db.sql "
gravityDBcopy = " ${ piholeGitDir } /advanced/Templates/gravity_copy.sql " gravityDBcopy = " ${ piholeGitDir } /advanced/Templates/gravity_copy.sql "
domainsExtension = "domains" domainsExtension = "domains"
curl_connect_timeout = 10 curl_connect_timeout = 10
# Check gravity temp directory
# Set up tmp dir variable in case it's not configured
: " ${ GRAVITY_TMPDIR : =/tmp } "
if [ ! -d " ${ GRAVITY_TMPDIR } " ] || [ ! -w " ${ GRAVITY_TMPDIR } " ] ; then if [ ! -d " ${ GRAVITY_TMPDIR } " ] || [ ! -w " ${ GRAVITY_TMPDIR } " ] ; then
echo -e " ${ COL_LIGHT_RED } Gravity temporary directory does not exist or is not a writeable directory, falling back to /tmp. ${ COL_NC } "
echo -e " ${ COL_LIGHT_RED } Gravity temporary directory does not exist or is not a writeable directory, falling back to /tmp. ${ COL_NC } "
GRAVITY_TMPDIR = "/tmp"
GRAVITY_TMPDIR = "/tmp"
@ -65,7 +60,7 @@ gravityOLDfile="${gravityDIR}/gravity_old.db"
# Generate new SQLite3 file from schema template # Generate new SQLite3 file from schema template
generate_gravity_database( ) {
generate_gravity_database( ) {
if ! pihole-FTL sqlite3 " ${ gravityDBfile } " < " ${ gravityDBschema } " ; then
if ! pihole-FTL sqlite3 -ni " ${ gravityDBfile } " <" ${ gravityDBschema } " ; then
echo -e " ${ CROSS } Unable to create ${ gravityDBfile } "
echo -e " ${ CROSS } Unable to create ${ gravityDBfile } "
return 1
return 1
fi
fi
@ -80,7 +75,7 @@ gravity_build_tree() {
echo -ne " ${ INFO } ${ str } ... "
echo -ne " ${ INFO } ${ str } ... "
# The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once
# The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once
output = $( { pihole-FTL sqlite3 " ${ gravityTEMPfile } " "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);" ; } 2>& )
output = $( { pihole-FTL sqlite3 -ni " ${ gravityTEMPfile } " "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);" ; } 2>& 1)
status = " $? "
status = " $? "
if [ [ " ${ status } " -ne 0 ] ] ; then
if [ [ " ${ status } " -ne 0 ] ] ; then
@ -119,7 +114,7 @@ gravity_swap_databases() {
# Update timestamp when the gravity table was last updated successfully # Update timestamp when the gravity table was last updated successfully
update_gravity_timestamp( ) {
update_gravity_timestamp( ) {
output = $( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 " ${ gravityTEMPfile } " ; } 2>& )
output = $( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 -ni " ${ gravityTEMPfile } " ; } 2>& 1)
status = " $? "
status = " $? "
if [ [ " ${ status } " -ne 0 ] ] ; then
if [ [ " ${ status } " -ne 0 ] ] ; then
@ -164,7 +159,7 @@ database_table_from_file() {
# Get MAX(id) from domainlist when INSERTing into this table
# Get MAX(id) from domainlist when INSERTing into this table
if [ [ " ${ table } " = = "domainlist" ] ] ; then
if [ [ " ${ table } " = = "domainlist" ] ] ; then
rowid = " $( pihole-FTL sqlite3 " ${ gravityDBfile } " "SELECT MAX(id) FROM domainlist;" ) "
rowid = " $( pihole-FTL sqlite3 -ni " ${ gravityDBfile } " "SELECT MAX(id) FROM domainlist;" ) "
if [ [ -z " $rowid " ] ] ; then
if [ [ -z " $rowid " ] ] ; then
rowid = 0
rowid = 0
fi
fi
@ -173,19 +168,18 @@ database_table_from_file() {
# Loop over all domains in ${src} file
# Loop over all domains in ${src} file
# Read file line by line
# Read file line by line
grep -v '^ *#' < " ${ src } " | while IFS = read -r domain
grep -v '^ *#' <" ${ src } " | while IFS = read -r domain; do
do
# Only add non-empty lines
# Only add non-empty lines
if [ [ -n " ${ domain } " ] ] ; then
if [ [ -n " ${ domain } " ] ] ; then
if [ [ " ${ table } " = = "domain_audit" ] ] ; then
if [ [ " ${ table } " = = "domain_audit" ] ] ; then
# domain_audit table format (no enable or modified fields)
# domain_audit table format (no enable or modified fields)
echo " ${ rowid } ,\" ${ domain } \", ${ timestamp } " >> " ${ tmpFile } "
echo " ${ rowid } ,\" ${ domain } \", ${ timestamp } " >>" ${ tmpFile } "
elif [ [ " ${ table } " = = "adlist" ] ] ; then
elif [ [ " ${ table } " = = "adlist" ] ] ; then
# Adlist table format
# Adlist table format
echo " ${ rowid } ,\" ${ domain } \",1, ${ timestamp } , ${ timestamp } ,\"Migrated from ${ src } \",,0,0,0,0,0 " >> " ${ tmpFile } "
echo " ${ rowid } ,\" ${ domain } \",1, ${ timestamp } , ${ timestamp } ,\"Migrated from ${ src } \",,0,0,0,0,0 " >>" ${ tmpFile } "
else
else
# White-, black-, and regexlist table format
# White-, black-, and regexlist table format
echo " ${ rowid } , ${ list_type } ,\" ${ domain } \",1, ${ timestamp } , ${ timestamp } ,\"Migrated from ${ src } \" " >> " ${ tmpFile } "
echo " ${ rowid } , ${ list_type } ,\" ${ domain } \",1, ${ timestamp } , ${ timestamp } ,\"Migrated from ${ src } \" " >>" ${ tmpFile } "
fi
fi
rowid += 1
rowid += 1
fi
fi
@ -194,7 +188,7 @@ database_table_from_file() {
# Store domains in database table specified by ${table}
# Store domains in database table specified by ${table}
# Use printf as .mode and .import need to be on separate lines
# Use printf as .mode and .import need to be on separate lines
# see https://unix.stackexchange.com/a/445615/83260
# see https://unix.stackexchange.com/a/445615/83260
output = $( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" " ${ tmpFile } " " ${ table } " | pihole-FTL sqlite3 " ${ gravityDBfile } " ; } 2>& )
output = $( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" " ${ tmpFile } " " ${ table } " | pihole-FTL sqlite3 -ni " ${ gravityDBfile } " ; } 2>& 1)
status = " $? "
status = " $? "
if [ [ " ${ status } " -ne 0 ] ] ; then
if [ [ " ${ status } " -ne 0 ] ] ; then
@ -204,17 +198,17 @@ database_table_from_file() {
# Move source file to backup directory, create directory if not existing
# Move source file to backup directory, create directory if not existing
mkdir -p " ${ backup_path } "
mkdir -p " ${ backup_path } "
mv " ${ src } " " ${ backup_file } " 2> /dev/null || \
mv " ${ src } " " ${ backup_file } " 2>/dev/null ||
echo -e " ${ CROSS } Unable to backup ${ src } to ${ backup_path } "
echo -e " ${ CROSS } Unable to backup ${ src } to ${ backup_path } "
# Delete tmpFile
# Delete tmpFile
rm " ${ tmpFile } " > /dev/null 2>& 1 || \
rm " ${ tmpFile } " >/dev/null 2>& 1 ||
echo -e " ${ CROSS } Unable to remove ${ tmpFile } "
echo -e " ${ CROSS } Unable to remove ${ tmpFile } "
} }
# Check if a column with name ${2} exists in gravity table with name ${1} # Check if a column with name ${2} exists in gravity table with name ${1}
gravity_column_exists( ) {
gravity_column_exists( ) {
output = $( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" " ${ 1 } " " ${ 2 } " | pihole-FTL sqlite3 " ${ gravityTEMPfile } " ; } 2>& )
output = $( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" " ${ 1 } " " ${ 2 } " | pihole-FTL sqlite3 -ni " ${ gravityTEMPfile } " ; } 2>& 1)
if [ [ " ${ output } " = = "1" ] ] ; then
if [ [ " ${ output } " = = "1" ] ] ; then
return 0 # Bash 0 is success
return 0 # Bash 0 is success
fi
fi
@ -226,10 +220,10 @@ gravity_column_exists() {
database_adlist_number( ) {
database_adlist_number( ) {
# Only try to set number of domains when this field exists in the gravity database
# Only try to set number of domains when this field exists in the gravity database
if ! gravity_column_exists "adlist" "number" ; then
if ! gravity_column_exists "adlist" "number" ; then
return ;
return
fi
fi
output = $( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" " ${ 2 } " " ${ 3 } " " ${ 1 } " | pihole-FTL sqlite3 " ${ gravityTEMPfile } " ; } 2>& )
output = $( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" " ${ 2 } " " ${ 3 } " " ${ 1 } " | pihole-FTL sqlite3 -ni " ${ gravityTEMPfile } " ; } 2>& 1)
status = " $? "
status = " $? "
if [ [ " ${ status } " -ne 0 ] ] ; then
if [ [ " ${ status } " -ne 0 ] ] ; then
@ -242,10 +236,10 @@ database_adlist_number() {
database_adlist_status( ) {
database_adlist_status( ) {
# Only try to set the status when this field exists in the gravity database
# Only try to set the status when this field exists in the gravity database
if ! gravity_column_exists "adlist" "status" ; then
if ! gravity_column_exists "adlist" "status" ; then
return ;
return
fi
fi
output = $( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" " ${ 2 } " " ${ 1 } " | pihole-FTL sqlite3 " ${ gravityTEMPfile } " ; } 2>& )
output = $( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" " ${ 2 } " " ${ 1 } " | pihole-FTL sqlite3 -ni " ${ gravityTEMPfile } " ; } 2>& 1)
status = " $? "
status = " $? "
if [ [ " ${ status } " -ne 0 ] ] ; then
if [ [ " ${ status } " -ne 0 ] ] ; then
@ -299,15 +293,10 @@ migrate_to_database() {
# Determine if DNS resolution is available before proceeding # Determine if DNS resolution is available before proceeding
gravity_CheckDNSResolutionAvailable( ) {
gravity_CheckDNSResolutionAvailable( ) {
local lookupDomain = "pi.hole"
local lookupDomain = "raw.githubusercontent.com"
# Determine if $localList does not exist, and ensure it is not empty
if [ [ ! -e " ${ localList } " ] ] || [ [ -s " ${ localList } " ] ] ; then
lookupDomain = "raw.githubusercontent.com"
fi
# Determine if $lookupDomain is resolvable
# Determine if $lookupDomain is resolvable
if timeout 4 getent hosts " ${ lookupDomain } " & > /dev/null; then
if timeout 4 getent hosts " ${ lookupDomain } " & >/dev/null; then
# Print confirmation of resolvability if it had previously failed
# Print confirmation of resolvability if it had previously failed
if [ [ -n " ${ secs :- } " ] ] ; then
if [ [ -n " ${ secs :- } " ] ] ; then
echo -e " ${ OVER } ${ TICK } DNS resolution is now available\\n "
echo -e " ${ OVER } ${ TICK } DNS resolution is now available\\n "
@ -321,7 +310,7 @@ gravity_CheckDNSResolutionAvailable() {
# If the /etc/resolv.conf contains resolvers other than 127.0.0.1 then the local dnsmasq will not be queried and pi.hole is NXDOMAIN.
# If the /etc/resolv.conf contains resolvers other than 127.0.0.1 then the local dnsmasq will not be queried and pi.hole is NXDOMAIN.
# This means that even though name resolution is working, the getent hosts check fails and the holddown timer keeps ticking and eventually fails
# This means that even though name resolution is working, the getent hosts check fails and the holddown timer keeps ticking and eventually fails
# So we check the output of the last command and if it failed, attempt to use dig +short as a fallback
# So we check the output of the last command and if it failed, attempt to use dig +short as a fallback
if timeout 4 dig +short " ${ lookupDomain } " & > /dev/null; then
if timeout 4 dig +short " ${ lookupDomain } " & >/dev/null; then
if [ [ -n " ${ secs :- } " ] ] ; then
if [ [ -n " ${ secs :- } " ] ] ; then
echo -e " ${ OVER } ${ TICK } DNS resolution is now available\\n "
echo -e " ${ OVER } ${ TICK } DNS resolution is now available\\n "
fi
fi
@ -332,7 +321,7 @@ gravity_CheckDNSResolutionAvailable() {
fi
fi
# Determine error output message
# Determine error output message
if pgrep pihole-FTL & > /dev/null; then
if pgrep pihole-FTL & >/dev/null; then
echo -e " ${ CROSS } DNS resolution is currently unavailable "
echo -e " ${ CROSS } DNS resolution is currently unavailable "
else
else
echo -e " ${ CROSS } DNS service is not running "
echo -e " ${ CROSS } DNS service is not running "
@ -342,7 +331,7 @@ gravity_CheckDNSResolutionAvailable() {
# Ensure DNS server is given time to be resolvable
# Ensure DNS server is given time to be resolvable
secs = "120"
secs = "120"
echo -ne " ${ INFO } Time until retry: ${ secs } "
echo -ne " ${ INFO } Time until retry: ${ secs } "
until timeout 1 getent hosts " ${ lookupDomain } " & > /dev/null; do
until timeout 1 getent hosts " ${ lookupDomain } " & >/dev/null; do
[ [ " ${ secs :- } " -eq 0 ] ] && break
[ [ " ${ secs :- } " -eq 0 ] ] && break
echo -ne " ${ OVER } ${ INFO } Time until retry: ${ secs } "
echo -ne " ${ OVER } ${ INFO } Time until retry: ${ secs } "
: $(( secs--))
: $(( secs--))
@ -363,19 +352,19 @@ gravity_DownloadBlocklists() {
# Retrieve source URLs from gravity database
# Retrieve source URLs from gravity database
# We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true)
# We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true)
mapfile -t sources <<< " $( pihole-FTL sqlite3 " ${ gravityDBfile } " "SELECT address FROM vw_adlist;" 2> /dev/null) "
mapfile -t sources <<< " $( pihole-FTL sqlite3 -ni " ${ gravityDBfile } " "SELECT address FROM vw_adlist;" 2>/dev/null) "
mapfile -t sourceIDs <<< " $( pihole-FTL sqlite3 " ${ gravityDBfile } " "SELECT id FROM vw_adlist;" 2> /dev/null) "
mapfile -t sourceIDs <<< " $( pihole-FTL sqlite3 -ni " ${ gravityDBfile } " "SELECT id FROM vw_adlist;" 2>/dev/null) "
mapfile -t sourceTypes <<< " $( pihole-FTL sqlite3 " ${ gravityDBfile } " "SELECT type FROM vw_adlist;" 2> /dev/null) "
mapfile -t sourceTypes <<< " $( pihole-FTL sqlite3 -ni " ${ gravityDBfile } " "SELECT type FROM vw_adlist;" 2>/dev/null) "
# Parse source domains from $sources
# Parse source domains from $sources
mapfile -t sourceDomains <<< " $(
mapfile -t sourceDomains <<< " $(
# Logic: Split by folder/port
# Logic: Split by folder/port
awk -F '[/:]' ' {
awk -F '[/:]' ' {
# Remove URL protocol & optional username:password@
# Remove URL protocol & optional username:password@
gsub( /( .*:\/ \/ | .*:.*@) /, "" , $0 )
gsub( /( .*:\/ \/ | .*:.*@) /, "" , $0 )
if ( length( $1 ) >0) { print $1 }
if ( length( $1 ) >0) { print $1 }
else { print "local" }
else { print "local" }
} ' <<< "$(printf '%s\n ' " ${ sources [@] } " ) " 2> /dev/null
} ' <<< "$(printf '%s\n ' " ${ sources [@] } " ) " 2> /dev/null
) "
) "
local str = "Pulling blocklist source list into range"
local str = "Pulling blocklist source list into range"
@ -393,8 +382,8 @@ gravity_DownloadBlocklists() {
# Prepare new gravity database
# Prepare new gravity database
str = "Preparing new gravity database"
str = "Preparing new gravity database"
echo -ne " ${ INFO } ${ str } ... "
echo -ne " ${ INFO } ${ str } ... "
rm " ${ gravityTEMPfile } " > /dev/null 2>& 1
rm " ${ gravityTEMPfile } " >/dev/null 2>& 1
output = $( { pihole-FTL sqlite3 " ${ gravityTEMPfile } " < " ${ gravityDBschema } " ; } 2>& )
output = $( { pihole-FTL sqlite3 -ni " ${ gravityTEMPfile } " <" ${ gravityDBschema } " ; } 2>& 1)
status = " $? "
status = " $? "
if [ [ " ${ status } " -ne 0 ] ] ; then
if [ [ " ${ status } " -ne 0 ] ] ; then
@ -414,7 +403,7 @@ gravity_DownloadBlocklists() {
copyGravity = " ${ copyGravity // " ${ gravityDBfile_default } " / " ${ gravityDBfile } " } "
copyGravity = " ${ copyGravity // " ${ gravityDBfile_default } " / " ${ gravityDBfile } " } "
fi
fi
output = $( { pihole-FTL sqlite3 " ${ gravityTEMPfile } " <<< " ${ copyGravity } " ; } 2>& )
output = $( { pihole-FTL sqlite3 -ni " ${ gravityTEMPfile } " <<< " ${ copyGravity } " ; } 2>& 1)
status = " $? "
status = " $? "
if [ [ " ${ status } " -ne 0 ] ] ; then
if [ [ " ${ status } " -ne 0 ] ] ; then
@ -459,7 +448,7 @@ gravity_DownloadBlocklists() {
# this will remove first @ that is after schema and before domain
# this will remove first @ that is after schema and before domain
# \1 is optional schema, \2 is userinfo
# \1 is optional schema, \2 is userinfo
check_url = " $( sed -re 's#([^:/]*://)?([^/]+)@#\1\2#' <<< " $url " ) "
check_url = " $( sed -re 's#([^:/]*://)?([^/]+)@#\1\2#' <<< " $url " ) "
if [ [ " ${ check_url } " = ~ ${ regex } ] ] ; then
if [ [ " ${ check_url } " = ~ ${ regex } ] ] ; then
echo -e " ${ CROSS } Invalid Target "
echo -e " ${ CROSS } Invalid Target "
@ -479,7 +468,7 @@ compareLists() {
if [ [ -s " ${ target } .sha1 " ] ] ; then
if [ [ -s " ${ target } .sha1 " ] ] ; then
if ! sha1sum --check --status --strict " ${ target } .sha1 " ; then
if ! sha1sum --check --status --strict " ${ target } .sha1 " ; then
# The list changed upstream, we need to update the checksum
# The list changed upstream, we need to update the checksum
sha1sum " ${ target } " > " ${ target } .sha1 "
sha1sum " ${ target } " >" ${ target } .sha1 "
echo " ${ INFO } List has been updated "
echo " ${ INFO } List has been updated "
database_adlist_status " ${ adlistID } " "1"
database_adlist_status " ${ adlistID } " "1"
else
else
@ -488,7 +477,7 @@ compareLists() {
fi
fi
else
else
# No checksum available, create one for comparing on the next run
# No checksum available, create one for comparing on the next run
sha1sum " ${ target } " > " ${ target } .sha1 "
sha1sum " ${ target } " >" ${ target } .sha1 "
# We assume here it was changed upstream
# We assume here it was changed upstream
database_adlist_status " ${ adlistID } " "1"
database_adlist_status " ${ adlistID } " "1"
fi
fi
@ -517,25 +506,29 @@ gravity_DownloadBlocklistFromUrl() {
echo -ne " ${ INFO } ${ str } Pending... "
echo -ne " ${ INFO } ${ str } Pending... "
blocked = false
blocked = false
case $( getFTLConfigValue dns.blocking.mode) in
case $( getFTLConfigValue dns.blocking.mode) in
"IP-NODATA-AAAA" | "IP" )
"IP-NODATA-AAAA" | "IP" )
# Get IP address of this domain
# Get IP address of this domain
ip = " $( dig " ${ domain } " +short) "
ip = " $( dig " ${ domain } " +short) "
# Check if this IP matches any IP of the system
# Check if this IP matches any IP of the system
if [ [ -n " ${ ip } " && $( grep -Ec " inet(|6) ${ ip } " <<< " $( ip a) " ) -gt 0 ] ] ; then
if [ [ -n " ${ ip } " && $( grep -Ec " inet(|6) ${ ip } " <<< " $( ip a) " ) -gt 0 ] ] ; then
blocked = true
blocked = true
fi ; ;
fi
"NXDOMAIN" )
; ;
if [ [ $( dig " ${ domain } " | grep "NXDOMAIN" -c) -ge 1 ] ] ; then
"NXDOMAIN" )
blocked = true
if [ [ $( dig " ${ domain } " | grep "NXDOMAIN" -c) -ge 1 ] ] ; then
fi ; ;
blocked = true
"NODATA" )
fi
if [ [ $( dig " ${ domain } " | grep "NOERROR" -c) -ge 1 ] ] && [ [ -z $( dig +short " ${ domain } " ) ] ] ; then
; ;
blocked = true
"NODATA" )
fi ; ;
if [ [ $( dig " ${ domain } " | grep "NOERROR" -c) -ge 1 ] ] && [ [ -z $( dig +short " ${ domain } " ) ] ] ; then
"NULL" | *)
blocked = true
if [ [ $( dig " ${ domain } " +short | grep "0.0.0.0" -c) -ge 1 ] ] ; then
fi
blocked = true
; ;
fi ; ;
"NULL" | *)
if [ [ $( dig " ${ domain } " +short | grep "0.0.0.0" -c) -ge 1 ] ] ; then
blocked = true
fi
; ;
esac
esac
if [ [ " ${ blocked } " = = true ] ] ; then
if [ [ " ${ blocked } " = = true ] ] ; then
@ -547,43 +540,53 @@ gravity_DownloadBlocklistFromUrl() {
fi
fi
ip = $( dig " @ ${ ip_addr } " -p " ${ port } " +short " ${ domain } " | tail -1)
ip = $( dig " @ ${ ip_addr } " -p " ${ port } " +short " ${ domain } " | tail -1)
if [ [ $( echo " ${ url } " | awk -F '://' '{print $1}' ) = "https" ] ] ; then
if [ [ $( echo " ${ url } " | awk -F '://' '{print $1}' ) = "https" ] ] ; then
port = 443;
port = 443
else port = 80
else
port = 80
fi
fi
bad_list = $( pihole -q -adlist " ${ domain } " | head -n1 | awk -F 'Match found in ' '{print $2}' )
bad_list = $( pihole -q -adlist " ${ domain } " | head -n1 | awk -F 'Match found in ' '{print $2}' )
echo -e " ${ OVER } ${ CROSS } ${ str } ${ domain } is blocked by ${ bad_list % : } . Using DNS on ${ PIHOLE_DNS_1 } to download ${ url } " ;
echo -e " ${ OVER } ${ CROSS } ${ str } ${ domain } is blocked by ${ bad_list % : } . Using DNS on ${ PIHOLE_DNS_1 } to download ${ url } "
echo -ne " ${ INFO } ${ str } Pending... "
echo -ne " ${ INFO } ${ str } Pending... "
cmd_ext = " --resolve $domain : $port : $ip "
cmd_ext = " --resolve $domain : $port : $ip "
fi
fi
# shellcheck disable=SC2086
# shellcheck disable=SC2086
httpCode = $( curl --connect-timeout ${ curl_connect_timeout } -s -L ${ compression } ${ cmd_ext } ${ heisenbergCompensator } -w "%{http_code}" " ${ url } " -o " ${ listCurlBuffer } " 2> /dev/null)
httpCode = $( curl --connect-timeout ${ curl_connect_timeout } -s -L ${ compression } ${ cmd_ext } ${ heisenbergCompensator } -w "%{http_code}" " ${ url } " -o " ${ listCurlBuffer } " 2>/dev/null)
case $url in
case $url in
# Did we "download" a local file?
# Did we "download" a local file?
"file" *)
"file" *)
if [ [ -s " ${ listCurlBuffer } " ] ] ; then
if [ [ -s " ${ listCurlBuffer } " ] ] ; then
echo -e " ${ OVER } ${ TICK } ${ str } Retrieval successful " ; success = true
echo -e " ${ OVER } ${ TICK } ${ str } Retrieval successful "
else
success = true
echo -e " ${ OVER } ${ CROSS } ${ str } Not found / empty list "
else
fi ; ;
echo -e " ${ OVER } ${ CROSS } ${ str } Not found / empty list "
# Did we "download" a remote file?
fi
*)
; ;
# Determine "Status:" output based on HTTP response
# Did we "download" a remote file?
case " ${ httpCode } " in
*)
"200" ) echo -e " ${ OVER } ${ TICK } ${ str } Retrieval successful " ; success = true; ;
# Determine "Status:" output based on HTTP response
"304" ) echo -e " ${ OVER } ${ TICK } ${ str } No changes detected " ; success = true; ;
case " ${ httpCode } " in
"000" ) echo -e " ${ OVER } ${ CROSS } ${ str } Connection Refused " ; ;
"200" )
"403" ) echo -e " ${ OVER } ${ CROSS } ${ str } Forbidden " ; ;
echo -e " ${ OVER } ${ TICK } ${ str } Retrieval successful "
"404" ) echo -e " ${ OVER } ${ CROSS } ${ str } Not found " ; ;
success = true
"408" ) echo -e " ${ OVER } ${ CROSS } ${ str } Time-out " ; ;
; ;
"451" ) echo -e " ${ OVER } ${ CROSS } ${ str } Unavailable For Legal Reasons " ; ;
"304" )
"500" ) echo -e " ${ OVER } ${ CROSS } ${ str } Internal Server Error " ; ;
echo -e " ${ OVER } ${ TICK } ${ str } No changes detected "
"504" ) echo -e " ${ OVER } ${ CROSS } ${ str } Connection Timed Out (Gateway) " ; ;
success = true
"521" ) echo -e " ${ OVER } ${ CROSS } ${ str } Web Server Is Down (Cloudflare) " ; ;
; ;
"522" ) echo -e " ${ OVER } ${ CROSS } ${ str } Connection Timed Out (Cloudflare) " ; ;
"000" ) echo -e " ${ OVER } ${ CROSS } ${ str } Connection Refused " ; ;
* ) echo -e " ${ OVER } ${ CROSS } ${ str } ${ url } ( ${ httpCode } ) " ; ;
"403" ) echo -e " ${ OVER } ${ CROSS } ${ str } Forbidden " ; ;
esac ; ;
"404" ) echo -e " ${ OVER } ${ CROSS } ${ str } Not found " ; ;
"408" ) echo -e " ${ OVER } ${ CROSS } ${ str } Time-out " ; ;
"451" ) echo -e " ${ OVER } ${ CROSS } ${ str } Unavailable For Legal Reasons " ; ;
"500" ) echo -e " ${ OVER } ${ CROSS } ${ str } Internal Server Error " ; ;
"504" ) echo -e " ${ OVER } ${ CROSS } ${ str } Connection Timed Out (Gateway) " ; ;
"521" ) echo -e " ${ OVER } ${ CROSS } ${ str } Web Server Is Down (Cloudflare) " ; ;
"522" ) echo -e " ${ OVER } ${ CROSS } ${ str } Connection Timed Out (Cloudflare) " ; ;
*) echo -e " ${ OVER } ${ CROSS } ${ str } ${ url } ( ${ httpCode } ) " ; ;
esac
; ;
esac
esac
local done = "false"
local done = "false"
@ -637,7 +640,7 @@ gravity_ParseFileIntoDomains() {
# This helps with that and makes it easier to read
# This helps with that and makes it easier to read
# It also helps with debugging so each stage of the script can be researched more in depth
# It also helps with debugging so each stage of the script can be researched more in depth
# 1) Convert all characters to lowercase
# 1) Convert all characters to lowercase
tr '[:upper:]' '[:lower:]' < " ${ src } " > " ${ destination } "
tr '[:upper:]' '[:lower:]' <" ${ src } " >" ${ destination } "
# 2) Remove carriage returns
# 2) Remove carriage returns
# 3) Remove lines starting with ! (ABP Comments)
# 3) Remove lines starting with ! (ABP Comments)
@ -647,7 +650,7 @@ gravity_ParseFileIntoDomains() {
# 7) Remove leading tabs, spaces, etc. (Also removes leading IP addresses)
# 7) Remove leading tabs, spaces, etc. (Also removes leading IP addresses)
# 8) Remove empty lines
# 8) Remove empty lines
sed -i -r \ sed -i -r \
-e 's/\r$//' \
-e 's/\r$//' \
-e 's/\s*!.*//g' \
-e 's/\s*!.*//g' \
-e 's/\s*\[.*//g' \
-e 's/\s*\[.*//g' \
@ -664,12 +667,12 @@ gravity_Table_Count() {
local table = " ${ 1 } "
local table = " ${ 1 } "
local str = " ${ 2 } "
local str = " ${ 2 } "
local num
local num
num = " $( pihole-FTL sqlite3 " ${ gravityTEMPfile } " " SELECT COUNT(*) FROM ${ table } ; " ) "
num = " $( pihole-FTL sqlite3 -ni " ${ gravityTEMPfile } " " SELECT COUNT(*) FROM ${ table } ; " ) "
if [ [ " ${ table } " = = "gravity" ] ] ; then
if [ [ " ${ table } " = = "gravity" ] ] ; then
local unique
local unique
unique = " $( pihole-FTL sqlite3 " ${ gravityTEMPfile } " " SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${ table } ); " ) "
unique = " $( pihole-FTL sqlite3 -ni " ${ gravityTEMPfile } " " SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${ table } ); " ) "
echo -e " ${ INFO } Number of ${ str } : ${ num } ( ${ COL_BOLD } ${ unique } unique domains ${ COL_NC } ) "
echo -e " ${ INFO } Number of ${ str } : ${ num } ( ${ COL_BOLD } ${ unique } unique domains ${ COL_NC } ) "
pihole-FTL sqlite3 " ${ gravityTEMPfile } " " INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count', ${ unique } ); "
pihole-FTL sqlite3 -ni " ${ gravityTEMPfile } " " INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count', ${ unique } ); "
else
else
echo -e " ${ INFO } Number of ${ str } : ${ num } "
echo -e " ${ INFO } Number of ${ str } : ${ num } "
fi
fi
@ -686,18 +689,6 @@ gravity_ShowCount() {
gravity_Table_Count "vw_regex_whitelist" "regex allowed filters"
gravity_Table_Count "vw_regex_whitelist" "regex allowed filters"
} }
# Create "localhost" entries into hosts format
gravity_generateLocalList( ) {
# Empty $localList if it already exists, otherwise, create it
echo "### Do not modify this file, it will be overwritten by pihole -g" > " ${ localList } "
chmod 644 " ${ localList } "
# Add additional LAN hosts provided by OpenVPN (if available)
if [ [ -f " ${ VPNList } " ] ] ; then
awk -F, '{printf $2"\t"$1".vpn\n"}' " ${ VPNList } " >> " ${ localList } "
fi
}
# Trap Ctrl-C # Trap Ctrl-C
gravity_Trap( ) {
gravity_Trap( ) {
trap '{ echo -e "\\n\\n ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT
trap '{ echo -e "\\n\\n ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT
@ -711,12 +702,12 @@ gravity_Cleanup() {
echo -ne " ${ INFO } ${ str } ... "
echo -ne " ${ INFO } ${ str } ... "
# Delete tmp content generated by Gravity
# Delete tmp content generated by Gravity
rm ${ piholeDir } /pihole.*.txt 2> /dev/null
rm ${ piholeDir } /pihole.*.txt 2>/dev/null
rm ${ piholeDir } /*.tmp 2> /dev/null
rm ${ piholeDir } /*.tmp 2>/dev/null
# listCurlBuffer location
# listCurlBuffer location
rm " ${ GRAVITY_TMPDIR } " /*.phgpb 2> /dev/null
rm " ${ GRAVITY_TMPDIR } " /*.phgpb 2>/dev/null
# invalid_domains location
# invalid_domains location
rm " ${ GRAVITY_TMPDIR } " /*.ph-non-domains 2> /dev/null
rm " ${ GRAVITY_TMPDIR } " /*.ph-non-domains 2>/dev/null
# Ensure this function only runs when gravity_SetDownloadOptions() has completed
# Ensure this function only runs when gravity_SetDownloadOptions() has completed
if [ [ " ${ gravity_Blackbody :- } " = = true ] ] ; then
if [ [ " ${ gravity_Blackbody :- } " = = true ] ] ; then
@ -724,7 +715,7 @@ gravity_Cleanup() {
for file in " ${ piholeDir } " /*." ${ domainsExtension } " ; do
for file in " ${ piholeDir } " /*." ${ domainsExtension } " ; do
# If list is not in active array, then remove it
# If list is not in active array, then remove it
if [ [ ! " ${ activeDomains [*] } " = = *" ${ file } " * ] ] ; then
if [ [ ! " ${ activeDomains [*] } " = = *" ${ file } " * ] ] ; then
rm -f " ${ file } " 2> /dev/null || \
rm -f " ${ file } " 2>/dev/null ||
echo -e " ${ CROSS } Failed to remove ${ file ##*/ } "
echo -e " ${ CROSS } Failed to remove ${ file ##*/ } "
fi
fi
done
done
@ -750,7 +741,7 @@ database_recovery() {
local str = "Checking integrity of existing gravity database (this can take a while)"
local str = "Checking integrity of existing gravity database (this can take a while)"
local option = " ${ 1 } "
local option = " ${ 1 } "
echo -ne " ${ INFO } ${ str } ... "
echo -ne " ${ INFO } ${ str } ... "
result = " $( pihole-FTL sqlite3 " ${ gravityDBfile } " "PRAGMA integrity_check" 2>& 1) "
result = " $( pihole-FTL sqlite3 -ni " ${ gravityDBfile } " "PRAGMA integrity_check" 2>& 1) "
if [ [ ${ result } = "ok" ] ] ; then
if [ [ ${ result } = "ok" ] ] ; then
echo -e " ${ OVER } ${ TICK } ${ str } - no errors found "
echo -e " ${ OVER } ${ TICK } ${ str } - no errors found "
@ -758,7 +749,7 @@ database_recovery() {
str = "Checking foreign keys of existing gravity database (this can take a while)"
str = "Checking foreign keys of existing gravity database (this can take a while)"
echo -ne " ${ INFO } ${ str } ... "
echo -ne " ${ INFO } ${ str } ... "
unset result
unset result
result = " $( pihole-FTL sqlite3 " ${ gravityDBfile } " "PRAGMA foreign_key_check" 2>& 1) "
result = " $( pihole-FTL sqlite3 -ni " ${ gravityDBfile } " "PRAGMA foreign_key_check" 2>& 1) "
if [ [ -z ${ result } ] ] ; then
if [ [ -z ${ result } ] ] ; then
echo -e " ${ OVER } ${ TICK } ${ str } - no errors found "
echo -e " ${ OVER } ${ TICK } ${ str } - no errors found "
if [ [ " ${ option } " != "force" ] ] ; then
if [ [ " ${ option } " != "force" ] ] ; then
@ -766,18 +757,18 @@ database_recovery() {
fi
fi
else
else
echo -e " ${ OVER } ${ CROSS } ${ str } - errors found: "
echo -e " ${ OVER } ${ CROSS } ${ str } - errors found: "
while IFS = read -r line ; do echo " - $line " ; done <<< " $result "
while IFS = read -r line; do echo " - $line " ; done <<< " $result "
fi
fi
else
else
echo -e " ${ OVER } ${ CROSS } ${ str } - errors found: "
echo -e " ${ OVER } ${ CROSS } ${ str } - errors found: "
while IFS = read -r line ; do echo " - $line " ; done <<< " $result "
while IFS = read -r line; do echo " - $line " ; done <<< " $result "
fi
fi
str = "Trying to recover existing gravity database"
str = "Trying to recover existing gravity database"
echo -ne " ${ INFO } ${ str } ... "
echo -ne " ${ INFO } ${ str } ... "
# We have to remove any possibly existing recovery database or this will fail
# We have to remove any possibly existing recovery database or this will fail
rm -f " ${ gravityDBfile } .recovered " > /dev/null 2>& 1
rm -f " ${ gravityDBfile } .recovered " >/dev/null 2>& 1
if result = " $( pihole-FTL sqlite3 " ${ gravityDBfile } " ".recover" | pihole-FTL sqlite3 " ${ gravityDBfile } .recovered " 2>& 1) " ; then
if result = " $( pihole-FTL sqlite3 -ni " ${ gravityDBfile } " ".recover" | pihole-FTL sqlite3 -ni " ${ gravityDBfile } .recovered " 2>& 1) " ; then
echo -e " ${ OVER } ${ TICK } ${ str } - success "
echo -e " ${ OVER } ${ TICK } ${ str } - success "
mv " ${ gravityDBfile } " " ${ gravityDBfile } .old "
mv " ${ gravityDBfile } " " ${ gravityDBfile } .old "
mv " ${ gravityDBfile } .recovered " " ${ gravityDBfile } "
mv " ${ gravityDBfile } .recovered " " ${ gravityDBfile } "
@ -785,7 +776,7 @@ database_recovery() {
echo -ne " ${ INFO } The old ${ gravityDBfile } has been moved to ${ gravityDBfile } .old "
echo -ne " ${ INFO } The old ${ gravityDBfile } has been moved to ${ gravityDBfile } .old "
else
else
echo -e " ${ OVER } ${ CROSS } ${ str } - the following errors happened: "
echo -e " ${ OVER } ${ CROSS } ${ str } - the following errors happened: "
while IFS = read -r line ; do echo " - $line " ; done <<< " $result "
while IFS = read -r line; do echo " - $line " ; done <<< " $result "
echo -e " ${ CROSS } Recovery failed. Try \"pihole -r recreate\" instead. "
echo -e " ${ CROSS } Recovery failed. Try \"pihole -r recreate\" instead. "
exit 1
exit 1
fi
fi
@ -804,9 +795,10 @@ Options:
repairSelector( ) {
repairSelector( ) {
case " $1 " in
case " $1 " in
"recover" ) recover_database = true; ;
"recover" ) recover_database = true ; ;
"recreate" ) recreate_database = true; ;
"recreate" ) recreate_database = true ; ;
*) echo " Usage: pihole -g -r {recover,recreate}
*)
echo " Usage: pihole -g -r {recover,recreate}
Attempt to repair gravity database
Attempt to repair gravity database
Available options:
Available options:
@ -825,15 +817,17 @@ Available options:
and create a new file from scratch. If you still
and create a new file from scratch. If you still
have the migration backup created when migrating
have the migration backup created when migrating
to Pi-hole v5.0, Pi-hole will import these files."
to Pi-hole v5.0, Pi-hole will import these files."
exit 0; ;
exit 0
; ;
esac
esac
} }
for var in " $@ " ; do for var in " $@ " ; do
case " ${ var } " in
case " ${ var } " in
"-f" | "--force" ) forceDelete = true; ;
"-f" | "--force" ) forceDelete = true; ;
"-r" | "--repair" ) repairSelector " $3 " ; ;
"-r" | "--repair" ) repairSelector " $3 " ; ;
"-h" | "--help" ) helpFunc; ;
"-u" | "--upgrade" ) upgrade_gravityDB " ${ gravityDBfile } " " ${ piholeDir } " ; exit 0; ;
"-h" | "--help" ) helpFunc; ;
esac
esac
done done
@ -849,9 +843,9 @@ if [[ "${recreate_database:-}" == true ]]; then
str = "Recreating gravity database from migration backup"
str = "Recreating gravity database from migration backup"
echo -ne " ${ INFO } ${ str } ... "
echo -ne " ${ INFO } ${ str } ... "
rm " ${ gravityDBfile } "
rm " ${ gravityDBfile } "
pushd " ${ piholeDir } " > /dev/null || exit
pushd " ${ piholeDir } " >/dev/null || exit
cp migration_backup/* .
cp migration_backup/* .
popd > /dev/null || exit
popd >/dev/null || exit
echo -e " ${ OVER } ${ TICK } ${ str } "
echo -e " ${ OVER } ${ TICK } ${ str } "
fi fi
@ -869,7 +863,7 @@ if [[ "${forceDelete:-}" == true ]]; then
str = "Deleting existing list cache"
str = "Deleting existing list cache"
echo -ne " ${ INFO } ${ str } ... "
echo -ne " ${ INFO } ${ str } ... "
rm /etc/pihole/list.* 2> /dev/null || true
rm /etc/pihole/list.* 2>/dev/null || true
echo -e " ${ OVER } ${ TICK } ${ str } "
echo -e " ${ OVER } ${ TICK } ${ str } "
fi fi
@ -884,9 +878,6 @@ if ! gravity_DownloadBlocklists; then
exit 1
exit 1
fi fi
# Create local.list
gravity_generateLocalList
# Update gravity timestamp # Update gravity timestamp
update_gravity_timestamp
update_gravity_timestamp
DL6ER
3 months ago