@ -217,6 +217,48 @@ database_adlist_updated() {
fi
fi
}
}
# Check if a column with name ${2} exists in gravity table with name ${1}
gravity_column_exists( ) {
output = $( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" " ${ 1 } " " ${ 2 } " | sqlite3 " ${ gravityDBfile } " ; } 2>& 1 )
if [ [ " ${ output } " = = "1" ] ] ; then
return 0 # Bash 0 is success
fi
return 1 # Bash non-0 is failure
}
# Update number of domain on this list. We store this in the "old" database as all values in the new database will later be overwritten
database_adlist_number( ) {
# Only try to set number of domains when this field exists in the gravity database
if ! gravity_column_exists "adlist" "number" ; then
return ;
fi
output = $( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" " ${ num_lines } " " ${ num_invalid } " " ${ 1 } " | sqlite3 " ${ gravityDBfile } " ; } 2>& 1 )
status = " $? "
if [ [ " ${ status } " -ne 0 ] ] ; then
echo -e " \\n ${ CROSS } Unable to update number of domains in adlist with ID ${ 1 } in database ${ gravityDBfile } \\n ${ output } "
gravity_Cleanup "error"
fi
}
# Update status of this list. We store this in the "old" database as all values in the new database will later be overwritten
database_adlist_status( ) {
# Only try to set the status when this field exists in the gravity database
if ! gravity_column_exists "adlist" "status" ; then
return ;
fi
output = $( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" " ${ 2 } " " ${ 1 } " | sqlite3 " ${ gravityDBfile } " ; } 2>& 1 )
status = " $? "
if [ [ " ${ status } " -ne 0 ] ] ; then
echo -e " \\n ${ CROSS } Unable to update status of adlist with ID ${ 1 } in database ${ gravityDBfile } \\n ${ output } "
gravity_Cleanup "error"
fi
}
# Migrate pre-v5.0 list files to database-based Pi-hole versions
# Migrate pre-v5.0 list files to database-based Pi-hole versions
migrate_to_database( ) {
migrate_to_database( ) {
# Create database file only if not present
# Create database file only if not present
@ -444,6 +486,8 @@ gravity_DownloadBlocklists() {
}
}
total_num = 0
total_num = 0
num_lines = 0
num_invalid = 0
parseList( ) {
parseList( ) {
local adlistID = " ${ 1 } " src = " ${ 2 } " target = " ${ 3 } " incorrect_lines
local adlistID = " ${ 1 } " src = " ${ 2 } " target = " ${ 3 } " incorrect_lines
# This sed does the following things:
# This sed does the following things:
@ -454,7 +498,7 @@ parseList() {
# Find (up to) five domains containing invalid characters (see above)
# Find (up to) five domains containing invalid characters (see above)
incorrect_lines = " $( sed -e "/[^a-zA-Z0-9.\_-]/!d" " ${ src } " | head -n 5) "
incorrect_lines = " $( sed -e "/[^a-zA-Z0-9.\_-]/!d" " ${ src } " | head -n 5) "
local num_lines num_ target_lines num_correct_lines num_invalid
local num_target_lines num_correct_lines num_invalid
# Get number of lines in source file
# Get number of lines in source file
num_lines = " $( grep -c "^" " ${ src } " ) "
num_lines = " $( grep -c "^" " ${ src } " ) "
# Get number of lines in destination file
# Get number of lines in destination file
@ -463,9 +507,9 @@ parseList() {
total_num = " $num_target_lines "
total_num = " $num_target_lines "
num_invalid = " $(( num_lines-num_correct_lines )) "
num_invalid = " $(( num_lines-num_correct_lines )) "
if [ [ " ${ num_invalid } " -eq 0 ] ] ; then
if [ [ " ${ num_invalid } " -eq 0 ] ] ; then
echo " ${ INFO } Receiv ed ${ num_lines } domains "
echo " ${ INFO } Analyz ed ${ num_lines } domains "
else
else
echo " ${ INFO } Receiv ed ${ num_lines } domains, ${ num_invalid } domains invalid! "
echo " ${ INFO } Analyz ed ${ num_lines } domains, ${ num_invalid } domains invalid! "
fi
fi
# Display sample of invalid lines if we found some
# Display sample of invalid lines if we found some
@ -476,6 +520,29 @@ parseList() {
done <<< " ${ incorrect_lines } "
done <<< " ${ incorrect_lines } "
fi
fi
}
}
compareLists( ) {
local adlistID = " ${ 1 } " target = " ${ 2 } "
# Verify checksum when an older checksum exists
if [ [ -s " ${ target } .sha1 " ] ] ; then
if ! sha1sum --check --status --strict " ${ target } .sha1 " ; then
# The list changed upstream, we need to update the checksum
sha1sum " ${ target } " > " ${ target } .sha1 "
echo " ${ INFO } List has been updated "
database_adlist_status " ${ adlistID } " "1"
database_adlist_updated " ${ adlistID } "
else
echo " ${ INFO } List stayed unchanged "
database_adlist_status " ${ adlistID } " "2"
fi
else
# No checksum available, create one for comparing on the next run
sha1sum " ${ target } " > " ${ target } .sha1 "
# We assume here it was changed upstream
database_adlist_status " ${ adlistID } " "1"
database_adlist_updated " ${ adlistID } "
fi
}
# Download specified URL and perform checks on HTTP status and file content
# Download specified URL and perform checks on HTTP status and file content
gravity_DownloadBlocklistFromUrl( ) {
gravity_DownloadBlocklistFromUrl( ) {
@ -559,31 +626,49 @@ gravity_DownloadBlocklistFromUrl() {
esac ; ;
esac ; ;
esac
esac
local done = "false"
# Determine if the blocklist was downloaded and saved correctly
# Determine if the blocklist was downloaded and saved correctly
if [ [ " ${ success } " = = true ] ] ; then
if [ [ " ${ success } " = = true ] ] ; then
if [ [ " ${ httpCode } " = = "304" ] ] ; then
if [ [ " ${ httpCode } " = = "304" ] ] ; then
# Add domains to database table file
# Add domains to database table file
parseList " ${ adlistID } " " ${ saveLocation } " " ${ target } "
parseList " ${ adlistID } " " ${ saveLocation } " " ${ target } "
database_adlist_status " ${ adlistID } " "2"
database_adlist_number " ${ adlistID } "
done = "true"
# Check if $patternbuffer is a non-zero length file
# Check if $patternbuffer is a non-zero length file
elif [ [ -s " ${ patternBuffer } " ] ] ; then
elif [ [ -s " ${ patternBuffer } " ] ] ; then
# Determine if blocklist is non-standard and parse as appropriate
# Determine if blocklist is non-standard and parse as appropriate
gravity_ParseFileIntoDomains " ${ patternBuffer } " " ${ saveLocation } "
gravity_ParseFileIntoDomains " ${ patternBuffer } " " ${ saveLocation } "
# Add domains to database table file
# Add domains to database table file
parseList " ${ adlistID } " " ${ saveLocation } " " ${ target } "
parseList " ${ adlistID } " " ${ saveLocation } " " ${ target } "
# Update date_updated field in gravity database table
# Compare lists, are they identical?
database_adlist_updated " ${ adlistID } "
compareLists " ${ adlistID } " " ${ saveLocation } "
# Update gravity database table (status and updated timestamp are set in
# compareLists)
database_adlist_number " ${ adlistID } "
done = "true"
else
else
# Fall back to previously cached list if $patternBuffer is empty
# Fall back to previously cached list if $patternBuffer is empty
echo -e " ${ INFO } Received empty file: ${ COL_LIGHT_GREEN } using previously cached list ${ COL_NC } "
echo -e " ${ INFO } Received empty file "
fi
fi
else
fi
# Do we need to fall back to a cached list (if available)?
if [ [ " ${ done } " != "true" ] ] ; then
# Determine if cached list has read permission
# Determine if cached list has read permission
if [ [ -r " ${ saveLocation } " ] ] ; then
if [ [ -r " ${ saveLocation } " ] ] ; then
echo -e " ${ CROSS } List download failed: ${ COL_LIGHT_GREEN } using previously cached list ${ COL_NC } "
echo -e " ${ CROSS } List download failed: ${ COL_LIGHT_GREEN } using previously cached list ${ COL_NC } "
# Add domains to database table file
# Add domains to database table file
parseList " ${ adlistID } " " ${ saveLocation } " " ${ target } "
parseList " ${ adlistID } " " ${ saveLocation } " " ${ target } "
database_adlist_number " ${ adlistID } "
database_adlist_status " ${ adlistID } " "3"
else
else
echo -e " ${ CROSS } List download failed: ${ COL_LIGHT_RED } no cached list available ${ COL_NC } "
echo -e " ${ CROSS } List download failed: ${ COL_LIGHT_RED } no cached list available ${ COL_NC } "
# Manually reset these two numbers because we do not call parseList here
num_lines = 0
num_invalid = 0
database_adlist_number " ${ adlistID } "
database_adlist_status " ${ adlistID } " "4"
fi
fi
fi
fi
}
}