arno
/
pi-hole
mirror of https://github.com/pi-hole/pi-hole
1
0
Fork 0
Code Issues Releases Wiki Activity

Allow limited parsing of ABP style adlists (#5179)

Browse Source
pull/5213/head
Dan Schaper 1 year ago committed by GitHub
parent 04b909c837 71e262c37f
commit 75a32d22a3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 93 additions and 41 deletions
Show Stats Download Patch File Download Diff File

2
advanced/Scripts/piholeDebug.sh
Unescape View File

@ -230,7 +230,7 @@ initialize_debug() {
# This is a function for visually displaying the current test that is being run.
# Accepts one variable: the name of what is being diagnosed
# Colors do not show in the dasboard, but the icons do: [i], [✓], and [✗]
# Colors do not show in the dashboard, but the icons do: [i], [✓], and [✗]
echo_current_diagnostic() {
# Colors are used for visually distinguishing each test in the output
# These colors do not show in the GUI, but the formatting will

14
advanced/Scripts/query.sh
Unescape View File

@ -119,7 +119,7 @@ scanDatabaseTable() {
fi
# Send prepared query to gravity database
result="$(pihole-FTL sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null
result="$(pihole-FTL sqlite3 -separator ',' "${gravityDBfile}" "${querystr}")" 2> /dev/null
if [[ -z "${result}" ]]; then
# Return early when there are no matches in this table
return
@ -139,8 +139,8 @@ scanDatabaseTable() {
# Loop over results and print them
mapfile -t results <<< "${result}"
for result in "${results[@]}"; do
domain="${result/|*}"
if [[ "${result#*|}" == "0" ]]; then
domain="${result/,*}"
if [[ "${result#*,}" == "0" ]]; then
extra=" (disabled)"
else
extra=""
@ -215,10 +215,10 @@ if [[ -n "${exact}" ]]; then
fi
for result in "${results[@]}"; do
match="${result/|*/}"
extra="${result#*|}"
adlistAddress="${extra/|*/}"
extra="${extra#*|}"
match="${result/,*/}"
extra="${result#*,}"
adlistAddress="${extra/,*/}"
extra="${extra#*,}"
if [[ "${extra}" == "0" ]]; then
extra=" (disabled)"
else

118
gravity.sh
Unescape View File

@ -137,6 +137,18 @@ update_gravity_timestamp() {
return 0
}
# Update timestamp when the gravity table was last updated successfully
set_abp_info() {
pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('abp_domains',${abp_domains});"
status="$?"
if [[ "${status}" -ne 0 ]]; then
echo -e "\\n ${CROSS} Unable to update ABP domain status in database ${gravityDBfile}\\n ${output}"
return 1
fi
return 0
}
# Import domains from file and store them in the specified database table
database_table_from_file() {
# Define locals
@ -519,25 +531,52 @@ gravity_DownloadBlocklists() {
gravity_Blackbody=true
}
# num_total_imported_domains increases for each list processed
num_total_imported_domains=0
num_domains=0
num_non_domains=0
# global variable to indicate if we found ABP style domains during the gravity run
# is saved in gravtiy's info table to signal FTL if such domains are available
abp_domains=0
parseList() {
local adlistID="${1}" src="${2}" target="${3}" non_domains sample_non_domains
# This sed does the following things:
# 1. Remove all lines containing no domains
# 2. Remove all domains containing invalid characters. Valid are: a-z, A-Z, 0-9, dot (.), minus (-), underscore (_)
# 3. Append ,adlistID to every line
# 4. Remove trailing period (see https://github.com/pi-hole/pi-hole/issues/4701)
# 5. Ensures there is a newline on the last line
sed -r "/([^\.]+\.)+[^\.]{2,}/!d;/[^a-zA-Z0-9.\_-]/d;s/\.$//;s/$/,${adlistID}/;/.$/a\\" "${src}" >> "${target}"
local adlistID="${1}" src="${2}" target="${3}" temp_file temp_file_base non_domains sample_non_domains valid_domain_pattern abp_domain_pattern
# Create a temporary file for the sed magic instead of using "${target}" directly
# this allows to split the sed commands to improve readability
# we use a file handle here and remove the temporary file immediately so the content will be deleted in any case
# when the script stops
temp_file_base="$(mktemp -p "/tmp" --suffix=".gravity")"
exec 3>"$temp_file_base"
rm "${temp_file_base}"
temp_file="/proc/$$/fd/3"
# define valid domain patterns
# no need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already
# adapted from https://stackoverflow.com/a/30007882
# supported ABP style: ||subdomain.domain.tlp^
valid_domain_pattern="([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]"
abp_domain_pattern="\|\|${valid_domain_pattern}\^"
# 1. Add all valid domains
sed -r "/^${valid_domain_pattern}$/!d" "${src}" > "${temp_file}"
# 2. Add valid ABP style domains if there is at least one such domain
if grep -E "^${abp_domain_pattern}$" -m 1 -q "${src}"; then
echo " ${INFO} List contained AdBlock Plus style domains"
abp_domains=1
sed -r "/^${abp_domain_pattern}$/!d" "${src}" >> "${temp_file}"
fi
# Find lines containing no domains or with invalid characters (see above)
# Find lines containing no domains or with invalid characters (not matching regex above)
# This is simply everything that is not in $temp_file compared to $src
# Remove duplicates from the list
mapfile -t non_domains <<< "$(sed -r "/([^\.]+\.)+[^\.]{2,}/d" < "${src}")"
mapfile -t -O "${#non_domains[@]}" non_domains <<< "$(sed -r "/[^a-zA-Z0-9.\_-]/!d" < "${src}")"
IFS=" " read -r -a non_domains <<< "$(tr ' ' '\n' <<< "${non_domains[@]}" | sort -u | tr '\n' ' ')"
mapfile -t non_domains < <(grep -Fvf "${temp_file}" "${src}" | sort -u )
# 3. Remove trailing period (see https://github.com/pi-hole/pi-hole/issues/4701)
# 4. Append ,adlistID to every line
# 5. Ensures there is a newline on the last line
# and write everything to the target file
sed "s/\.$//;s/$/,${adlistID}/;/.$/a\\" "${temp_file}" >> "${target}"
# A list of items of common local hostnames not to report as unusable
# Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files
@ -553,13 +592,8 @@ parseList() {
# Get a sample of non-domain entries, limited to 5 (the list should already have been de-duplicated)
IFS=" " read -r -a sample_non_domains <<< "$(tr ' ' '\n' <<< "${non_domains[@]}" | head -n 5 | tr '\n' ' ')"
local tmp_new_imported_total
# Get the new number of domains in destination file
tmp_new_imported_total="$(grep -c "^" "${target}")"
# Number of imported lines for this file is the difference between the new total and the old total. (Or, the number of domains we just added.)
num_domains="$(( tmp_new_imported_total-num_total_imported_domains ))"
# Replace the running total with the new total.
num_total_imported_domains="$tmp_new_imported_total"
# Get the number of domains added
num_domains="$(grep -c "^" "${temp_file}")"
# Get the number of non_domains (this is the number of entries left after stripping the source of comments/duplicates/false positives/domains)
num_non_domains="${#non_domains[@]}"
@ -574,6 +608,9 @@ parseList() {
else
echo " ${INFO} Imported ${num_domains} domains"
fi
# close file handle
exec 3<&-
}
compareLists() {
@ -744,18 +781,30 @@ gravity_ParseFileIntoDomains() {
# Most of the lists downloaded are already in hosts file format but the spacing/formatting is not contiguous
# This helps with that and makes it easier to read
# It also helps with debugging so each stage of the script can be researched more in depth
# 1) Remove carriage returns
# 2) Convert all characters to lowercase
# 3) Remove comments (text starting with "#", include possible spaces before the hash sign)
# 1) Convert all characters to lowercase
tr '[:upper:]' '[:lower:]' < "${src}" > "${destination}"
# 2) Remove carriage returns
sed -i 's/\r$//' "${destination}"
# 3a) Remove comments (text starting with "#", include possible spaces before the hash sign)
sed -i 's/\s*#.*//g' "${destination}"
# 3b) Remove lines starting with ! (ABP Comments)
sed -i 's/\s*!.*//g' "${destination}"
# 3c) Remove lines starting with [ (ABP Header)
sed -i 's/\s*\[.*//g' "${destination}"
# 4) Remove lines containing "/"
# 5) Remove leading tabs, spaces, etc.
sed -i -r '/(\/).*$/d' "${destination}"
# 5) Remove leading tabs, spaces, etc. (Also removes leading IP addresses)
sed -i -r 's/^.*\s+//g' "${destination}"
# 6) Remove empty lines
< "${src}" tr -d '\r' | \
tr '[:upper:]' '[:lower:]' | \
sed 's/\s*#.*//g' | \
sed -r '/(\/).*$/d' | \
sed -r 's/^.*\s+//g' | \
sed '/^$/d'> "${destination}"
sed -i '/^$/d' "${destination}"
chmod 644 "${destination}"
}
@ -988,6 +1037,9 @@ fi
# Update gravity timestamp
update_gravity_timestamp
# Set abp_domain info field
set_abp_info
# Ensure proper permissions are set for the database
chown pihole:pihole "${gravityDBfile}"
chmod g+w "${piholeDir}" "${gravityDBfile}"

Write Preview
Loading…
Cancel
Save