arno
/
pi-hole
mirror of https://github.com/pi-hole/pi-hole
1
0
Fork 0
Code Issues Releases Wiki Activity

Finish migrating gravity.db to using our new gravity database. This removes the --skip-download, --blacklist-only, --whitelist-only, and --wildcard-only options as it is not necessary to run gravity when modifying them from now on (gravity really only builds the gravity database and the local.list file).

Browse Source 
Signed-off-by: DL6ER <dl6er@dl6er.de>
pull/2611/head
DL6ER 5 years ago
parent 465a39427b
commit d94bbfca7f
No known key found for this signature in database
GPG Key ID: 00135ACBD90B28DD
1 changed files with 49 additions and 122 deletions
Show Stats Download Patch File Download Diff File

171
gravity.sh
Unescape View File

@ -44,8 +44,6 @@ matterAndLight="${basename}.0.matterandlight.txt"
parsedMatter="${basename}.1.parsedmatter.txt"
preEventHorizon="list.preEventHorizon"
skipDownload="false"
resolver="pihole-FTL"
haveSourceUrls=true
@ -103,8 +101,6 @@ database_table_from_file() {
generate_gravity_database
fi
echo -e " ${INFO} Pi-hole upgrade: Moving content of ${source} into database"
# Truncate table
output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 )
status="$?"
@ -153,22 +149,26 @@ database_table_from_file() {
# Migrate pre-v5.0 list files to database-based Pi-hole versions
migrate_to_database() {
if [[ -e "${whitelistFile}" ]]; then
# Store whitelisted domains in database
database_table_from_file "whitelist" "${whitelistFile}"
if [[ -e "${adListFile}" ]]; then
# Store adlists domains in database
echo -e " ${INFO} Pi-hole upgrade: Moving content of ${adListFile} into database"
database_table_from_file "adlists" "${adListFile}"
fi
if [[ -e "${blacklistFile}" ]]; then
# Store blacklisted domains in database
echo -e " ${INFO} Pi-hole upgrade: Moving content of ${blacklistFile} into database"
database_table_from_file "blacklist" "${blacklistFile}"
fi
if [[ -e "${whitelistFile}" ]]; then
# Store whitelisted domains in database
echo -e " ${INFO} Pi-hole upgrade: Moving content of ${whitelistFile} into database"
database_table_from_file "whitelist" "${whitelistFile}"
fi
if [[ -e "${regexFile}" ]]; then
# Store regex domains in database
echo -e " ${INFO} Pi-hole upgrade: Moving content of ${regexFile} into database"
database_table_from_file "regex" "${regexFile}"
fi
if [[ -e "${adListFile}" ]]; then
# Store adlists domains in database
database_table_from_file "adlists" "${adListFile}"
fi
}
# Determine if DNS resolution is available before proceeding
@ -282,11 +282,9 @@ gravity_SetDownloadOptions() {
*) cmd_ext="";;
esac
if [[ "${skipDownload}" == false ]]; then
echo -e " ${INFO} Target: ${domain} (${url##*/})"
gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}"
echo ""
fi
echo -e " ${INFO} Target: ${domain} (${url##*/})"
gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}"
echo ""
done
gravity_Blackbody=true
}
@ -543,55 +541,35 @@ gravity_SortAndFilterConsolidatedList() {
if [[ "${haveSourceUrls}" == true ]]; then
echo -e "${OVER} ${TICK} ${str}"
fi
echo -e " ${INFO} Number of domains being pulled in by gravity: ${COL_BLUE}${num}${COL_NC}"
echo -e " ${INFO} Gravity pulled in ${COL_BLUE}${num}${COL_NC} domains"
str="Removing duplicate domains"
if [[ "${haveSourceUrls}" == true ]]; then
echo -ne " ${INFO} ${str}..."
fi
echo -ne " ${INFO} ${str}..."
sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}"
echo -e "${OVER} ${TICK} ${str}"
if [[ "${haveSourceUrls}" == true ]]; then
echo -e "${OVER} ${TICK} ${str}"
# Format $preEventHorizon line total as currency
num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
echo -e " ${INFO} Number of unique domains trapped in the Event Horizon: ${COL_BLUE}${num}${COL_NC}"
fi
}
# Whitelist user-defined domains
gravity_Whitelist() {
local num str
if [[ ! -f "${whitelistFile}" ]]; then
echo -e " ${INFO} Nothing to whitelist!"
return 0
fi
num=$(wc -l < "${whitelistFile}")
str="Number of whitelisted domains: ${num}"
# Format $preEventHorizon line total as currency
num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
str="Storing ${COL_BLUE}${num}${COL_NC} unique blocking domains in database"
echo -ne " ${INFO} ${str}..."
echo -e "${OVER} ${INFO} ${str}"
database_table_from_file "gravity" "${piholeDir}/${preEventHorizon}"
echo -e "${OVER} ${TICK} ${str}"
}
# Output count of blacklisted domains and regex filters
gravity_ShowBlockCount() {
# Report number of entries in a table
gravity_Table_Count() {
local table="${1}"
local str="${2}"
local num
num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table} WHERE enabled = 1;")"
echo -e " ${INFO} Number of ${str}: ${num}"
}
if [[ -f "${blacklistFile}" ]]; then
num=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")")
echo -e " ${INFO} Number of blacklisted domains: ${num}"
fi
if [[ -f "${regexFile}" ]]; then
num=$(grep -cv "^#" "${regexFile}")
echo -e " ${INFO} Number of regex filters: ${num}"
fi
# Store regex files in gravity database
database_table_from_file "regex" "${regexFile}"
# Output count of blacklisted domains and regex filters
gravity_ShowCount() {
gravity_Table_Count "blacklist" "blacklisted domains"
gravity_Table_Count "whitelist" "whitelisted domains"
gravity_Table_Count "regex" "regex filters"
}
# Parse list of domains into hosts format
@ -611,7 +589,7 @@ gravity_ParseDomainsIntoHosts() {
}
# Create "localhost" entries into hosts format
gravity_ParseLocalDomains() {
gravity_generateLocalList() {
local hostname
if [[ -s "/etc/hostname" ]]; then
@ -636,24 +614,6 @@ gravity_ParseLocalDomains() {
fi
}
# Create primary blacklist entries
gravity_ParseBlacklistDomains() {
local output status
# Store gravity domains in gravity database
database_table_from_file "gravity" "${piholeDir}/${preEventHorizon}"
}
# Create user-added blacklist entries
gravity_ParseUserDomains() {
if [[ ! -f "${blacklistFile}" ]]; then
return 0
fi
# Fill database table
database_table_from_file "blacklist" "${blacklistFile}"
}
# Trap Ctrl-C
gravity_Trap() {
trap '{ echo -e "\\n\\n ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT
@ -689,7 +649,7 @@ gravity_Cleanup() {
str="Optimizing domains database"
echo -ne " ${INFO} ${str}..."
# Run VACUUM command on database to optimize it
output=$( { sqlite3 "${gravityDBfile}" <<< "VACUUM;"; } 2>&1 )
output=$( { sqlite3 "${gravityDBfile}" "VACUUM;"; } 2>&1 )
status="$?"
if [[ "${status}" -ne 0 ]]; then
@ -728,10 +688,6 @@ for var in "$@"; do
"-f" | "--force" ) forceDelete=true;;
"-o" | "--optimize" ) optimize_database=true;;
"-h" | "--help" ) helpFunc;;
"-sd" | "--skip-download" ) skipDownload=true;;
"-b" | "--blacklist-only" ) listType="blacklist";;
"-w" | "--whitelist-only" ) listType="whitelist";;
"-wild" | "--wildcard-only" ) listType="wildcard"; dnsRestartType="restart";;
esac
done
@ -749,54 +705,25 @@ if [[ "${forceDelete:-}" == true ]]; then
echo -e "${OVER} ${TICK} ${str}"
fi
# Determine which functions to run
if [[ "${skipDownload}" == false ]]; then
# Gravity needs to download blocklists
gravity_CheckDNSResolutionAvailable
gravity_GetBlocklistUrls
if [[ "${haveSourceUrls}" == true ]]; then
gravity_SetDownloadOptions
fi
gravity_ConsolidateDownloadedBlocklists
gravity_SortAndFilterConsolidatedList
else
# Gravity needs to modify Blacklist/Whitelist/Wildcards
echo -e " ${INFO} Using cached Event Horizon list..."
numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
echo -e " ${INFO} ${COL_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon"
fi
# Perform when downloading blocklists, or modifying the whitelist
if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "whitelist" ]]; then
gravity_Whitelist
# Gravity downloads blocklists next
gravity_CheckDNSResolutionAvailable
gravity_GetBlocklistUrls
if [[ "${haveSourceUrls}" == true ]]; then
gravity_SetDownloadOptions
fi
# Build preEventHorizon
gravity_ConsolidateDownloadedBlocklists
gravity_SortAndFilterConsolidatedList
convert_wildcard_to_regex
gravity_ShowBlockCount
# Perform when downloading blocklists, or modifying the white/blacklist (not wildcards)
if [[ "${skipDownload}" == false ]] || [[ "${listType}" == *"list" ]]; then
str="Parsing domains"
echo -ne " ${INFO} ${str}..."
gravity_ParseUserDomains
# Perform when downloading blocklists
if [[ ! "${listType:-}" == "blacklist" ]]; then
gravity_ParseLocalDomains
gravity_ParseBlacklistDomains
fi
echo -e "${OVER} ${TICK} ${str}"
gravity_Cleanup
fi
# Create local.list
gravity_generateLocalList
gravity_ShowCount
gravity_Cleanup
echo ""
# Determine if DNS has been restarted by this instance of gravity
if [[ -z "${dnsWasOffline:-}" ]]; then
# Use "force-reload" when restarting dnsmasq for everything but Wildcards
"${PIHOLE_COMMAND}" restartdns "${dnsRestartType:-force-reload}"
"${PIHOLE_COMMAND}" restartdns reload
fi
"${PIHOLE_COMMAND}" status

Write Preview
Loading…
Cancel
Save