arno/pi-hole
1
0
Fork 0
mirror of https://github.com/pi-hole/pi-hole synced 2025-01-05 05:30:55 +00:00
Code Issues Releases Wiki Activity

Finish migrating gravity.db to using our new gravity database. This removes the --skip-download, --blacklist-only, --whitelist-only, and --wildcard-only options as it is not necessary to run gravity when modifying them from now on (gravity really only builds the gravity database and the local.list file).

Browse Source 
Signed-off-by: DL6ER <dl6er@dl6er.de>
This commit is contained in:
DL6ER 2019-04-25 11:18:54 +02:00
parent 465a39427b
commit d94bbfca7f
No known key found for this signature in database
GPG Key ID: 00135ACBD90B28DD
1 changed files with 49 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

171
gravity.sh
View File

@ -44,8 +44,6 @@ matterAndLight="${basename}.0.matterandlight.txt"
parsedMatter="${basename}.1.parsedmatter.txt" parsedMatter="${basename}.1.parsedmatter.txt"
preEventHorizon="list.preEventHorizon" preEventHorizon="list.preEventHorizon"
skipDownload="false"
resolver="pihole-FTL" resolver="pihole-FTL"
haveSourceUrls=true haveSourceUrls=true
@ -103,8 +101,6 @@ database_table_from_file() {
generate_gravity_database generate_gravity_database
fi fi
echo -e " ${INFO} Pi-hole upgrade: Moving content of ${source} into database"
# Truncate table # Truncate table
output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 ) output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 )
status="$?" status="$?"
@ -153,22 +149,26 @@ database_table_from_file() {
# Migrate pre-v5.0 list files to database-based Pi-hole versions # Migrate pre-v5.0 list files to database-based Pi-hole versions
migrate_to_database() { migrate_to_database() {
if [[ -e "${whitelistFile}" ]]; then if [[ -e "${adListFile}" ]]; then
# Store whitelisted domains in database # Store adlists domains in database
database_table_from_file "whitelist" "${whitelistFile}" echo -e " ${INFO} Pi-hole upgrade: Moving content of ${adListFile} into database"
database_table_from_file "adlists" "${adListFile}"
fi fi
if [[ -e "${blacklistFile}" ]]; then if [[ -e "${blacklistFile}" ]]; then
# Store blacklisted domains in database # Store blacklisted domains in database
echo -e " ${INFO} Pi-hole upgrade: Moving content of ${blacklistFile} into database"
database_table_from_file "blacklist" "${blacklistFile}" database_table_from_file "blacklist" "${blacklistFile}"
fi fi
if [[ -e "${whitelistFile}" ]]; then
# Store whitelisted domains in database
echo -e " ${INFO} Pi-hole upgrade: Moving content of ${whitelistFile} into database"
database_table_from_file "whitelist" "${whitelistFile}"
fi
if [[ -e "${regexFile}" ]]; then if [[ -e "${regexFile}" ]]; then
# Store regex domains in database # Store regex domains in database
echo -e " ${INFO} Pi-hole upgrade: Moving content of ${regexFile} into database"
database_table_from_file "regex" "${regexFile}" database_table_from_file "regex" "${regexFile}"
fi fi
if [[ -e "${adListFile}" ]]; then
# Store adlists domains in database
database_table_from_file "adlists" "${adListFile}"
fi
} }
# Determine if DNS resolution is available before proceeding # Determine if DNS resolution is available before proceeding
@ -282,11 +282,9 @@ gravity_SetDownloadOptions() {
*) cmd_ext="";; *) cmd_ext="";;
esac esac
if [[ "${skipDownload}" == false ]]; then echo -e " ${INFO} Target: ${domain} (${url##*/})"
echo -e " ${INFO} Target: ${domain} (${url##*/})" gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}"
gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}" echo ""
echo ""
fi
done done
gravity_Blackbody=true gravity_Blackbody=true
} }
@ -543,55 +541,35 @@ gravity_SortAndFilterConsolidatedList() {
if [[ "${haveSourceUrls}" == true ]]; then if [[ "${haveSourceUrls}" == true ]]; then
echo -e "${OVER} ${TICK} ${str}" echo -e "${OVER} ${TICK} ${str}"
fi fi
echo -e " ${INFO} Number of domains being pulled in by gravity: ${COL_BLUE}${num}${COL_NC}" echo -e " ${INFO} Gravity pulled in ${COL_BLUE}${num}${COL_NC} domains"
str="Removing duplicate domains" str="Removing duplicate domains"
if [[ "${haveSourceUrls}" == true ]]; then echo -ne " ${INFO} ${str}..."
echo -ne " ${INFO} ${str}..."
fi
sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}" sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}"
echo -e "${OVER} ${TICK} ${str}"
if [[ "${haveSourceUrls}" == true ]]; then # Format $preEventHorizon line total as currency
echo -e "${OVER} ${TICK} ${str}" num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
# Format $preEventHorizon line total as currency str="Storing ${COL_BLUE}${num}${COL_NC} unique blocking domains in database"
num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") echo -ne " ${INFO} ${str}..."
echo -e " ${INFO} Number of unique domains trapped in the Event Horizon: ${COL_BLUE}${num}${COL_NC}" database_table_from_file "gravity" "${piholeDir}/${preEventHorizon}"
fi echo -e "${OVER} ${TICK} ${str}"
} }
# Whitelist user-defined domains # Report number of entries in a table
gravity_Whitelist() { gravity_Table_Count() {
local num str local table="${1}"
local str="${2}"
if [[ ! -f "${whitelistFile}" ]]; then local num
echo -e " ${INFO} Nothing to whitelist!" num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table} WHERE enabled = 1;")"
return 0 echo -e " ${INFO} Number of ${str}: ${num}"
fi
num=$(wc -l < "${whitelistFile}")
str="Number of whitelisted domains: ${num}"
echo -ne " ${INFO} ${str}..."
echo -e "${OVER} ${INFO} ${str}"
} }
# Output count of blacklisted domains and regex filters # Output count of blacklisted domains and regex filters
gravity_ShowBlockCount() { gravity_ShowCount() {
local num gravity_Table_Count "blacklist" "blacklisted domains"
gravity_Table_Count "whitelist" "whitelisted domains"
if [[ -f "${blacklistFile}" ]]; then gravity_Table_Count "regex" "regex filters"
num=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")")
echo -e " ${INFO} Number of blacklisted domains: ${num}"
fi
if [[ -f "${regexFile}" ]]; then
num=$(grep -cv "^#" "${regexFile}")
echo -e " ${INFO} Number of regex filters: ${num}"
fi
# Store regex files in gravity database
database_table_from_file "regex" "${regexFile}"
} }
# Parse list of domains into hosts format # Parse list of domains into hosts format
@ -611,7 +589,7 @@ gravity_ParseDomainsIntoHosts() {
} }
# Create "localhost" entries into hosts format # Create "localhost" entries into hosts format
gravity_ParseLocalDomains() { gravity_generateLocalList() {
local hostname local hostname
if [[ -s "/etc/hostname" ]]; then if [[ -s "/etc/hostname" ]]; then
@ -636,24 +614,6 @@ gravity_ParseLocalDomains() {
fi fi
} }
# Create primary blacklist entries
gravity_ParseBlacklistDomains() {
local output status
# Store gravity domains in gravity database
database_table_from_file "gravity" "${piholeDir}/${preEventHorizon}"
}
# Create user-added blacklist entries
gravity_ParseUserDomains() {
if [[ ! -f "${blacklistFile}" ]]; then
return 0
fi
# Fill database table
database_table_from_file "blacklist" "${blacklistFile}"
}
# Trap Ctrl-C # Trap Ctrl-C
gravity_Trap() { gravity_Trap() {
trap '{ echo -e "\\n\\n ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT trap '{ echo -e "\\n\\n ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT
@ -689,7 +649,7 @@ gravity_Cleanup() {
str="Optimizing domains database" str="Optimizing domains database"
echo -ne " ${INFO} ${str}..." echo -ne " ${INFO} ${str}..."
# Run VACUUM command on database to optimize it # Run VACUUM command on database to optimize it
output=$( { sqlite3 "${gravityDBfile}" <<< "VACUUM;"; } 2>&1 ) output=$( { sqlite3 "${gravityDBfile}" "VACUUM;"; } 2>&1 )
status="$?" status="$?"
if [[ "${status}" -ne 0 ]]; then if [[ "${status}" -ne 0 ]]; then
@ -728,10 +688,6 @@ for var in "$@"; do
"-f" | "--force" ) forceDelete=true;; "-f" | "--force" ) forceDelete=true;;
"-o" | "--optimize" ) optimize_database=true;; "-o" | "--optimize" ) optimize_database=true;;
"-h" | "--help" ) helpFunc;; "-h" | "--help" ) helpFunc;;
"-sd" | "--skip-download" ) skipDownload=true;;
"-b" | "--blacklist-only" ) listType="blacklist";;
"-w" | "--whitelist-only" ) listType="whitelist";;
"-wild" | "--wildcard-only" ) listType="wildcard"; dnsRestartType="restart";;
esac esac
done done
@ -749,54 +705,25 @@ if [[ "${forceDelete:-}" == true ]]; then
echo -e "${OVER} ${TICK} ${str}" echo -e "${OVER} ${TICK} ${str}"
fi fi
# Determine which functions to run # Gravity downloads blocklists next
if [[ "${skipDownload}" == false ]]; then gravity_CheckDNSResolutionAvailable
# Gravity needs to download blocklists gravity_GetBlocklistUrls
gravity_CheckDNSResolutionAvailable if [[ "${haveSourceUrls}" == true ]]; then
gravity_GetBlocklistUrls gravity_SetDownloadOptions
if [[ "${haveSourceUrls}" == true ]]; then
gravity_SetDownloadOptions
fi
gravity_ConsolidateDownloadedBlocklists
gravity_SortAndFilterConsolidatedList
else
# Gravity needs to modify Blacklist/Whitelist/Wildcards
echo -e " ${INFO} Using cached Event Horizon list..."
numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
echo -e " ${INFO} ${COL_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon"
fi fi
# Build preEventHorizon
gravity_ConsolidateDownloadedBlocklists
gravity_SortAndFilterConsolidatedList
# Perform when downloading blocklists, or modifying the whitelist # Create local.list
if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "whitelist" ]]; then gravity_generateLocalList
gravity_Whitelist gravity_ShowCount
fi
convert_wildcard_to_regex
gravity_ShowBlockCount
# Perform when downloading blocklists, or modifying the white/blacklist (not wildcards)
if [[ "${skipDownload}" == false ]] || [[ "${listType}" == *"list" ]]; then
str="Parsing domains"
echo -ne " ${INFO} ${str}..."
gravity_ParseUserDomains
# Perform when downloading blocklists
if [[ ! "${listType:-}" == "blacklist" ]]; then
gravity_ParseLocalDomains
gravity_ParseBlacklistDomains
fi
echo -e "${OVER} ${TICK} ${str}"
gravity_Cleanup
fi
gravity_Cleanup
echo "" echo ""
# Determine if DNS has been restarted by this instance of gravity # Determine if DNS has been restarted by this instance of gravity
if [[ -z "${dnsWasOffline:-}" ]]; then if [[ -z "${dnsWasOffline:-}" ]]; then
# Use "force-reload" when restarting dnsmasq for everything but Wildcards "${PIHOLE_COMMAND}" restartdns reload
"${PIHOLE_COMMAND}" restartdns "${dnsRestartType:-force-reload}"
fi fi
"${PIHOLE_COMMAND}" status "${PIHOLE_COMMAND}" status