mirror of
https://github.com/pi-hole/pi-hole
synced 2024-12-22 14:58:08 +00:00
Merge pull request #4584 from pi-hole/fix/gravity_internal_sqlite3
Replace calls to sqlite3 by calls to pihole-FTL sqlite3
This commit is contained in:
DL6ER
GitHub
commit
844c4dcdc8
@ -19,13 +19,13 @@ upgrade_gravityDB(){
|
||||
auditFile="${piholeDir}/auditlog.list"
|
||||
|
||||
# Get database version
|
||||
version="$(sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")"
|
||||
version="$(pihole-FTL sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")"
|
||||
|
||||
if [[ "$version" == "1" ]]; then
|
||||
# This migration script upgrades the gravity.db file by
|
||||
# adding the domain_audit table
|
||||
echo -e " ${INFO} Upgrading gravity database from version 1 to 2"
|
||||
sqlite3 "${database}" < "${scriptPath}/1_to_2.sql"
|
||||
pihole-FTL sqlite3 "${database}" < "${scriptPath}/1_to_2.sql"
|
||||
version=2
|
||||
|
||||
# Store audit domains in database table
|
||||
@ -40,28 +40,28 @@ upgrade_gravityDB(){
|
||||
# renaming the regex table to regex_blacklist, and
|
||||
# creating a new regex_whitelist table + corresponding linking table and views
|
||||
echo -e " ${INFO} Upgrading gravity database from version 2 to 3"
|
||||
sqlite3 "${database}" < "${scriptPath}/2_to_3.sql"
|
||||
pihole-FTL sqlite3 "${database}" < "${scriptPath}/2_to_3.sql"
|
||||
version=3
|
||||
fi
|
||||
if [[ "$version" == "3" ]]; then
|
||||
# This migration script unifies the formally separated domain
|
||||
# lists into a single table with a UNIQUE domain constraint
|
||||
echo -e " ${INFO} Upgrading gravity database from version 3 to 4"
|
||||
sqlite3 "${database}" < "${scriptPath}/3_to_4.sql"
|
||||
pihole-FTL sqlite3 "${database}" < "${scriptPath}/3_to_4.sql"
|
||||
version=4
|
||||
fi
|
||||
if [[ "$version" == "4" ]]; then
|
||||
# This migration script upgrades the gravity and list views
|
||||
# implementing necessary changes for per-client blocking
|
||||
echo -e " ${INFO} Upgrading gravity database from version 4 to 5"
|
||||
sqlite3 "${database}" < "${scriptPath}/4_to_5.sql"
|
||||
pihole-FTL sqlite3 "${database}" < "${scriptPath}/4_to_5.sql"
|
||||
version=5
|
||||
fi
|
||||
if [[ "$version" == "5" ]]; then
|
||||
# This migration script upgrades the adlist view
|
||||
# to return an ID used in gravity.sh
|
||||
echo -e " ${INFO} Upgrading gravity database from version 5 to 6"
|
||||
sqlite3 "${database}" < "${scriptPath}/5_to_6.sql"
|
||||
pihole-FTL sqlite3 "${database}" < "${scriptPath}/5_to_6.sql"
|
||||
version=6
|
||||
fi
|
||||
if [[ "$version" == "6" ]]; then
|
||||
@ -69,7 +69,7 @@ upgrade_gravityDB(){
|
||||
# which is automatically associated to all clients not
|
||||
# having their own group assignments
|
||||
echo -e " ${INFO} Upgrading gravity database from version 6 to 7"
|
||||
sqlite3 "${database}" < "${scriptPath}/6_to_7.sql"
|
||||
pihole-FTL sqlite3 "${database}" < "${scriptPath}/6_to_7.sql"
|
||||
version=7
|
||||
fi
|
||||
if [[ "$version" == "7" ]]; then
|
||||
@ -77,21 +77,21 @@ upgrade_gravityDB(){
|
||||
# to ensure uniqueness on the group name
|
||||
# We also add date_added and date_modified columns
|
||||
echo -e " ${INFO} Upgrading gravity database from version 7 to 8"
|
||||
sqlite3 "${database}" < "${scriptPath}/7_to_8.sql"
|
||||
pihole-FTL sqlite3 "${database}" < "${scriptPath}/7_to_8.sql"
|
||||
version=8
|
||||
fi
|
||||
if [[ "$version" == "8" ]]; then
|
||||
# This migration fixes some issues that were introduced
|
||||
# in the previous migration script.
|
||||
echo -e " ${INFO} Upgrading gravity database from version 8 to 9"
|
||||
sqlite3 "${database}" < "${scriptPath}/8_to_9.sql"
|
||||
pihole-FTL sqlite3 "${database}" < "${scriptPath}/8_to_9.sql"
|
||||
version=9
|
||||
fi
|
||||
if [[ "$version" == "9" ]]; then
|
||||
# This migration drops unused tables and creates triggers to remove
|
||||
# obsolete groups assignments when the linked items are deleted
|
||||
echo -e " ${INFO} Upgrading gravity database from version 9 to 10"
|
||||
sqlite3 "${database}" < "${scriptPath}/9_to_10.sql"
|
||||
pihole-FTL sqlite3 "${database}" < "${scriptPath}/9_to_10.sql"
|
||||
version=10
|
||||
fi
|
||||
if [[ "$version" == "10" ]]; then
|
||||
@ -101,31 +101,31 @@ upgrade_gravityDB(){
|
||||
# to keep the copying process generic (needs the same columns in both the
|
||||
# source and the destination databases).
|
||||
echo -e " ${INFO} Upgrading gravity database from version 10 to 11"
|
||||
sqlite3 "${database}" < "${scriptPath}/10_to_11.sql"
|
||||
pihole-FTL sqlite3 "${database}" < "${scriptPath}/10_to_11.sql"
|
||||
version=11
|
||||
fi
|
||||
if [[ "$version" == "11" ]]; then
|
||||
# Rename group 0 from "Unassociated" to "Default"
|
||||
echo -e " ${INFO} Upgrading gravity database from version 11 to 12"
|
||||
sqlite3 "${database}" < "${scriptPath}/11_to_12.sql"
|
||||
pihole-FTL sqlite3 "${database}" < "${scriptPath}/11_to_12.sql"
|
||||
version=12
|
||||
fi
|
||||
if [[ "$version" == "12" ]]; then
|
||||
# Add column date_updated to adlist table
|
||||
echo -e " ${INFO} Upgrading gravity database from version 12 to 13"
|
||||
sqlite3 "${database}" < "${scriptPath}/12_to_13.sql"
|
||||
pihole-FTL sqlite3 "${database}" < "${scriptPath}/12_to_13.sql"
|
||||
version=13
|
||||
fi
|
||||
if [[ "$version" == "13" ]]; then
|
||||
# Add columns number and status to adlist table
|
||||
echo -e " ${INFO} Upgrading gravity database from version 13 to 14"
|
||||
sqlite3 "${database}" < "${scriptPath}/13_to_14.sql"
|
||||
pihole-FTL sqlite3 "${database}" < "${scriptPath}/13_to_14.sql"
|
||||
version=14
|
||||
fi
|
||||
if [[ "$version" == "14" ]]; then
|
||||
# Changes the vw_adlist created in 5_to_6
|
||||
echo -e " ${INFO} Upgrading gravity database from version 14 to 15"
|
||||
sqlite3 "${database}" < "${scriptPath}/14_to_15.sql"
|
||||
pihole-FTL sqlite3 "${database}" < "${scriptPath}/14_to_15.sql"
|
||||
version=15
|
||||
fi
|
||||
}
|
||||
|
||||
@ -142,18 +142,18 @@ AddDomain() {
|
||||
domain="$1"
|
||||
|
||||
# Is the domain in the list we want to add it to?
|
||||
num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")"
|
||||
num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")"
|
||||
requestedListname="$(GetListnameFromTypeId "${typeId}")"
|
||||
|
||||
if [[ "${num}" -ne 0 ]]; then
|
||||
existingTypeId="$(sqlite3 "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")"
|
||||
existingTypeId="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")"
|
||||
if [[ "${existingTypeId}" == "${typeId}" ]]; then
|
||||
if [[ "${verbose}" == true ]]; then
|
||||
echo -e " ${INFO} ${1} already exists in ${requestedListname}, no need to add!"
|
||||
fi
|
||||
else
|
||||
existingListname="$(GetListnameFromTypeId "${existingTypeId}")"
|
||||
sqlite3 "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';"
|
||||
pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';"
|
||||
if [[ "${verbose}" == true ]]; then
|
||||
echo -e " ${INFO} ${1} already exists in ${existingListname}, it has been moved to ${requestedListname}!"
|
||||
fi
|
||||
@ -169,10 +169,10 @@ AddDomain() {
|
||||
# Insert only the domain here. The enabled and date_added fields will be filled
|
||||
# with their default values (enabled = true, date_added = current timestamp)
|
||||
if [[ -z "${comment}" ]]; then
|
||||
sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});"
|
||||
pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});"
|
||||
else
|
||||
# also add comment when variable has been set through the "--comment" option
|
||||
sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');"
|
||||
pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');"
|
||||
fi
|
||||
}
|
||||
|
||||
@ -181,7 +181,7 @@ RemoveDomain() {
|
||||
domain="$1"
|
||||
|
||||
# Is the domain in the list we want to remove it from?
|
||||
num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")"
|
||||
num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")"
|
||||
|
||||
requestedListname="$(GetListnameFromTypeId "${typeId}")"
|
||||
|
||||
@ -198,14 +198,14 @@ RemoveDomain() {
|
||||
fi
|
||||
reload=true
|
||||
# Remove it from the current list
|
||||
sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};"
|
||||
pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};"
|
||||
}
|
||||
|
||||
Displaylist() {
|
||||
local count num_pipes domain enabled status nicedate requestedListname
|
||||
|
||||
requestedListname="$(GetListnameFromTypeId "${typeId}")"
|
||||
data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)"
|
||||
data="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)"
|
||||
|
||||
if [[ -z $data ]]; then
|
||||
echo -e "Not showing empty list"
|
||||
@ -243,10 +243,10 @@ Displaylist() {
|
||||
}
|
||||
|
||||
NukeList() {
|
||||
count=$(sqlite3 "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};")
|
||||
count=$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};")
|
||||
listname="$(GetListnameFromTypeId "${typeId}")"
|
||||
if [ "$count" -gt 0 ];then
|
||||
sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};"
|
||||
pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};"
|
||||
echo " ${TICK} Removed ${count} domain(s) from the ${listname}"
|
||||
else
|
||||
echo " ${INFO} ${listname} already empty. Nothing to do!"
|
||||
|
||||
@ -39,7 +39,7 @@ flushARP(){
|
||||
# Truncate network_addresses table in pihole-FTL.db
|
||||
# This needs to be done before we can truncate the network table due to
|
||||
# foreign key constraints
|
||||
if ! output=$(sqlite3 "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then
|
||||
if ! output=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then
|
||||
echo -e "${OVER} ${CROSS} Failed to truncate network_addresses table"
|
||||
echo " Database location: ${DBFILE}"
|
||||
echo " Output: ${output}"
|
||||
@ -47,7 +47,7 @@ flushARP(){
|
||||
fi
|
||||
|
||||
# Truncate network table in pihole-FTL.db
|
||||
if ! output=$(sqlite3 "${DBFILE}" "DELETE FROM network" 2>&1); then
|
||||
if ! output=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM network" 2>&1); then
|
||||
echo -e "${OVER} ${CROSS} Failed to truncate network table"
|
||||
echo " Database location: ${DBFILE}"
|
||||
echo " Output: ${output}"
|
||||
|
||||
@ -888,7 +888,7 @@ dig_at() {
|
||||
# This helps emulate queries to different domains that a user might query
|
||||
# It will also give extra assurance that Pi-hole is correctly resolving and blocking domains
|
||||
local random_url
|
||||
random_url=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity ORDER BY RANDOM() LIMIT 1")
|
||||
random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity ORDER BY RANDOM() LIMIT 1")
|
||||
|
||||
# Next we need to check if Pi-hole can resolve a domain when the query is sent to it's IP address
|
||||
# This better emulates how clients will interact with Pi-hole as opposed to above where Pi-hole is
|
||||
@ -1202,7 +1202,7 @@ show_db_entries() {
|
||||
IFS=$'\r\n'
|
||||
local entries=()
|
||||
mapfile -t entries < <(\
|
||||
sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" \
|
||||
pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" \
|
||||
-cmd ".headers on" \
|
||||
-cmd ".mode column" \
|
||||
-cmd ".width ${widths}" \
|
||||
@ -1227,7 +1227,7 @@ show_FTL_db_entries() {
|
||||
IFS=$'\r\n'
|
||||
local entries=()
|
||||
mapfile -t entries < <(\
|
||||
sqlite3 "${PIHOLE_FTL_DB_FILE}" \
|
||||
pihole-FTL sqlite3 "${PIHOLE_FTL_DB_FILE}" \
|
||||
-cmd ".headers on" \
|
||||
-cmd ".mode column" \
|
||||
-cmd ".width ${widths}" \
|
||||
@ -1284,7 +1284,7 @@ analyze_gravity_list() {
|
||||
log_write "${COL_GREEN}${gravity_permissions}${COL_NC}"
|
||||
|
||||
show_db_entries "Info table" "SELECT property,value FROM info" "20 40"
|
||||
gravity_updated_raw="$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT value FROM info where property = 'updated'")"
|
||||
gravity_updated_raw="$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT value FROM info where property = 'updated'")"
|
||||
gravity_updated="$(date -d @"${gravity_updated_raw}")"
|
||||
log_write " Last gravity run finished at: ${COL_CYAN}${gravity_updated}${COL_NC}"
|
||||
log_write ""
|
||||
@ -1292,7 +1292,7 @@ analyze_gravity_list() {
|
||||
OLD_IFS="$IFS"
|
||||
IFS=$'\r\n'
|
||||
local gravity_sample=()
|
||||
mapfile -t gravity_sample < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10")
|
||||
mapfile -t gravity_sample < <(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10")
|
||||
log_write " ${COL_CYAN}----- First 10 Gravity Domains -----${COL_NC}"
|
||||
|
||||
for line in "${gravity_sample[@]}"; do
|
||||
|
||||
@ -63,7 +63,7 @@ else
|
||||
fi
|
||||
fi
|
||||
# Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history)
|
||||
deleted=$(sqlite3 "${DBFILE}" "DELETE FROM queries WHERE timestamp >= strftime('%s','now')-86400; select changes() from queries limit 1")
|
||||
deleted=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM queries WHERE timestamp >= strftime('%s','now')-86400; select changes() from queries limit 1")
|
||||
|
||||
# Restart pihole-FTL to force reloading history
|
||||
sudo pihole restartdns
|
||||
|
||||
@ -121,7 +121,7 @@ scanDatabaseTable() {
|
||||
fi
|
||||
|
||||
# Send prepared query to gravity database
|
||||
result="$(sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null
|
||||
result="$(pihole-FTL sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null
|
||||
if [[ -z "${result}" ]]; then
|
||||
# Return early when there are no matches in this table
|
||||
return
|
||||
@ -164,7 +164,7 @@ scanRegexDatabaseTable() {
|
||||
type="${3:-}"
|
||||
|
||||
# Query all regex from the corresponding database tables
|
||||
mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM domainlist WHERE type = ${type}" 2> /dev/null)
|
||||
mapfile -t regexList < <(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain FROM domainlist WHERE type = ${type}" 2> /dev/null)
|
||||
|
||||
# If we have regexps to process
|
||||
if [[ "${#regexList[@]}" -ne 0 ]]; then
|
||||
|
||||
@ -523,13 +523,13 @@ CustomizeAdLists() {
|
||||
|
||||
if CheckUrl "${address}"; then
|
||||
if [[ "${args[2]}" == "enable" ]]; then
|
||||
sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'"
|
||||
pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'"
|
||||
elif [[ "${args[2]}" == "disable" ]]; then
|
||||
sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'"
|
||||
pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'"
|
||||
elif [[ "${args[2]}" == "add" ]]; then
|
||||
sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address, comment) VALUES ('${address}', '${comment}')"
|
||||
pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address, comment) VALUES ('${address}', '${comment}')"
|
||||
elif [[ "${args[2]}" == "del" ]]; then
|
||||
sqlite3 "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'"
|
||||
pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'"
|
||||
else
|
||||
echo "Not permitted"
|
||||
return 1
|
||||
@ -681,12 +681,12 @@ addAudit()
|
||||
done
|
||||
# Insert only the domain here. The date_added field will be
|
||||
# filled with its default value (date_added = current timestamp)
|
||||
sqlite3 "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};"
|
||||
pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};"
|
||||
}
|
||||
|
||||
clearAudit()
|
||||
{
|
||||
sqlite3 "${gravityDBfile}" "DELETE FROM domain_audit;"
|
||||
pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domain_audit;"
|
||||
}
|
||||
|
||||
SetPrivacyLevel() {
|
||||
|
||||
@ -12,14 +12,17 @@ INSERT OR REPLACE INTO "group" SELECT * FROM OLD."group";
|
||||
INSERT OR REPLACE INTO domain_audit SELECT * FROM OLD.domain_audit;
|
||||
|
||||
INSERT OR REPLACE INTO domainlist SELECT * FROM OLD.domainlist;
|
||||
DELETE FROM OLD.domainlist_by_group WHERE domainlist_id NOT IN (SELECT id FROM OLD.domainlist);
|
||||
INSERT OR REPLACE INTO domainlist_by_group SELECT * FROM OLD.domainlist_by_group;
|
||||
|
||||
INSERT OR REPLACE INTO adlist SELECT * FROM OLD.adlist;
|
||||
DELETE FROM OLD.adlist_by_group WHERE adlist_id NOT IN (SELECT id FROM OLD.adlist);
|
||||
INSERT OR REPLACE INTO adlist_by_group SELECT * FROM OLD.adlist_by_group;
|
||||
|
||||
INSERT OR REPLACE INTO info SELECT * FROM OLD.info;
|
||||
|
||||
INSERT OR REPLACE INTO client SELECT * FROM OLD.client;
|
||||
DELETE FROM OLD.client_by_group WHERE client_id NOT IN (SELECT id FROM OLD.client);
|
||||
INSERT OR REPLACE INTO client_by_group SELECT * FROM OLD.client_by_group;
|
||||
|
||||
|
||||
|
||||
@ -287,7 +287,7 @@ package_manager_detect() {
|
||||
# Packages required to run this install script (stored as an array)
|
||||
INSTALLER_DEPS=(git iproute2 whiptail ca-certificates)
|
||||
# Packages required to run Pi-hole (stored as an array)
|
||||
PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 sqlite3 libcap2-bin dns-root-data libcap2 netcat-openbsd)
|
||||
PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 libcap2-bin dns-root-data libcap2 netcat-openbsd)
|
||||
# Packages required for the Web admin interface (stored as an array)
|
||||
# It's useful to separate this from Pi-hole, since the two repos are also setup separately
|
||||
PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-sqlite3" "${phpVer}-xml" "${phpVer}-intl")
|
||||
@ -332,7 +332,7 @@ package_manager_detect() {
|
||||
PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l"
|
||||
OS_CHECK_DEPS=(grep bind-utils)
|
||||
INSTALLER_DEPS=(git iproute newt procps-ng which chkconfig ca-certificates)
|
||||
PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc sqlite libcap nmap-ncat)
|
||||
PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc libcap nmap-ncat)
|
||||
PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo php-xml php-json php-intl)
|
||||
LIGHTTPD_USER="lighttpd"
|
||||
LIGHTTPD_GROUP="lighttpd"
|
||||
|
||||
81
gravity.sh
81
gravity.sh
@ -73,9 +73,9 @@ if [[ -r "${piholeDir}/pihole.conf" ]]; then
|
||||
echo -e " ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}"
|
||||
fi
|
||||
|
||||
# Generate new sqlite3 file from schema template
|
||||
# Generate new SQLite3 file from schema template
|
||||
generate_gravity_database() {
|
||||
if ! sqlite3 "${gravityDBfile}" < "${gravityDBschema}"; then
|
||||
if ! pihole-FTL sqlite3 "${gravityDBfile}" < "${gravityDBschema}"; then
|
||||
echo -e " ${CROSS} Unable to create ${gravityDBfile}"
|
||||
return 1
|
||||
fi
|
||||
@ -85,12 +85,12 @@ generate_gravity_database() {
|
||||
|
||||
# Copy data from old to new database file and swap them
|
||||
gravity_swap_databases() {
|
||||
local str copyGravity
|
||||
local str copyGravity oldAvail
|
||||
str="Building tree"
|
||||
echo -ne " ${INFO} ${str}..."
|
||||
|
||||
# The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once
|
||||
output=$( { sqlite3 "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 )
|
||||
output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 )
|
||||
status="$?"
|
||||
|
||||
if [[ "${status}" -ne 0 ]]; then
|
||||
@ -102,22 +102,6 @@ gravity_swap_databases() {
|
||||
str="Swapping databases"
|
||||
echo -ne " ${INFO} ${str}..."
|
||||
|
||||
# Gravity copying SQL script
|
||||
copyGravity="$(cat "${gravityDBcopy}")"
|
||||
if [[ "${gravityDBfile}" != "${gravityDBfile_default}" ]]; then
|
||||
# Replace default gravity script location by custom location
|
||||
copyGravity="${copyGravity//"${gravityDBfile_default}"/"${gravityDBfile}"}"
|
||||
fi
|
||||
|
||||
output=$( { sqlite3 "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 )
|
||||
status="$?"
|
||||
|
||||
if [[ "${status}" -ne 0 ]]; then
|
||||
echo -e "\\n ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n ${output}"
|
||||
return 1
|
||||
fi
|
||||
echo -e "${OVER} ${TICK} ${str}"
|
||||
|
||||
# Swap databases and remove or conditionally rename old database
|
||||
# Number of available blocks on disk
|
||||
availableBlocks=$(stat -f --format "%a" "${gravityDIR}")
|
||||
@ -125,18 +109,24 @@ gravity_swap_databases() {
|
||||
gravityBlocks=$(stat --format "%b" ${gravityDBfile})
|
||||
# Only keep the old database if available disk space is at least twice the size of the existing gravity.db.
|
||||
# Better be safe than sorry...
|
||||
oldAvail=false
|
||||
if [ "${availableBlocks}" -gt "$((gravityBlocks * 2))" ] && [ -f "${gravityDBfile}" ]; then
|
||||
echo -e " ${TICK} The old database remains available."
|
||||
oldAvail=true
|
||||
mv "${gravityDBfile}" "${gravityOLDfile}"
|
||||
else
|
||||
rm "${gravityDBfile}"
|
||||
fi
|
||||
mv "${gravityTEMPfile}" "${gravityDBfile}"
|
||||
echo -e "${OVER} ${TICK} ${str}"
|
||||
|
||||
if $oldAvail; then
|
||||
echo -e " ${TICK} The old database remains available."
|
||||
fi
|
||||
}
|
||||
|
||||
# Update timestamp when the gravity table was last updated successfully
|
||||
update_gravity_timestamp() {
|
||||
output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | sqlite3 "${gravityDBfile}"; } 2>&1 )
|
||||
output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 )
|
||||
status="$?"
|
||||
|
||||
if [[ "${status}" -ne 0 ]]; then
|
||||
@ -177,7 +167,7 @@ database_table_from_file() {
|
||||
|
||||
# Get MAX(id) from domainlist when INSERTing into this table
|
||||
if [[ "${table}" == "domainlist" ]]; then
|
||||
rowid="$(sqlite3 "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")"
|
||||
rowid="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")"
|
||||
if [[ -z "$rowid" ]]; then
|
||||
rowid=0
|
||||
fi
|
||||
@ -207,7 +197,7 @@ database_table_from_file() {
|
||||
# Store domains in database table specified by ${table}
|
||||
# Use printf as .mode and .import need to be on separate lines
|
||||
# see https://unix.stackexchange.com/a/445615/83260
|
||||
output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | sqlite3 "${gravityDBfile}"; } 2>&1 )
|
||||
output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 )
|
||||
status="$?"
|
||||
|
||||
if [[ "${status}" -ne 0 ]]; then
|
||||
@ -227,7 +217,7 @@ database_table_from_file() {
|
||||
|
||||
# Update timestamp of last update of this list. We store this in the "old" database as all values in the new database will later be overwritten
|
||||
database_adlist_updated() {
|
||||
output=$( { printf ".timeout 30000\\nUPDATE adlist SET date_updated = (cast(strftime('%%s', 'now') as int)) WHERE id = %i;\\n" "${1}" | sqlite3 "${gravityDBfile}"; } 2>&1 )
|
||||
output=$( { printf ".timeout 30000\\nUPDATE adlist SET date_updated = (cast(strftime('%%s', 'now') as int)) WHERE id = %i;\\n" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 )
|
||||
status="$?"
|
||||
|
||||
if [[ "${status}" -ne 0 ]]; then
|
||||
@ -238,7 +228,7 @@ database_adlist_updated() {
|
||||
|
||||
# Check if a column with name ${2} exists in gravity table with name ${1}
|
||||
gravity_column_exists() {
|
||||
output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | sqlite3 "${gravityDBfile}"; } 2>&1 )
|
||||
output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 )
|
||||
if [[ "${output}" == "1" ]]; then
|
||||
return 0 # Bash 0 is success
|
||||
fi
|
||||
@ -253,7 +243,7 @@ database_adlist_number() {
|
||||
return;
|
||||
fi
|
||||
|
||||
output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${num_source_lines}" "${num_invalid}" "${1}" | sqlite3 "${gravityDBfile}"; } 2>&1 )
|
||||
output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${num_source_lines}" "${num_invalid}" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 )
|
||||
status="$?"
|
||||
|
||||
if [[ "${status}" -ne 0 ]]; then
|
||||
@ -269,7 +259,7 @@ database_adlist_status() {
|
||||
return;
|
||||
fi
|
||||
|
||||
output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | sqlite3 "${gravityDBfile}"; } 2>&1 )
|
||||
output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 )
|
||||
status="$?"
|
||||
|
||||
if [[ "${status}" -ne 0 ]]; then
|
||||
@ -386,9 +376,9 @@ gravity_DownloadBlocklists() {
|
||||
fi
|
||||
|
||||
# Retrieve source URLs from gravity database
|
||||
# We source only enabled adlists, sqlite3 stores boolean values as 0 (false) or 1 (true)
|
||||
mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)"
|
||||
mapfile -t sourceIDs <<< "$(sqlite3 "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)"
|
||||
# We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true)
|
||||
mapfile -t sources <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)"
|
||||
mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)"
|
||||
|
||||
# Parse source domains from $sources
|
||||
mapfile -t sourceDomains <<< "$(
|
||||
@ -417,7 +407,7 @@ gravity_DownloadBlocklists() {
|
||||
str="Preparing new gravity database"
|
||||
echo -ne " ${INFO} ${str}..."
|
||||
rm "${gravityTEMPfile}" > /dev/null 2>&1
|
||||
output=$( { sqlite3 "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 )
|
||||
output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 )
|
||||
status="$?"
|
||||
|
||||
if [[ "${status}" -ne 0 ]]; then
|
||||
@ -475,9 +465,28 @@ gravity_DownloadBlocklists() {
|
||||
echo ""
|
||||
done
|
||||
|
||||
str="Creating new gravity databases"
|
||||
echo -ne " ${INFO} ${str}..."
|
||||
|
||||
# Gravity copying SQL script
|
||||
copyGravity="$(cat "${gravityDBcopy}")"
|
||||
if [[ "${gravityDBfile}" != "${gravityDBfile_default}" ]]; then
|
||||
# Replace default gravity script location by custom location
|
||||
copyGravity="${copyGravity//"${gravityDBfile_default}"/"${gravityDBfile}"}"
|
||||
fi
|
||||
|
||||
output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 )
|
||||
status="$?"
|
||||
|
||||
if [[ "${status}" -ne 0 ]]; then
|
||||
echo -e "\\n ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n ${output}"
|
||||
return 1
|
||||
fi
|
||||
echo -e "${OVER} ${TICK} ${str}"
|
||||
|
||||
str="Storing downloaded domains in new gravity database"
|
||||
echo -ne " ${INFO} ${str}..."
|
||||
output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" gravity\\n" "${target}" | sqlite3 "${gravityTEMPfile}"; } 2>&1 )
|
||||
output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" gravity\\n" "${target}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 )
|
||||
status="$?"
|
||||
|
||||
if [[ "${status}" -ne 0 ]]; then
|
||||
@ -782,12 +791,12 @@ gravity_Table_Count() {
|
||||
local table="${1}"
|
||||
local str="${2}"
|
||||
local num
|
||||
num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table};")"
|
||||
num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table};")"
|
||||
if [[ "${table}" == "vw_gravity" ]]; then
|
||||
local unique
|
||||
unique="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(DISTINCT domain) FROM ${table};")"
|
||||
unique="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(DISTINCT domain) FROM ${table};")"
|
||||
echo -e " ${INFO} Number of ${str}: ${num} (${COL_BOLD}${unique} unique domains${COL_NC})"
|
||||
sqlite3 "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});"
|
||||
pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});"
|
||||
else
|
||||
echo -e " ${INFO} Number of ${str}: ${num}"
|
||||
fi
|
||||
|
||||
Loading…
Reference in New Issue
Block a user