From 6b33b8b4c09d3f33524a1b89df5dc8e01d12efbe Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 27 Dec 2020 19:14:52 +0100 Subject: [PATCH 01/21] Store status of downloaded list (downloaded, using cache, some error, ...) and number of (in-)valid domains on this list in the gravity database. This updates the gravity databaes to version 14. Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 8 +- .../database_migration/gravity/13_to_14.sql | 13 ++++ advanced/Templates/gravity.db.sql | 7 +- gravity.sh | 74 +++++++++++++++++-- 4 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 advanced/Scripts/database_migration/gravity/13_to_14.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 282ea07b..22f241dd 100644 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -111,9 +111,15 @@ upgrade_gravityDB(){ version=12 fi if [[ "$version" == "12" ]]; then - # Add column date_updated to alist table + # Add column date_updated to adlist table echo -e " ${INFO} Upgrading gravity database from version 12 to 13" sqlite3 "${database}" < "${scriptPath}/12_to_13.sql" version=13 fi + if [[ "$version" == "13" ]]; then + # Add columns number and status to adlist table + echo -e " ${INFO} Upgrading gravity database from version 13 to 14" + sqlite3 "${database}" < "${scriptPath}/13_to_14.sql" + version=14 + fi } diff --git a/advanced/Scripts/database_migration/gravity/13_to_14.sql b/advanced/Scripts/database_migration/gravity/13_to_14.sql new file mode 100644 index 00000000..fa230865 --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/13_to_14.sql @@ -0,0 +1,13 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +ALTER TABLE adlist ADD COLUMN number INTEGER NOT NULL DEFAULT 0; +ALTER TABLE adlist ADD COLUMN invalid_domains INTEGER NOT NULL DEFAULT 0; +ALTER TABLE adlist ADD COLUMN status INTEGER NOT NULL DEFAULT 0; + +UPDATE info SET value = 14 WHERE property = 'version'; + +COMMIT; \ No newline at end of file diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 26b030c8..5d7bafa9 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -32,7 +32,10 @@ CREATE TABLE adlist date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT, - date_updated INTEGER + date_updated INTEGER, + number INTEGER NOT NULL DEFAULT 0, + invalid_domains INTEGER NOT NULL DEFAULT 0, + status INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE adlist_by_group @@ -54,7 +57,7 @@ CREATE TABLE info value TEXT NOT NULL ); -INSERT INTO "info" VALUES('version','13'); +INSERT INTO "info" VALUES('version','14'); CREATE TABLE domain_audit ( diff --git a/gravity.sh b/gravity.sh index 5a831dae..b238275f 100755 --- a/gravity.sh +++ b/gravity.sh @@ -217,6 +217,48 @@ database_adlist_updated() { fi } +# Check if a column with name ${2} exists in gravity table with name ${1} +gravity_column_exists() { + output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + if [[ "${output}" == "1" ]]; then + return 0 # Bash 0 is success + fi + + return 1 # Bash non-0 is failure +} + +# Update number of domain on this list. We store this in the "old" database as all values in the new database will later be overwritten +database_adlist_number() { + # Only try to set number of domains when this field exists in the gravity database + if ! gravity_column_exists "adlist" "number"; then + return; + fi + + output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${total_num}" "${invalid_num}" "${1}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to update number of domains in adlist with ID ${1} in database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + fi +} + +# Update status of this list. We store this in the "old" database as all values in the new database will later be overwritten +database_adlist_status() { + # Only try to set the status when this field exists in the gravity database + if ! gravity_column_exists "adlist" "status"; then + return; + fi + + output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to update status of adlist with ID ${1} in database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + fi +} + # Migrate pre-v5.0 list files to database-based Pi-hole versions migrate_to_database() { # Create database file only if not present @@ -439,6 +481,7 @@ gravity_DownloadBlocklists() { } total_num=0 +invalid_num=0 parseList() { local adlistID="${1}" src="${2}" target="${3}" incorrect_lines # This sed does the following things: @@ -456,11 +499,11 @@ parseList() { num_target_lines="$(grep -c "^" "${target}")" num_correct_lines="$(( num_target_lines-total_num ))" total_num="$num_target_lines" - num_invalid="$(( num_lines-num_correct_lines ))" - if [[ "${num_invalid}" -eq 0 ]]; then - echo " ${INFO} Received ${num_lines} domains" + invalid_num="$(( num_lines-num_correct_lines ))" + if [[ "${invalid_num}" -eq 0 ]]; then + echo " ${INFO} Analyzed ${num_lines} domains" else - echo " ${INFO} Received ${num_lines} domains, ${num_invalid} domains invalid!" + echo " ${INFO} Analyzed ${num_lines} domains, ${invalid_num} domains invalid!" fi # Display sample of invalid lines if we found some @@ -554,31 +597,48 @@ gravity_DownloadBlocklistFromUrl() { esac;; esac + local done="false" # Determine if the blocklist was downloaded and saved correctly if [[ "${success}" == true ]]; then if [[ "${httpCode}" == "304" ]]; then # Add domains to database table file parseList "${adlistID}" "${saveLocation}" "${target}" + database_adlist_status "${adlistID}" "2" + database_adlist_number "${adlistID}" + done="true" # Check if $patternbuffer is a non-zero length file elif [[ -s "${patternBuffer}" ]]; then # Determine if blocklist is non-standard and parse as appropriate gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}" # Add domains to database table file parseList "${adlistID}" "${saveLocation}" "${target}" - # Update date_updated field in gravity database table + # Update gravity database table + database_adlist_status "${adlistID}" "1" database_adlist_updated "${adlistID}" + database_adlist_number "${adlistID}" + done="true" else # Fall back to previously cached list if $patternBuffer is empty - echo -e " ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" + echo -e " ${INFO} Received empty file" fi - else + fi + + # Do we need to fall back to a cached list (if available)? + if [[ "${done}" != "true" ]]; then # Determine if cached list has read permission if [[ -r "${saveLocation}" ]]; then echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" # Add domains to database table file parseList "${adlistID}" "${saveLocation}" "${target}" + database_adlist_number "${adlistID}" + database_adlist_status "${adlistID}" "3" else echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" + # Total number == -1 means there was no cached list that could have been used + total_num=-1 + invalid_num=0 + database_adlist_number "${adlistID}" + database_adlist_status "${adlistID}" "4" fi fi } From 8c56f54a1e4a0ad659914471f8aeb8d92b259956 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 29 Dec 2020 09:54:25 +0100 Subject: [PATCH 02/21] Compare checksum of downloaded list against older checksums to see if the list content changed since the last download Signed-off-by: DL6ER --- gravity.sh | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index b238275f..1defa06f 100755 --- a/gravity.sh +++ b/gravity.sh @@ -514,6 +514,27 @@ parseList() { done <<< "${incorrect_lines}" fi } +compareLists() { + local adlistID="${1}" target="${2}" result + + # Verify checksum when an older checksum exists + if [[ -s "${target}.sha1" ]]; then + if ! sha1sum --check --status --strict "${target}.sha1"; then + # The list changed upstream, we need to update the checksum + sha1sum "${target}" > "${target}.sha1" + echo " ${INFO} List has been updated" + database_adlist_status "${adlistID}" "1" + else + echo " ${INFO} List stayed unchanged" + database_adlist_status "${adlistID}" "2" + fi + else + # No checksum available, create one for comparing on the next run + sha1sum "${target}" > "${target}.sha1" + # We assume here it was changed upstream + database_adlist_status "${adlistID}" "1" + fi +} # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { @@ -612,8 +633,9 @@ gravity_DownloadBlocklistFromUrl() { gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}" # Add domains to database table file parseList "${adlistID}" "${saveLocation}" "${target}" - # Update gravity database table - database_adlist_status "${adlistID}" "1" + # Compare lists, are they identical? + compareLists "${adlistID}" "${saveLocation}" + # Update gravity database table (status is set in compareLists) database_adlist_updated "${adlistID}" database_adlist_number "${adlistID}" done="true" From a216848c1db65dc1be5a1928e3f225c7f10be694 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 29 Dec 2020 20:28:09 +0100 Subject: [PATCH 03/21] Only update time of last list change when we see a list for the first time or when it really changed content Signed-off-by: DL6ER --- gravity.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 1defa06f..0119b710 100755 --- a/gravity.sh +++ b/gravity.sh @@ -524,6 +524,7 @@ compareLists() { sha1sum "${target}" > "${target}.sha1" echo " ${INFO} List has been updated" database_adlist_status "${adlistID}" "1" + database_adlist_updated "${adlistID}" else echo " ${INFO} List stayed unchanged" database_adlist_status "${adlistID}" "2" @@ -531,8 +532,10 @@ compareLists() { else # No checksum available, create one for comparing on the next run sha1sum "${target}" > "${target}.sha1" + echo " ${INFO} This list is new" # We assume here it was changed upstream database_adlist_status "${adlistID}" "1" + database_adlist_updated "${adlistID}" fi } @@ -635,8 +638,8 @@ gravity_DownloadBlocklistFromUrl() { parseList "${adlistID}" "${saveLocation}" "${target}" # Compare lists, are they identical? compareLists "${adlistID}" "${saveLocation}" - # Update gravity database table (status is set in compareLists) - database_adlist_updated "${adlistID}" + # Update gravity database table (status and updated timestamp are set in + # compareLists) database_adlist_number "${adlistID}" done="true" else From 0944807491ca40009eabe47c5198907b518100c4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 29 Dec 2020 20:35:48 +0100 Subject: [PATCH 04/21] Actually store correct number of domains for the individual lists (and not the sum of the so far collected number of domains) Signed-off-by: DL6ER --- gravity.sh | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/gravity.sh b/gravity.sh index 0119b710..684d64b3 100755 --- a/gravity.sh +++ b/gravity.sh @@ -234,7 +234,7 @@ database_adlist_number() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${total_num}" "${invalid_num}" "${1}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${num_lines}" "${num_invalid}" "${1}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -481,7 +481,8 @@ gravity_DownloadBlocklists() { } total_num=0 -invalid_num=0 +num_lines=0 +num_invalid=0 parseList() { local adlistID="${1}" src="${2}" target="${3}" incorrect_lines # This sed does the following things: @@ -492,18 +493,18 @@ parseList() { # Find (up to) five domains containing invalid characters (see above) incorrect_lines="$(sed -e "/[^a-zA-Z0-9.\_-]/!d" "${src}" | head -n 5)" - local num_lines num_target_lines num_correct_lines num_invalid + local num_target_lines num_correct_lines num_invalid # Get number of lines in source file num_lines="$(grep -c "^" "${src}")" # Get number of lines in destination file num_target_lines="$(grep -c "^" "${target}")" num_correct_lines="$(( num_target_lines-total_num ))" total_num="$num_target_lines" - invalid_num="$(( num_lines-num_correct_lines ))" - if [[ "${invalid_num}" -eq 0 ]]; then + num_invalid="$(( num_lines-num_correct_lines ))" + if [[ "${num_invalid}" -eq 0 ]]; then echo " ${INFO} Analyzed ${num_lines} domains" else - echo " ${INFO} Analyzed ${num_lines} domains, ${invalid_num} domains invalid!" + echo " ${INFO} Analyzed ${num_lines} domains, ${num_invalid} domains invalid!" fi # Display sample of invalid lines if we found some @@ -515,7 +516,7 @@ parseList() { fi } compareLists() { - local adlistID="${1}" target="${2}" result + local adlistID="${1}" target="${2}" # Verify checksum when an older checksum exists if [[ -s "${target}.sha1" ]]; then @@ -659,9 +660,9 @@ gravity_DownloadBlocklistFromUrl() { database_adlist_status "${adlistID}" "3" else echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" - # Total number == -1 means there was no cached list that could have been used - total_num=-1 - invalid_num=0 + # Manually reset these two numbers because we do not call parseList here + num_lines=0 + num_invalid=0 database_adlist_number "${adlistID}" database_adlist_status "${adlistID}" "4" fi From a2625df5e2a7e406cadcd430ea3902c47234769c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 30 Dec 2020 11:27:34 +0100 Subject: [PATCH 05/21] Remove "The list is new" comment because it is superfluous Signed-off-by: DL6ER --- gravity.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 684d64b3..0dd65bfb 100755 --- a/gravity.sh +++ b/gravity.sh @@ -533,7 +533,6 @@ compareLists() { else # No checksum available, create one for comparing on the next run sha1sum "${target}" > "${target}.sha1" - echo " ${INFO} This list is new" # We assume here it was changed upstream database_adlist_status "${adlistID}" "1" database_adlist_updated "${adlistID}" From bfee230c799185b5898f226269e9efb85207d125 Mon Sep 17 00:00:00 2001 From: Andras Tim Date: Mon, 11 Jan 2021 17:05:17 +0100 Subject: [PATCH 06/21] scripts/list: User same wildcard regex that the UI use The CLI and the UI should do the same to be able to add/remove the same records via any supported interface. Signed-off-by: Andras Tim --- advanced/Scripts/list.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 16bb2001..fde46552 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -112,7 +112,7 @@ ProcessDomainList() { for dom in "${domList[@]}"; do # Format domain into regex filter if requested if [[ "${wildcard}" == true ]]; then - dom="(^|\\.)${dom//\./\\.}$" + dom="(\\.|^)${dom//\./\\.}$" fi # Logic: If addmode then add to desired list and remove from the other; From e5bfafefb97995ce4cdfadfed1e745141ac57c28 Mon Sep 17 00:00:00 2001 From: mtzfederico Date: Fri, 22 Jan 2021 13:23:59 -0600 Subject: [PATCH 07/21] Moved where external.conf is included Signed-off-by: mtzfederico --- advanced/lighttpd.conf.debian | 8 ++++---- advanced/lighttpd.conf.fedora | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/advanced/lighttpd.conf.debian b/advanced/lighttpd.conf.debian index 6020dfd8..cd6d7737 100644 --- a/advanced/lighttpd.conf.debian +++ b/advanced/lighttpd.conf.debian @@ -81,6 +81,10 @@ mimetype.assign = ( ".woff2" => "font/woff2" ) +# Add user chosen options held in external file +# This uses include_shell instead of an include wildcard for compatibility +include_shell "cat external.conf 2>/dev/null" + # default listening port for IPv6 falls back to the IPv4 port include_shell "/usr/share/lighttpd/use-ipv6.pl " + server.port @@ -109,7 +113,3 @@ $HTTP["url"] =~ "^/admin/\.(.*)" { # Default expire header expire.url = ( "" => "access plus 0 seconds" ) - -# Add user chosen options held in external file -# This uses include_shell instead of an include wildcard for compatibility -include_shell "cat external.conf 2>/dev/null" diff --git a/advanced/lighttpd.conf.fedora b/advanced/lighttpd.conf.fedora index 8e3bddc4..64428617 100644 --- a/advanced/lighttpd.conf.fedora +++ b/advanced/lighttpd.conf.fedora @@ -82,6 +82,10 @@ mimetype.assign = ( ".woff2" => "font/woff2" ) +# Add user chosen options held in external file +# This uses include_shell instead of an include wildcard for compatibility +include_shell "cat external.conf 2>/dev/null" + # default listening port for IPv6 falls back to the IPv4 port #include_shell "/usr/share/lighttpd/use-ipv6.pl " + server.port #include_shell "/usr/share/lighttpd/create-mime.assign.pl" @@ -117,7 +121,3 @@ $HTTP["url"] =~ "^/admin/\.(.*)" { # Default expire header expire.url = ( "" => "access plus 0 seconds" ) - -# Add user chosen options held in external file -# This uses include_shell instead of an include wildcard for compatibility -include_shell "cat external.conf 2>/dev/null" From c42b97ddb22fd138ed5fd1cc6740acebf0f9d125 Mon Sep 17 00:00:00 2001 From: Michael Woolweaver Date: Tue, 26 Jan 2021 01:04:37 -0600 Subject: [PATCH 08/21] fix typo in comment Signed-off-by: Michael Woolweaver --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index dfb863e9..862722a9 100755 --- a/gravity.sh +++ b/gravity.sh @@ -88,7 +88,7 @@ gravity_swap_databases() { str="Building tree" echo -ne " ${INFO} ${str}..." - # The index is intentionally not UNIQUE as prro quality adlists may contain domains more than once + # The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once output=$( { sqlite3 "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) status="$?" From 1ab193fa9d5f94798cd4e50d969495d60913ed78 Mon Sep 17 00:00:00 2001 From: Alexander Schmitz Date: Wed, 27 Jan 2021 11:00:43 +0100 Subject: [PATCH 09/21] Update piholeDebug.sh Signed-off-by: Alexander Schmitz --- advanced/Scripts/piholeDebug.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 318e3252..13a886f1 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -695,7 +695,7 @@ ping_gateway() { # Check if we are using IPv4 or IPv6 # Find the default gateway using IPv4 or IPv6 local gateway - gateway="$(ip -"${protocol}" route | grep default | cut -d ' ' -f 3)" + gateway="$(ip -"${protocol}" route | grep default | grep "${PIHOLE_INTERFACE}" | cut -d ' ' -f 3)" # If the gateway variable has a value (meaning a gateway was found), if [[ -n "${gateway}" ]]; then From 7dc7cbb80e4156cab3078c442aa38a0ca00cb2dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 11 Feb 2021 19:45:32 +0100 Subject: [PATCH 10/21] Add hostname to teleporter backup file if called from cli MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index da2afb0f..5e7d90f9 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -238,18 +238,18 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423 # 168.192.in-addr.arpa to 192.168.0.0/16 # 192.in-addr.arpa to 192.0.0.0/8 if [[ "${CONDITIONAL_FORWARDING_REVERSE}" == *"in-addr.arpa" ]];then - arrRev=("${CONDITIONAL_FORWARDING_REVERSE//./ }") - case ${#arrRev[@]} in + arrRev=("${CONDITIONAL_FORWARDING_REVERSE//./ }") + case ${#arrRev[@]} in 6 ) REV_SERVER_CIDR="${arrRev[3]}.${arrRev[2]}.${arrRev[1]}.${arrRev[0]}/32";; 5 ) REV_SERVER_CIDR="${arrRev[2]}.${arrRev[1]}.${arrRev[0]}.0/24";; 4 ) REV_SERVER_CIDR="${arrRev[1]}.${arrRev[0]}.0.0/16";; - 3 ) REV_SERVER_CIDR="${arrRev[0]}.0.0.0/8";; + 3 ) REV_SERVER_CIDR="${arrRev[0]}.0.0.0/8";; esac else # Set REV_SERVER_CIDR to whatever value it was set to REV_SERVER_CIDR="${CONDITIONAL_FORWARDING_REVERSE}" fi - + # If REV_SERVER_CIDR is not converted by the above, then use the REV_SERVER_TARGET variable to derive it if [ -z "${REV_SERVER_CIDR}" ]; then # Convert existing input to /24 subnet (preserves legacy behavior) @@ -636,8 +636,11 @@ Interfaces: Teleporter() { local datetimestamp + local host datetimestamp=$(date "+%Y-%m-%d_%H-%M-%S") - php /var/www/html/admin/scripts/pi-hole/php/teleporter.php > "pi-hole-teleporter_${datetimestamp}.tar.gz" + host=$(hostname) + host="${host//./_}" + php /var/www/html/admin/scripts/pi-hole/php/teleporter.php > "pi-hole-${host}-teleporter_${datetimestamp}.tar.gz" } checkDomain() From 78027bd2bfeba890af7a0e440b9c7b6b15c9b3d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 12 Feb 2021 20:25:01 +0100 Subject: [PATCH 11/21] Add default noname if hostname is empty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 5e7d90f9..53f815c5 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -640,7 +640,7 @@ Teleporter() { datetimestamp=$(date "+%Y-%m-%d_%H-%M-%S") host=$(hostname) host="${host//./_}" - php /var/www/html/admin/scripts/pi-hole/php/teleporter.php > "pi-hole-${host}-teleporter_${datetimestamp}.tar.gz" + php /var/www/html/admin/scripts/pi-hole/php/teleporter.php > "pi-hole-${host:-noname}-teleporter_${datetimestamp}.tar.gz" } checkDomain() From 126b9ae381d2a263cd888df5c97d3099a44d4064 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 2 Mar 2021 20:08:09 +0100 Subject: [PATCH 12/21] Add missing three new columns to pihole -g -r Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 3fe43527..abed28b4 100755 --- a/gravity.sh +++ b/gravity.sh @@ -176,7 +176,7 @@ database_table_from_file() { echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" elif [[ "${table}" == "adlist" ]]; then # Adlist table format - echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"," >> "${tmpFile}" + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\",,0,0,0" >> "${tmpFile}" else # White-, black-, and regexlist table format echo "${rowid},${type},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" From 13cf0c12881447a4fc7cab5cedee6e2cd7514e0c Mon Sep 17 00:00:00 2001 From: Dany <69208423+Cafetier@users.noreply.github.com> Date: Wed, 17 Mar 2021 14:18:50 -0400 Subject: [PATCH 13/21] Aligned pihole logo to center, Fixed responsive issue (#4036) * Aligned pihole logo to center, Fixed responsive issue * removed duplicate instance of #splashpage Signed-off-by: Dany Gauthier --- advanced/blockingpage.css | 74 +++++++++++++++++++++++++++++++++++---- advanced/index.php | 10 +++--- 2 files changed, 74 insertions(+), 10 deletions(-) diff --git a/advanced/blockingpage.css b/advanced/blockingpage.css index 9f82a27e..0cc7a65c 100644 --- a/advanced/blockingpage.css +++ b/advanced/blockingpage.css @@ -145,7 +145,17 @@ body { } /* User is greeted with a splash page when browsing to Pi-hole IP address */ -#splashpage { background: #222; color: rgba(255, 255, 255, 0.7); text-align: center; } +#splashpage { + background: #222; + color: rgba(255, 255, 255, 0.7); + text-align: center; + width: 100%; + height: 100%; + display: flex; + align-items: center; + justify-content: center; +} + #splashpage img { margin: 5px; width: 256px; } #splashpage b { color: inherit; } @@ -196,6 +206,26 @@ header #bpAlt label { display: block; } +html, body { + height: 100%; +} + +#pihole_card { + width: 400px; + height: auto; + max-width: 400px; +} + + #pihole_card p, #pihole_card a { + font-size: 13pt; + text-align: center; + } + +#pihole_logo_splash { + height: auto; + width: 100%; +} + /* Click anywhere else on screen to hide #bpAbout */ #bpAboutToggle:checked { display: block; @@ -382,12 +412,44 @@ footer { /* Responsive Content */ @media only screen and (max-width: 500px) { - h1 a { font-size: 1.8rem; min-width: 170px; } - footer span::before { content: "Generated "; } - footer span { display: block; } + h1 a { + font-size: 1.8rem; + min-width: 170px; + } + + footer span::before { + content: "Generated "; + } + + footer span { + display: block; + } } @media only screen and (min-width: 1251px) { - #bpWrapper, footer { border-radius: 0 0 5px 5px; } - #bpAbout { border-right-width: 1px; } + #bpWrapper, footer { + border-radius: 0 0 5px 5px; + } + + #bpAbout { + border-right-width: 1px; + } +} + +@media only screen and (max-width: 400px) { + #pihole_card { + width: 100%; + height: auto; + } + + #pihole_card p, #pihole_card a { + font-size: 100%; + } +} + +@media only screen and (max-width: 256px) { + #pihole_logo_splash { + width: 90% !important; + height: auto; + } } diff --git a/advanced/index.php b/advanced/index.php index ca8c35e4..eea44462 100644 --- a/advanced/index.php +++ b/advanced/index.php @@ -71,15 +71,17 @@ if ($serverName === "pi.hole" $viewPort + ● $serverName - Pi-hole logo -
-

Pi-hole: Your black hole for Internet advertisements

- Did you mean to go to the admin panel? +
+ Pi-hole logo id="pihole_logo_splash" /> + <p>Pi-<strong>hole</strong>: Your black hole for Internet advertisements</p> + <a href=Did you mean to go to the admin panel? +
"; From 20b6f9cceb9135dc8997974c898fbcd22dce427f Mon Sep 17 00:00:00 2001 From: jbzdarkid Date: Wed, 17 Mar 2021 18:35:27 -0700 Subject: [PATCH 14/21] Improve comments in basic-install.sh Signed-off-by: jbzdarkid --- automated install/basic-install.sh | 321 +++++++++++++---------------- 1 file changed, 148 insertions(+), 173 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4ab5c0e2..762ee110 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -21,8 +21,8 @@ # instead of continuing the installation with something broken set -e -# Set PATH to a usual default to assure that all basic commands are available. -# When using "su" an uncomplete PATH could be passed: https://github.com/pi-hole/pi-hole/issues/3209 +# Append common folders to the PATH to ensure that all basic commands are available. +# When using "su" an incomplete PATH could be passed: https://github.com/pi-hole/pi-hole/issues/3209 export PATH+=':/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' ######## VARIABLES ######### @@ -59,12 +59,17 @@ coltable=/opt/pihole/COL_TABLE # Root of the web server webroot="/var/www/html" -# We store several other directories and + +# We clone (or update) two git repositories during the install. This helps to make sure that we always have the latest versions of the relevant files. +# AdminLTE is used to set up the Web admin interface. +# Pi-hole contains various setup scripts and files which are critical to the installation. +# Search for "PI_HOLE_LOCAL_REPO" in this file to see all such scripts. +# Two notable scripts are gravity.sh (used to generate the HOSTS file) and advanced/Scripts/webpage.sh (used to install the Web admin interface) webInterfaceGitUrl="https://github.com/pi-hole/AdminLTE.git" webInterfaceDir="${webroot}/admin" piholeGitUrl="https://github.com/pi-hole/pi-hole.git" PI_HOLE_LOCAL_REPO="/etc/.pihole" -# These are the names of pi-holes files, stored in an array +# List of pihole scripts, stored in an array PI_HOLE_FILES=(chronometer list piholeDebug piholeLogFlush setupLCD update version gravity uninstall webpage) # This directory is where the Pi-hole scripts will be installed PI_HOLE_INSTALL_DIR="/opt/pihole" @@ -76,11 +81,10 @@ if [ -z "$useUpdateVars" ]; then fi adlistFile="/etc/pihole/adlists.list" -# Pi-hole needs an IP address; to begin, these variables are empty since we don't know what the IP is until -# this script can run +# Pi-hole needs an IP address; to begin, these variables are empty since we don't know what the IP is until this script can run IPV4_ADDRESS=${IPV4_ADDRESS} IPV6_ADDRESS=${IPV6_ADDRESS} -# By default, query logging is enabled and the dashboard is set to be installed +# Give settings their default values. These may be changed by prompts later in the script. QUERY_LOGGING=true INSTALL_WEB_INTERFACE=true PRIVACY_LEVEL=0 @@ -98,9 +102,8 @@ if [ -t 0 ] ; then else screen_size="24 80" fi -# Set rows variable to contain first number +# Determine terminal rows and columns by parsing screen_size printf -v rows '%d' "${screen_size%% *}" -# Set columns variable to contain second number printf -v columns '%d' "${screen_size##* }" # Divide by two so the dialogs take up half of the screen, which looks nice. @@ -171,9 +174,8 @@ show_ascii_berry() { } is_command() { - # Checks for existence of string passed in as only function argument. - # Exit value of 0 when exists, 1 if not exists. Value is the result - # of the `command` shell built-in call. + # Checks to see if the given command (passed as a string argument) exists on the system. + # The function returns 0 (success) if the command exists, and 1 if it doesn't. local check_command="$1" command -v "${check_command}" >/dev/null 2>&1 @@ -190,15 +192,15 @@ os_check() { detected_version=$(grep VERSION_ID /etc/os-release | cut -d '=' -f2 | tr -d '"') cmdResult="$(dig +short -t txt ${remote_os_domain} @ns1.pi-hole.net 2>&1; echo $?)" - #Get the return code of the previous command (last line) + # Gets the return code of the previous command (last line) digReturnCode="${cmdResult##*$'\n'}" if [ ! "${digReturnCode}" == "0" ]; then valid_response=false else - # Dig returned 0 code, so get the actual response, and loop through it to determine if the detected variables above are valid + # Dig returned 0 (success), so get the actual response, and loop through it to determine if the detected variables above are valid response="${cmdResult%%$'\n'*}" - # If the value of ${result} is a single 0, then this is the return code, not the response. Response is blank + # If the value of ${response} is a single 0, then this is the return code, not an actual response. if [ "${response}" == 0 ]; then valid_response=false fi @@ -209,6 +211,7 @@ os_check() { distro_part="${distro_and_versions%%=*}" versions_part="${distro_and_versions##*=}" + # If the distro part is a (case-insensistive) substring of the computer OS if [[ "${detected_os^^}" =~ ${distro_part^^} ]]; then valid_os=true IFS="," read -r -a supportedVer <<<"${versions_part}" @@ -281,9 +284,9 @@ if is_command apt-get ; then PKG_MANAGER="apt-get" # A variable to store the command used to update the package cache UPDATE_PKG_CACHE="${PKG_MANAGER} update" - # An array for something... + # The command we will use to actually install packages PKG_INSTALL=("${PKG_MANAGER}" -qq --no-install-recommends install) - # grep -c will return 1 retVal on 0 matches, block this throwing the set -e with an OR TRUE + # grep -c will return 1 if there are no matches. This is an acceptable condition, so we OR TRUE to prevent set -e exiting the script. PKG_COUNT="${PKG_MANAGER} -s -o Debug::NoLocking=true upgrade | grep -c ^Inst || true" # Some distros vary slightly so these fixes for dependencies may apply # on Ubuntu 18.04.1 LTS we need to add the universe repository to gain access to dhcpcd5 @@ -291,7 +294,7 @@ if is_command apt-get ; then if awk 'BEGIN{a=1;b=0}/bionic main/{a=0}/bionic.*universe/{b=1}END{exit a + b}' ${APT_SOURCES}; then if ! whiptail --defaultno --title "Dependencies Require Update to Allowed Repositories" --yesno "Would you like to enable 'universe' repository?\\n\\nThis repository is required by the following packages:\\n\\n- dhcpcd5" "${r}" "${c}"; then printf " %b Aborting installation: Dependencies could not be installed.\\n" "${CROSS}" - exit 1 # exit the installer + exit 1 else printf " %b Enabling universe package repository for Ubuntu Bionic\\n" "${INFO}" cp -p ${APT_SOURCES} ${APT_SOURCES}.backup # Backup current repo list @@ -323,7 +326,8 @@ if is_command apt-get ; then phpInsNewer=true fi fi - # Check if installed php is v 7.0, or newer to determine packages to install + # Several other packages depend on the version of PHP. If PHP is not installed, or an insufficient version, + # those packages should fall back to the default (latest?) if [[ "$phpInsNewer" != true ]]; then # Prefer the php metapackage if it's there if apt-cache show php > /dev/null 2>&1; then @@ -337,7 +341,8 @@ if is_command apt-get ; then exit 1 fi else - # Newer php is installed, its common, cgi & sqlite counterparts are deps + # Else, PHP is already installed at a version beyond v7.0, so the additional packages + # should match version with the current PHP version. phpVer="php$phpInsMajor.$phpInsMinor" fi # We also need the correct version for `php-sqlite` (which differs across distros) @@ -349,13 +354,12 @@ if is_command apt-get ; then printf " %b Aborting installation: No SQLite PHP module was found in APT repository.\\n" "${CROSS}" exit 1 fi - # Since our install script is so large, we need several other programs to successfully get a machine provisioned - # These programs are stored in an array so they can be looped through later + # Packages required to run this install script (stored as an array) INSTALLER_DEPS=(dhcpcd5 git "${iproute_pkg}" whiptail dnsutils) - # Pi-hole itself has several dependencies that also need to be installed + # Packages required to run Pi-hole (stored as an array) PIHOLE_DEPS=(cron curl iputils-ping lsof netcat psmisc sudo unzip wget idn2 sqlite3 libcap2-bin dns-root-data libcap2) - # The Web dashboard has some that also need to be installed - # It's useful to separate the two since our repos are also setup as "Core" code and "Web" code + # Packages required for the Web admin interface (stored as an array) + # It's useful to separate this from Pi-hole, since the two repos are also setup separately PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-${phpSqlite}" "${phpVer}-xml" "${phpVer}-json" "${phpVer}-intl") # The Web server user, LIGHTTPD_USER="www-data" @@ -364,20 +368,19 @@ if is_command apt-get ; then # and config file LIGHTTPD_CFG="lighttpd.conf.debian" - # A function to check... + # This function waits for dpkg to unlock, which signals that the previous apt-get command has finished. test_dpkg_lock() { - # An iterator used for counting loop iterations i=0 # fuser is a program to show which processes use the named files, sockets, or filesystems - # So while the command is true - while fuser /var/lib/dpkg/lock >/dev/null 2>&1 ; do - # Wait half a second + # So while the lock is held, + while fuser /var/lib/dpkg/lock >/dev/null 2>&1 + do + # we wait half a second, sleep 0.5 - # and increase the iterator + # increase the iterator, ((i=i+1)) done - # Always return success, since we only return if there is no - # lock (anymore) + # and then report success once dpkg is unlocked. return 0 } @@ -390,6 +393,7 @@ elif is_command rpm ; then PKG_MANAGER="yum" fi + # These variable names match the ones in the Debian family. See above for an explanation of what they are for. PKG_INSTALL=("${PKG_MANAGER}" install -y) PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l" INSTALLER_DEPS=(git iproute newt procps-ng which chkconfig bind-utils) @@ -476,7 +480,7 @@ elif is_command rpm ; then # Warn user of unsupported version of Fedora or CentOS if ! whiptail --defaultno --title "Unsupported RPM based distribution" --yesno "Would you like to continue installation on an unsupported RPM based distribution?\\n\\nPlease ensure the following packages have been installed manually:\\n\\n- lighttpd\\n- lighttpd-fastcgi\\n- PHP version 7+" "${r}" "${c}"; then printf " %b Aborting installation due to unsupported RPM based distribution\\n" "${CROSS}" - exit # exit the installer + exit else printf " %b Continuing installation with unsupported RPM based distribution\\n" "${INFO}" fi @@ -540,8 +544,9 @@ make_repo() { # Check current branch. If it is master, then reset to the latest available tag. # In case extra commits have been added after tagging/release (i.e in case of metadata updates/README.MD tweaks) curBranch=$(git rev-parse --abbrev-ref HEAD) - if [[ "${curBranch}" == "master" ]]; then #If we're calling make_repo() then it should always be master, we may not need to check. - git reset --hard "$(git describe --abbrev=0 --tags)" || return $? + if [[ "${curBranch}" == "master" ]]; then + # If we're calling make_repo() then it should always be master, we may not need to check. + git reset --hard "$(git describe --abbrev=0 --tags)" || return $? fi # Show a colored message showing it's status printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" @@ -589,7 +594,7 @@ update_repo() { return 0 } -# A function that combines the functions previously made +# A function that combines the previous git functions to update or clone a repo getGitFiles() { # Setup named variables for the git repos # We need the directory @@ -613,9 +618,8 @@ getGitFiles() { # Attempt to make the repository, showing an error on failure make_repo "${directory}" "${remoteRepo}" || { printf "\\n %bError: Could not update local repository. Contact support.%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; exit 1; } fi - # echo a blank line echo "" - # and return success? + # Success via one of the two branches, as the commands would exit if they failed. return 0 } @@ -637,7 +641,7 @@ resetRepo() { printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" # Return to where we came from popd &> /dev/null || return 1 - # Returning success anyway? + # Function succeeded, as "git reset" would have triggered a return earlier if it failed return 0 } @@ -689,7 +693,7 @@ welcomeDialogs() { In the next section, you can choose to use your current network settings (DHCP) or to manually edit them." "${r}" "${c}" } -# A function that let's the user pick an interface to use with Pi-hole +# A function that lets the user pick an interface to use with Pi-hole chooseInterface() { # Turn the available interfaces into an array so it can be used with a whiptail dialog local interfacesArray=() @@ -713,7 +717,7 @@ chooseInterface() { else # While reading through the available interfaces while read -r line; do - # use a variable to set the option as OFF to begin with + # Use a variable to set the option as OFF to begin with mode="OFF" # If it's the first loop, if [[ "${firstLoop}" -eq 1 ]]; then @@ -747,9 +751,9 @@ chooseInterface() { testIPv6() { # first will contain fda2 (ULA) printf -v first "%s" "${1%%:*}" - # value1 will contain 253 which is the decimal value corresponding to 0xfd + # value1 will contain 253 which is the decimal value corresponding to 0xFD value1=$(( (0x$first)/256 )) - # will contain 162 which is the decimal value corresponding to 0xa2 + # value2 will contain 162 which is the decimal value corresponding to 0xA2 value2=$(( (0x$first)%256 )) # the ULA test is testing for fc00::/7 according to RFC 4193 if (( (value1&254)==252 )); then @@ -779,8 +783,9 @@ useIPv6dialog() { result=$(testIPv6 "$i") # If it's a ULA address, use it and store it as a global variable [[ "${result}" == "ULA" ]] && ULA_ADDRESS="${i%/*}" - # If it's a GUA address, we can still use it si store it as a global variable + # If it's a GUA address, use it and store it as a global variable [[ "${result}" == "GUA" ]] && GUA_ADDRESS="${i%/*}" + # Else if it's a Link-local address, we cannot use it, so just continue done # Determine which address to be used: Prefer ULA over GUA or don't use any if none found @@ -858,7 +863,6 @@ use4andor6() { fi } -# getStaticIPv4Settings() { # Local, named variables local ipSettingsCorrect @@ -905,9 +909,9 @@ It is also possible to use a DHCP reservation, but if you are going to do that, fi } -# configure networking via dhcpcd +# Configure networking via dhcpcd setDHCPCD() { - # check if the IP is already in the file + # Check if the IP is already in the file if grep -q "${IPV4_ADDRESS}" /etc/dhcpcd.conf; then printf " %b Static IP already configured\\n" "${INFO}" # If it's not, @@ -925,8 +929,8 @@ setDHCPCD() { fi } -# configure networking ifcfg-xxxx file found at /etc/sysconfig/network-scripts/ -# this function requires the full path of an ifcfg file passed as an argument +# Configure networking ifcfg-xxxx file found at /etc/sysconfig/network-scripts/ +# This function requires the full path of an ifcfg file passed as an argument setIFCFG() { # Local, named variables local IFCFG_FILE @@ -934,12 +938,11 @@ setIFCFG() { local CIDR IFCFG_FILE=$1 printf -v IPADDR "%s" "${IPV4_ADDRESS%%/*}" - # check if the desired IP is already set + # Check if the desired IP is already set if grep -Eq "${IPADDR}(\\b|\\/)" "${IFCFG_FILE}"; then printf " %b Static IP already configured\\n" "${INFO}" - # Otherwise, else - # Put the IP in variables without the CIDR notation + # Otherwise, put the IP in variables without the CIDR notation printf -v CIDR "%s" "${IPV4_ADDRESS##*/}" # Backup existing interface configuration: cp -p "${IFCFG_FILE}" "${IFCFG_FILE}".pihole.orig @@ -981,24 +984,23 @@ setStaticIPv4() { return 0 fi fi - # For the Debian family, if dhcpcd.conf exists, + # For the Debian family, if dhcpcd.conf exists then we can just configure using DHCPD. if [[ -f "/etc/dhcpcd.conf" ]]; then - # configure networking via dhcpcd setDHCPCD return 0 fi - # If a DHCPCD config file was not found, check for an ifcfg config file based on interface name + # If a DHCPCD config file was not found, check for an ifcfg config file based on the interface name if [[ -f "/etc/sysconfig/network-scripts/ifcfg-${PIHOLE_INTERFACE}" ]];then - # If it exists, + # If it exists, then we can configure using IFCFG IFCFG_FILE=/etc/sysconfig/network-scripts/ifcfg-${PIHOLE_INTERFACE} setIFCFG "${IFCFG_FILE}" return 0 fi - # if an ifcfg config does not exists for the interface name, try the connection name via network manager + # If an ifcfg config does not exists for the interface name, search for one based on the connection name via network manager if is_command nmcli && nmcli general status &> /dev/null; then CONNECTION_NAME=$(nmcli dev show "${PIHOLE_INTERFACE}" | grep 'GENERAL.CONNECTION' | cut -d: -f2 | sed 's/^System//' | xargs | tr ' ' '_') if [[ -f "/etc/sysconfig/network-scripts/ifcfg-${CONNECTION_NAME}" ]];then - # If it exists, + # If it exists, then we can configure using IFCFG IFCFG_FILE=/etc/sysconfig/network-scripts/ifcfg-${CONNECTION_NAME} setIFCFG "${IFCFG_FILE}" return 0 @@ -1015,17 +1017,17 @@ valid_ip() { local ip=${1} local stat=1 - # One IPv4 element is 8bit: 0 - 256 + # Regex matching one IPv4 component, i.e. an integer from 0 to 255. local ipv4elem="(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]?|0)"; - # optional port number starting '#' with range of 1-65536 + # Regex matching an optional port beginning with # matching optional port number starting '#' with range of 1 to 65536 local portelem="(#([1-9]|[1-8][0-9]|9[0-9]|[1-8][0-9]{2}|9[0-8][0-9]|99[0-9]|[1-8][0-9]{3}|9[0-8][0-9]{2}|99[0-8][0-9]|999[0-9]|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-6]))?" - # build a full regex string from the above parts + # Build a full IPv4 regex from the above subexpressions local regex="^${ipv4elem}\.${ipv4elem}\.${ipv4elem}\.${ipv4elem}${portelem}$" + # Evaluate the regex, and return the result [[ $ip =~ ${regex} ]] stat=$? - # Return the exit code return "${stat}" } @@ -1033,19 +1035,19 @@ valid_ip6() { local ip=${1} local stat=1 - # One IPv6 element is 16bit: 0000 - FFFF + # Regex matching one IPv6 element, i.e. a hex value from 0000 to FFFF local ipv6elem="[0-9a-fA-F]{1,4}" - # CIDR for IPv6 is 1- 128 bit + # Regex matching an IPv6 CIDR, i.e. 1 to 128 local v6cidr="(\\/([1-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])){0,1}" - # optional port number starting '#' with range of 1-65536 + # Regex matching an optional port beginning with # matching optional port number starting '#' with range of 1-65536 local portelem="(#([1-9]|[1-8][0-9]|9[0-9]|[1-8][0-9]{2}|9[0-8][0-9]|99[0-9]|[1-8][0-9]{3}|9[0-8][0-9]{2}|99[0-8][0-9]|999[0-9]|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-6]))?" - # build a full regex string from the above parts + # Build a full IPv6 regex from the above subexpressions local regex="^(((${ipv6elem}))*((:${ipv6elem}))*::((${ipv6elem}))*((:${ipv6elem}))*|((${ipv6elem}))((:${ipv6elem})){7})${v6cidr}${portelem}$" + # Evaluate the regex, and return the result [[ ${ip} =~ ${regex} ]] stat=$? - # Return the exit code return "${stat}" } @@ -1057,7 +1059,7 @@ setDNS() { # In an array, list the available upstream providers DNSChooseOptions=() local DNSServerCount=0 - # Save the old Internal Field Separator in a variable + # Save the old Internal Field Separator in a variable, OIFS=$IFS # and set the new one to newline IFS=$'\n' @@ -1078,23 +1080,22 @@ setDNS() { # In a whiptail dialog, show the options DNSchoices=$(whiptail --separate-output --menu "Select Upstream DNS Provider. To use your own, select Custom." "${r}" "${c}" 7 \ "${DNSChooseOptions[@]}" 2>&1 >/dev/tty) || \ - # exit if Cancel is selected + # Exit if the user selects "Cancel" { printf " %bCancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; exit 1; } - # Depending on the user's choice, set the GLOBAl variables to the IP of the respective provider + # Depending on the user's choice, set the GLOBAL variables to the IP of the respective provider if [[ "${DNSchoices}" == "Custom" ]] then - # Until the DNS settings are selected, + # Loop until we have a valid DNS setting until [[ "${DNSSettingsCorrect}" = True ]]; do - # + # Signal value, to be used if the user inputs an invalid IP address strInvalid="Invalid" - # If the first if [[ ! "${PIHOLE_DNS_1}" ]]; then - # and second upstream servers do not exist if [[ ! "${PIHOLE_DNS_2}" ]]; then + # If the first and second upstream servers do not exist, do not prepopulate an IP address prePopulate="" - # Otherwise, else + # Otherwise, prepopulate the whiptail dialogue with the appropriate DNS value(s) prePopulate=", ${PIHOLE_DNS_2}" fi elif [[ "${PIHOLE_DNS_1}" ]] && [[ ! "${PIHOLE_DNS_2}" ]]; then @@ -1103,52 +1104,48 @@ setDNS() { prePopulate="${PIHOLE_DNS_1}, ${PIHOLE_DNS_2}" fi - # Dialog for the user to enter custom upstream servers + # Prompt the user to enter custom upstream servers piholeDNS=$(whiptail --backtitle "Specify Upstream DNS Provider(s)" --inputbox "Enter your desired upstream DNS provider(s), separated by a comma.\\n\\nFor example '8.8.8.8, 8.8.4.4'" "${r}" "${c}" "${prePopulate}" 3>&1 1>&2 2>&3) || \ { printf " %bCancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; exit 1; } # Clean user input and replace whitespace with comma. piholeDNS=$(sed 's/[, \t]\+/,/g' <<< "${piholeDNS}") + # Separate the user input into the two DNS values (separated by a comma) printf -v PIHOLE_DNS_1 "%s" "${piholeDNS%%,*}" printf -v PIHOLE_DNS_2 "%s" "${piholeDNS##*,}" - # If the IP is valid, + # If the first DNS value is invalid or empty, this if statement will be true and we will set PIHOLE_DNS_1="Invalid" if ! valid_ip "${PIHOLE_DNS_1}" || [[ ! "${PIHOLE_DNS_1}" ]]; then - # store it in the variable so we can use it PIHOLE_DNS_1=${strInvalid} fi - # Do the same for the secondary server + # If the second DNS value is invalid or empty, this if statement will be true and we will set PIHOLE_DNS_2="Invalid" if ! valid_ip "${PIHOLE_DNS_2}" && [[ "${PIHOLE_DNS_2}" ]]; then PIHOLE_DNS_2=${strInvalid} fi # If either of the DNS servers are invalid, if [[ "${PIHOLE_DNS_1}" == "${strInvalid}" ]] || [[ "${PIHOLE_DNS_2}" == "${strInvalid}" ]]; then - # explain this to the user + # explain this to the user, whiptail --msgbox --backtitle "Invalid IP" --title "Invalid IP" "One or both entered IP addresses were invalid. Please try again.\\n\\n DNS Server 1: $PIHOLE_DNS_1\\n DNS Server 2: ${PIHOLE_DNS_2}" ${r} ${c} - # and set the variables back to nothing + # set the variables back to nothing, if [[ "${PIHOLE_DNS_1}" == "${strInvalid}" ]]; then PIHOLE_DNS_1="" fi if [[ "${PIHOLE_DNS_2}" == "${strInvalid}" ]]; then PIHOLE_DNS_2="" fi - # Since the settings will not work, stay in the loop + # and continue the loop. DNSSettingsCorrect=False - # Otherwise, else - # Show the settings + # Otherwise, show the DNS setting to the user, and break the loop if they confirm them. if (whiptail --backtitle "Specify Upstream DNS Provider(s)" --title "Upstream DNS Provider(s)" --yesno "Are these settings correct?\\n DNS Server 1: $PIHOLE_DNS_1\\n DNS Server 2: ${PIHOLE_DNS_2}" "${r}" "${c}"); then - # and break from the loop since the servers are valid DNSSettingsCorrect=True - # Otherwise, else - # If the settings are wrong, the loop continues DNSSettingsCorrect=False fi fi done else - # Save the old Internal Field Separator in a variable + # Save the old Internal Field Separator in a variable, OIFS=$IFS # and set the new one to newline IFS=$'\n' @@ -1187,16 +1184,16 @@ setLogging() { # Get the user's choice LogChoices=$("${LogToggleCommand[@]}" "${LogChooseOptions[@]}" 2>&1 >/dev/tty) || (printf " %bCancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}" && exit 1) case ${LogChoices} in - # If it's on + # If it's on, "On (Recommended)") printf " %b Logging On.\\n" "${INFO}" - # Set the GLOBAL variable to true so we know what they selected + # set the GLOBAL variable setting to true QUERY_LOGGING=true ;; # Otherwise, it's off, Off) printf " %b Logging Off.\\n" "${INFO}" - # So set it to false + # set the GLOBAL variable setting to false QUERY_LOGGING=false ;; esac @@ -1252,7 +1249,7 @@ setAdminFlag() { ;; esac - # Request user to install web server, if it has not been deselected before (INSTALL_WEB_SERVER=true is default). + # If the user wants to install the Web admin interface (i.e. it has not been deselected above) if [[ "${INSTALL_WEB_SERVER}" == true ]]; then # Get list of required PHP modules, excluding base package (common) and handler (cgi) local i php_modules @@ -1291,18 +1288,18 @@ chooseBlocklists() { # In a variable, show the choices available; exit if Cancel is selected choices=$("${cmd[@]}" "${options[@]}" 2>&1 >/dev/tty) || { printf " %bCancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; rm "${adlistFile}" ;exit 1; } - # For each choice available, + # Add all selected choices to the lists file for choice in ${choices} do appendToListsFile "${choice}" done + # Create an empty adList file with appropriate permissions. touch "${adlistFile}" chmod 644 "${adlistFile}" } # Accept a string parameter, it must be one of the default lists -# This function allow to not duplicate code in chooseBlocklists and -# in installDefaultBlocklists +# This function saves duplication between chooseBlocklists and installDefaultBlocklists appendToListsFile() { case $1 in StevenBlack ) echo "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" >> "${adlistFile}";; @@ -1339,16 +1336,15 @@ version_check_dnsmasq() { grep -q "${dnsmasq_pihole_id_string2}" "${dnsmasq_conf}"; then printf " it is from a previous Pi-hole install.\\n" printf " %b Backing up dnsmasq.conf to dnsmasq.conf.orig..." "${INFO}" - # so backup the original file + # so backup the original file, mv -f "${dnsmasq_conf}" "${dnsmasq_conf_orig}" printf "%b %b Backing up dnsmasq.conf to dnsmasq.conf.orig...\\n" "${OVER}" "${TICK}" printf " %b Restoring default dnsmasq.conf..." "${INFO}" # and replace it with the default install -D -m 644 -T "${dnsmasq_original_config}" "${dnsmasq_conf}" printf "%b %b Restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" - # Otherwise, else - # Don't to anything + # Otherwise, don't to anything printf " it is not a Pi-hole file, leaving alone!\\n" fi else @@ -1368,36 +1364,33 @@ version_check_dnsmasq() { install -D -m 644 -T "${dnsmasq_pihole_01_snippet}" "${dnsmasq_pihole_01_location}" printf "%b %b Copying 01-pihole.conf to /etc/dnsmasq.d/01-pihole.conf\\n" "${OVER}" "${TICK}" # Replace our placeholder values with the GLOBAL DNS variables that we populated earlier - # First, swap in the interface to listen on + # First, swap in the interface to listen on, sed -i "s/@INT@/$PIHOLE_INTERFACE/" "${dnsmasq_pihole_01_location}" if [[ "${PIHOLE_DNS_1}" != "" ]]; then - # Then swap in the primary DNS server + # then swap in the primary DNS server. sed -i "s/@DNS1@/$PIHOLE_DNS_1/" "${dnsmasq_pihole_01_location}" else - # + # Otherwise, remove the line which sets DNS1. sed -i '/^server=@DNS1@/d' "${dnsmasq_pihole_01_location}" fi + # Ditto if DNS2 is not empty if [[ "${PIHOLE_DNS_2}" != "" ]]; then - # Then swap in the primary DNS server sed -i "s/@DNS2@/$PIHOLE_DNS_2/" "${dnsmasq_pihole_01_location}" else - # sed -i '/^server=@DNS2@/d' "${dnsmasq_pihole_01_location}" fi # Set the cache size sed -i "s/@CACHE_SIZE@/$CACHE_SIZE/" ${dnsmasq_pihole_01_location} - # sed -i 's/^#conf-dir=\/etc\/dnsmasq.d$/conf-dir=\/etc\/dnsmasq.d/' "${dnsmasq_conf}" # If the user does not want to enable logging, if [[ "${QUERY_LOGGING}" == false ]] ; then - # Disable it by commenting out the directive in the DNS config file + # disable it by commenting out the directive in the DNS config file sed -i 's/^log-queries/#log-queries/' "${dnsmasq_pihole_01_location}" - # Otherwise, else - # enable it by uncommenting the directive in the DNS config file + # Otherwise, enable it by uncommenting the directive in the DNS config file sed -i 's/^#log-queries/log-queries/' "${dnsmasq_pihole_01_location}" fi } @@ -1407,14 +1400,13 @@ clean_existing() { # Local, named variables # ${1} Directory to clean local clean_directory="${1}" - # Make ${2} the new one? + # Pop the first argument, and shift all addresses down by one (i.e. ${2} becomes ${1}) shift - # ${2} Array of files to remove + # Then, we can access all arguments ($@) without including the directory to clean local old_files=( "$@" ) - # For each script found in the old files array + # Remove each script in the old_files array for script in "${old_files[@]}"; do - # Remove them rm -f "${clean_directory}/${script}.sh" done } @@ -1447,9 +1439,8 @@ installScripts() { install -Dm644 ./advanced/bash-completion/pihole /etc/bash_completion.d/pihole printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" - # Otherwise, else - # Show an error and exit + # Otherwise, show an error and exit printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" printf "\\t\\t%bError: Local repo %s not found, exiting installer%b\\n" "${COL_LIGHT_RED}" "${PI_HOLE_LOCAL_REPO}" "${COL_NC}" return 1 @@ -1501,7 +1492,7 @@ installConfigs() { # Make sure the external.conf file exists, as lighttpd v1.4.50 crashes without it touch /etc/lighttpd/external.conf chmod 644 /etc/lighttpd/external.conf - # if there is a custom block page in the html/pihole directory, replace 404 handler in lighttpd config + # If there is a custom block page in the html/pihole directory, replace 404 handler in lighttpd config if [[ -f "${PI_HOLE_BLOCKPAGE_DIR}/custom.php" ]]; then sed -i 's/^\(server\.error-handler-404\s*=\s*\).*$/\1"pihole\/custom\.php"/' /etc/lighttpd/lighttpd.conf fi @@ -1575,9 +1566,8 @@ restart_service() { if is_command systemctl ; then # use that to restart the service systemctl restart "${1}" &> /dev/null - # Otherwise, else - # fall back to the service command + # Otherwise, fall back to the service command service "${1}" restart &> /dev/null fi printf "%b %b %s...\\n" "${OVER}" "${TICK}" "${str}" @@ -1592,9 +1582,8 @@ enable_service() { if is_command systemctl ; then # use that to enable the service systemctl enable "${1}" &> /dev/null - # Otherwise, else - # use update-rc.d to accomplish this + # Otherwise, use update-rc.d to accomplish this update-rc.d "${1}" defaults &> /dev/null fi printf "%b %b %s...\\n" "${OVER}" "${TICK}" "${str}" @@ -1609,9 +1598,8 @@ disable_service() { if is_command systemctl ; then # use that to disable the service systemctl disable "${1}" &> /dev/null - # Otherwise, else - # use update-rc.d to accomplish this + # Otherwise, use update-rc.d to accomplish this update-rc.d "${1}" disable &> /dev/null fi printf "%b %b %s...\\n" "${OVER}" "${TICK}" "${str}" @@ -1622,9 +1610,8 @@ check_service_active() { if is_command systemctl ; then # use that to check the status of the service systemctl is-enabled "${1}" &> /dev/null - # Otherwise, else - # fall back to service command + # Otherwise, fall back to service command service "${1}" status &> /dev/null fi } @@ -1666,9 +1653,8 @@ update_package_cache() { # Create a command from the package cache variable if eval "${UPDATE_PKG_CACHE}" &> /dev/null; then printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" - # Otherwise, else - # show an error and exit + # Otherwise, show an error and exit printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" printf " %bError: Unable to update package cache. Please try \"%s\"%b" "${COL_LIGHT_RED}" "${UPDATE_PKG_CACHE}" "${COL_NC}" return 1 @@ -1697,19 +1683,18 @@ notify_package_updates_available() { fi } -# What's this doing outside of a function in the middle of nowhere? +# This counter is outside of install_dependent_packages so that it can count the number of times the function is called. counter=0 install_dependent_packages() { # Local, named variables should be used here, especially for an iterator # Add one to the counter counter=$((counter+1)) - # If it equals 1, if [[ "${counter}" == 1 ]]; then - # + # On the first loop, print a special message printf " %b Installer Dependency checks...\\n" "${INFO}" else - # + # On all subsequent loops, print a generic message. printf " %b Main Dependency checks...\\n" "${INFO}" fi @@ -1723,7 +1708,7 @@ install_dependent_packages() { # NOTE: We may be able to use this installArray in the future to create a list of package that were # installed by us, and remove only the installed packages, and not the entire list. if is_command apt-get ; then - # For each package, + # For each package, check if it's already installed (and if so, don't add it to the installArray) for i in "$@"; do printf " %b Checking for %s..." "${INFO}" "${i}" if dpkg-query -W -f='${Status}' "${i}" 2>/dev/null | grep "ok installed" &> /dev/null; then @@ -1733,6 +1718,7 @@ install_dependent_packages() { installArray+=("${i}") fi done + # If there's anything to install, install everything in the list. if [[ "${#installArray[@]}" -gt 0 ]]; then test_dpkg_lock printf " %b Processing %s install(s) for: %s, please wait...\\n" "${INFO}" "${PKG_MANAGER}" "${installArray[*]}" @@ -1747,6 +1733,7 @@ install_dependent_packages() { # Install Fedora/CentOS packages for i in "$@"; do + # For each package, check if it's already installed (and if so, don't add it to the installArray) printf " %b Checking for %s..." "${INFO}" "${i}" if "${PKG_MANAGER}" -q list installed "${i}" &> /dev/null; then printf "%b %b Checking for %s\\n" "${OVER}" "${TICK}" "${i}" @@ -1755,6 +1742,7 @@ install_dependent_packages() { installArray+=("${i}") fi done + # If there's anything to install, install everything in the list. if [[ "${#installArray[@]}" -gt 0 ]]; then printf " %b Processing %s install(s) for: %s, please wait...\\n" "${INFO}" "${PKG_MANAGER}" "${installArray[*]}" printf '%*s\n' "$columns" '' | tr " " -; @@ -1772,7 +1760,7 @@ installPiholeWeb() { local str="Creating directory for blocking page, and copying files" printf " %b %s..." "${INFO}" "${str}" - # Install the directory + # Install the directory, install -d -m 0755 ${PI_HOLE_BLOCKPAGE_DIR} # and the blockpage install -D -m 644 ${PI_HOLE_LOCAL_REPO}/advanced/{index,blockingpage}.* ${PI_HOLE_BLOCKPAGE_DIR}/ @@ -1791,9 +1779,8 @@ installPiholeWeb() { # back it up mv ${webroot}/index.lighttpd.html ${webroot}/index.lighttpd.orig printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" - # Otherwise, else - # don't do anything + # Otherwise, don't do anything printf "%b %b %s\\n" "${OVER}" "${INFO}" "${str}" printf " No default index.lighttpd.html file found... not backing up\\n" fi @@ -1801,7 +1788,7 @@ installPiholeWeb() { # Install Sudoers file local str="Installing sudoer file" printf "\\n %b %s..." "${INFO}" "${str}" - # Make the .d directory if it doesn't exist + # Make the .d directory if it doesn't exist, install -d -m 755 /etc/sudoers.d/ # and copy in the pihole sudoers file install -m 0640 ${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole.sudo /etc/sudoers.d/pihole @@ -1845,11 +1832,11 @@ runGravity() { create_pihole_user() { local str="Checking for user 'pihole'" printf " %b %s..." "${INFO}" "${str}" - # If the user pihole exists, + # If the pihole user exists, if id -u pihole &> /dev/null; then - # if group exists + # and if the pihole group exists, if getent group pihole > /dev/null 2>&1; then - # just show a success + # succeed printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" else local str="Checking for group 'pihole'" @@ -1870,14 +1857,14 @@ create_pihole_user() { printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" fi fi - # Otherwise, else + # If the pihole user doesn't exist, printf "%b %b %s" "${OVER}" "${CROSS}" "${str}" local str="Creating user 'pihole'" printf "%b %b %s..." "${OVER}" "${INFO}" "${str}" - # create her with the useradd command + # create her with the useradd command, if getent group pihole > /dev/null 2>&1; then - # add primary group pihole as it already exists + # then add her to the pihole group (as it already exists) if useradd -r --no-user-group -g pihole -s /usr/sbin/nologin pihole; then printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" else @@ -1894,7 +1881,7 @@ create_pihole_user() { fi } -# +# This function saves any changes to the setup variables into the setupvars.conf file for future runs finalExports() { # If the Web interface is not set to be installed, if [[ "${INSTALL_WEB_INTERFACE}" == false ]]; then @@ -1946,7 +1933,6 @@ finalExports() { # Install the logrotate script installLogrotate() { - local str="Installing latest logrotate script" printf "\\n %b %s..." "${INFO}" "${str}" # Copy the file over from the local repo @@ -1958,9 +1944,9 @@ installLogrotate() { # customize the logrotate script here in order to reflect # the local properties of the /var/log directory logusergroup="$(stat -c '%U %G' /var/log)" - # If the variable has a value, + # If there is a usergroup for log rotation, if [[ ! -z "${logusergroup}" ]]; then - # + # replace the line in the logrotate script with that usergroup. sed -i "s/# su #/su ${logusergroup}/g;" /etc/pihole/logrotate fi printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" @@ -2099,15 +2085,15 @@ checkSelinux() { # Installation complete message with instructions for the user displayFinalMessage() { - # If + # If the number of arguments is > 0, if [[ "${#1}" -gt 0 ]] ; then + # set the password to the first argument. pwstring="$1" - # else, if the dashboard password in the setup variables exists, elif [[ $(grep 'WEBPASSWORD' -c /etc/pihole/setupVars.conf) -gt 0 ]]; then - # set a variable for evaluation later + # Else if the password exists from previous setup, we'll load it later pwstring="unchanged" else - # set a variable for evaluation later + # Else, inform the user that there is no set password. pwstring="NOT SET" fi # If the user wants to install the dashboard, @@ -2138,9 +2124,8 @@ update_dialogs() { opt1a="Repair" opt1b="This will retain existing settings" strAdd="You will remain on the same version" - # Otherwise, else - # set some variables with different values + # Otherwise, set some variables with different values opt1a="Update" opt1b="This will retain existing settings." strAdd="You will be updated to the latest version." @@ -2291,7 +2276,6 @@ clone_or_update_repos() { # Disable directive for SC2120 a value _can_ be passed to this function, but it is passed from an external script that sources this one # shellcheck disable=SC2120 FTLinstall() { - # Local, named variables local latesttag local str="Downloading and Installing FTL" @@ -2322,9 +2306,8 @@ FTLinstall() { url="https://ftl.pi-hole.net/${ftlBranch}" fi - # If the download worked, if curl -sSL --fail "${url}/${binary}" -o "${binary}"; then - # get sha1 of the binary we just downloaded for verification. + # If the download worked, get sha1 of the binary we just downloaded for verification. curl -sSL --fail "${url}/${binary}.sha1" -o "${binary}.sha1" # If we downloaded binary file (as opposed to text), @@ -2348,16 +2331,15 @@ FTLinstall() { # Installed the FTL service printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" return 0 - # Otherwise, else - # the download failed, so just go back to the original directory + # Otherwise, the hash download failed, so print and exit. popd > /dev/null || { printf "Unable to return to original directory after FTL binary download.\\n"; return 1; } printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" printf " %bError: Download of %s/%s failed (checksum error)%b\\n" "${COL_LIGHT_RED}" "${url}" "${binary}" "${COL_NC}" return 1 fi - # Otherwise, else + # Otherwise, the download failed, so print and exit. popd > /dev/null || { printf "Unable to return to original directory after FTL binary download.\\n"; return 1; } printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" # The URL could not be found @@ -2400,18 +2382,14 @@ get_binary_name() { # If the machine is arm or aarch if [[ "${machine}" == "arm"* || "${machine}" == *"aarch"* ]]; then # ARM - # local rev rev=$(uname -m | sed "s/[^0-9]//g;") - # local lib lib=$(ldd /bin/ls | grep -E '^\s*/lib' | awk '{ print $1 }') - # if [[ "${lib}" == "/lib/ld-linux-aarch64.so.1" ]]; then printf "%b %b Detected AArch64 (64 Bit ARM) processor\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-aarch64-linux-gnu" - # elif [[ "${lib}" == "/lib/ld-linux-armhf.so.3" ]]; then # Hard-float available: Use gnueabihf binaries # If ARMv8 or higher is found (e.g., BCM2837 as found in Raspberry Pi Model 3B) @@ -2419,13 +2397,13 @@ get_binary_name() { printf "%b %b Detected ARMv8 (or newer) processor\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-armv8-linux-gnueabihf" - # Otherwise, if ARMv7 is found (e.g., BCM2836 as found in Raspberry Pi Model 2) elif [[ "${rev}" -eq 7 ]]; then + # Otherwise, if ARMv7 is found (e.g., BCM2836 as found in Raspberry Pi Model 2) printf "%b %b Detected ARMv7 processor (with hard-float support)\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-armv7-linux-gnueabihf" - # Otherwise, use the ARMv6 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) else + # Otherwise, use the ARMv6 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) printf "%b %b Detected ARMv6 processor (with hard-float support)\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-armv6-linux-gnueabihf" @@ -2475,6 +2453,7 @@ get_binary_name() { l_binary="pihole-FTL-linux-x86_32" fi + # Returning a string value via echo echo ${l_binary} } @@ -2611,9 +2590,8 @@ main() { # Show the Pi-hole logo so people know it's genuine since the logo and name are trademarked show_ascii_berry make_temporary_log - # Otherwise, else - # They do not have enough privileges, so let the user know + # Otherwise, they do not have enough privileges, so let the user know printf " %b %s\\n" "${INFO}" "${str}" printf " %b %bScript called with non-root privileges%b\\n" "${INFO}" "${COL_LIGHT_RED}" "${COL_NC}" printf " The Pi-hole requires elevated privileges to install and run\\n" @@ -2621,7 +2599,7 @@ main() { printf " Make sure to download this script from a trusted source\\n\\n" printf " %b Sudo utility check" "${INFO}" - # If the sudo command exists, + # If the sudo command exists, try rerunning as admin if is_command sudo ; then printf "%b %b Sudo utility check\\n" "${OVER}" "${TICK}" @@ -2635,9 +2613,8 @@ main() { fi exit $? - # Otherwise, else - # Let them know they need to run it as root + # Otherwise, tell the user they need to run the script as root, and bail printf "%b %b Sudo utility check\\n" "${OVER}" "${CROSS}" printf " %b Sudo is needed for the Web Interface to run pihole commands\\n\\n" "${INFO}" printf " %b %bPlease re-run this installer as root${COL_NC}\\n" "${INFO}" "${COL_LIGHT_RED}" @@ -2657,9 +2634,8 @@ main() { useUpdateVars=true # also disable debconf-apt-progress dialogs export DEBIAN_FRONTEND="noninteractive" - # Otherwise, else - # show the available options (repair/reconfigure) + # If running attended, show the available options (repair/reconfigure) update_dialogs fi fi @@ -2713,13 +2689,13 @@ main() { PRIVACY_LEVEL="${PRIVACY_LEVEL:-0}" fi fi - # Clone/Update the repos + # Download or update the scripts by updating the appropriate git repos clone_or_update_repos # Install the Core dependencies local dep_install_list=("${PIHOLE_DEPS[@]}") if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - # Install the Web dependencies + # And, if the setting says so, install the Web admin interface dependencies dep_install_list+=("${PIHOLE_WEB_DEPS[@]}") fi @@ -2727,7 +2703,7 @@ main() { unset dep_install_list # On some systems, lighttpd is not enabled on first install. We need to enable it here if the user - # has chosen to install the web interface, else the `LIGHTTPD_ENABLED` check will fail + # has chosen to install the web interface, else the LIGHTTPD_ENABLED check will fail if [[ "${INSTALL_WEB_SERVER}" == true ]]; then enable_service lighttpd fi @@ -2779,7 +2755,6 @@ main() { # If the Web server was installed, if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - if [[ "${LIGHTTPD_ENABLED}" == true ]]; then restart_service lighttpd enable_service lighttpd From d25240fe9fff05ce633c8029ac74f953c511824e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 18 Mar 2021 08:57:03 +0100 Subject: [PATCH 15/21] pihole -g should respose GRAVITYDB in pihole-FTL.conf Signed-off-by: DL6ER --- gravity.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index abed28b4..d07d558d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -35,8 +35,9 @@ localList="${piholeDir}/local.list" VPNList="/etc/openvpn/ipp.txt" piholeGitDir="/etc/.pihole" -gravityDBfile="${piholeDir}/gravity.db" -gravityTEMPfile="${piholeDir}/gravity_temp.db" +gravityDBfile_default="${piholeDir}/gravity.db" +# GRAVITYDB may be overwritten by source pihole-FTL.conf below +GRAVITYDB="${gravityDBfile_default}" gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql" gravityDBcopy="${piholeGitDir}/advanced/Templates/gravity_copy.sql" @@ -68,6 +69,11 @@ if [[ -f "${pihole_FTL}" ]]; then source "${pihole_FTL}" fi +# Set this only after sourcing pihole-FTL.conf as the gravity database path may +# have changed +gravityDBfile="${GRAVITYDB}" +gravityTEMPfile="${GRAVITYDB}_temp" + if [[ -z "${BLOCKINGMODE}" ]] ; then BLOCKINGMODE="NULL" fi @@ -359,6 +365,10 @@ gravity_CheckDNSResolutionAvailable() { gravity_DownloadBlocklists() { echo -e " ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..." + if [[ "${gravityDBfile}" != "${gravityDBfile_default}" ]]; then + echo -e " ${INFO} Storing gravity database in ${COL_BOLD}${gravityDBfile}${COL_NC}" + fi + # Retrieve source URLs from gravity database # We source only enabled adlists, sqlite3 stores boolean values as 0 (false) or 1 (true) mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" From 9bdbe88a661aaeed9d775b7d266032005ff388f3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 18 Mar 2021 09:01:22 +0100 Subject: [PATCH 16/21] Update gravity path in list.sh Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index fde46552..975738b1 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -11,7 +11,16 @@ # Globals basename=pihole piholeDir=/etc/"${basename}" -gravityDBfile="${piholeDir}/gravity.db" +GRAVITYDB="${piholeDir}/gravity.db" +# Source pihole-FTL from install script +pihole_FTL="${piholeDir}/pihole-FTL.conf" +if [[ -f "${pihole_FTL}" ]]; then + source "${pihole_FTL}" +fi + +# Set this only after sourcing pihole-FTL.conf as the gravity database path may +# have changed +gravityDBfile="${GRAVITYDB}" reload=false addmode=true From f1740da9ffcbabc58b56107d2e1131aae380dd90 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 18 Mar 2021 09:02:20 +0100 Subject: [PATCH 17/21] Update gravity path in query.sh Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 3 +-- advanced/Scripts/query.sh | 11 ++++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 975738b1..062c52be 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -9,8 +9,7 @@ # Please see LICENSE file for your rights under this license. # Globals -basename=pihole -piholeDir=/etc/"${basename}" +piholeDir="/etc/pihole" GRAVITYDB="${piholeDir}/gravity.db" # Source pihole-FTL from install script pihole_FTL="${piholeDir}/pihole-FTL.conf" diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index fe9b8ebf..0872f060 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -11,12 +11,21 @@ # Globals piholeDir="/etc/pihole" -gravityDBfile="${piholeDir}/gravity.db" +GRAVITYDB="${piholeDir}/gravity.db" options="$*" all="" exact="" blockpage="" matchType="match" +# Source pihole-FTL from install script +pihole_FTL="${piholeDir}/pihole-FTL.conf" +if [[ -f "${pihole_FTL}" ]]; then + source "${pihole_FTL}" +fi + +# Set this only after sourcing pihole-FTL.conf as the gravity database path may +# have changed +gravityDBfile="${GRAVITYDB}" colfile="/opt/pihole/COL_TABLE" source "${colfile}" From f536718aaab7fd4ed1d22a9b0e5f3357cc1f9efb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 18 Mar 2021 09:04:09 +0100 Subject: [PATCH 18/21] Use possibly custom gravity.db location in the copying SQL script Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 2 ++ advanced/Scripts/query.sh | 1 + gravity.sh | 11 +++++++++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 062c52be..e213b014 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# shellcheck disable=SC1090 + # Pi-hole: A black hole for Internet advertisements # (c) 2017 Pi-hole, LLC (https://pi-hole.net) # Network-wide ad blocking via your own hardware. diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 0872f060..26b4508e 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash # shellcheck disable=SC1090 + # Pi-hole: A black hole for Internet advertisements # (c) 2018 Pi-hole, LLC (https://pi-hole.net) # Network-wide ad blocking via your own hardware. diff --git a/gravity.sh b/gravity.sh index d07d558d..24a41c48 100755 --- a/gravity.sh +++ b/gravity.sh @@ -90,7 +90,7 @@ generate_gravity_database() { # Copy data from old to new database file and swap them gravity_swap_databases() { - local str + local str copyGravity str="Building tree" echo -ne " ${INFO} ${str}..." @@ -107,7 +107,14 @@ gravity_swap_databases() { str="Swapping databases" echo -ne " ${INFO} ${str}..." - output=$( { sqlite3 "${gravityTEMPfile}" < "${gravityDBcopy}"; } 2>&1 ) + # Gravity copying SQL script + copyGravity="$(cat "${gravityDBcopy}")" + if [[ "${gravityDBfile}" != "${gravityDBfile_default}" ]]; then + # Replace default gravity script location by custom location + copyGravity="${copyGravity//"${gravityDBfile_default}"/"${gravityDBfile}"}" + fi + + output=$( { sqlite3 "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then From 89c80947df964b6853ce22dba1f20e246bdcf48d Mon Sep 17 00:00:00 2001 From: Joseph Blackman Date: Wed, 17 Mar 2021 03:09:13 -0700 Subject: [PATCH 19/21] Improve regexes for ipv4 matching Signed-off-by: jbzdarkid --- automated install/basic-install.sh | 4 +-- test/test_automated_install.py | 58 ++++++++++++++---------------- 2 files changed, 28 insertions(+), 34 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 762ee110..4c9352b8 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1018,9 +1018,9 @@ valid_ip() { local stat=1 # Regex matching one IPv4 component, i.e. an integer from 0 to 255. - local ipv4elem="(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]?|0)"; + local ipv4elem="(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9]{1,2})"; # Regex matching an optional port beginning with # matching optional port number starting '#' with range of 1 to 65536 - local portelem="(#([1-9]|[1-8][0-9]|9[0-9]|[1-8][0-9]{2}|9[0-8][0-9]|99[0-9]|[1-8][0-9]{3}|9[0-8][0-9]{2}|99[0-8][0-9]|999[0-9]|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-6]))?" + local portelem="(:(6553[0-5]|655[0-2][0-9]|65[0-4][0-9]{2}|6[0-4][0-9]{3}|[1-5][0-9]{4}|[0-9]{1,4}))?"; # Build a full IPv4 regex from the above subexpressions local regex="^${ipv4elem}\.${ipv4elem}\.${ipv4elem}\.${ipv4elem}${portelem}$" diff --git a/test/test_automated_install.py b/test/test_automated_install.py index f6b5a87e..20dadc26 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -524,43 +524,37 @@ def test_IPv6_ULA_GUA_test(Pihole): assert expected_stdout in detectPlatform.stdout -def test_validate_ip_valid(Pihole): +def test_validate_ip(Pihole): ''' - Given a valid IP address, valid_ip returns success + Tests valid_ip for various IP addresses ''' - output = Pihole.run(''' - source /opt/pihole/basic-install.sh - valid_ip "192.168.1.1" - ''') + def test_address(addr, success=True): + output = Pihole.run(''' + source /opt/pihole/basic-install.sh + valid_ip "{addr}" + '''.format(addr=addr)) - assert output.rc == 0 + assert output.rc == 0 if success else 1 - -def test_validate_ip_invalid_octet(Pihole): - ''' - Given an invalid IP address (large octet), valid_ip returns an error - ''' - - output = Pihole.run(''' - source /opt/pihole/basic-install.sh - valid_ip "1092.168.1.1" - ''') - - assert output.rc == 1 - - -def test_validate_ip_invalid_letters(Pihole): - ''' - Given an invalid IP address (contains letters), valid_ip returns an error - ''' - - output = Pihole.run(''' - source /opt/pihole/basic-install.sh - valid_ip "not an IP" - ''') - - assert output.rc == 1 + test_address('192.168.1.1') + test_address('127.0.0.1') + test_address('255.255.255.255') + test_address('255.255.255.256', False) + test_address('255.255.256.255', False) + test_address('255.256.255.255', False) + test_address('256.255.255.255', False) + test_address('1092.168.1.1', False) + test_address('not an IP', False) + test_address('8.8.8.8:', False) + test_address('8.8.8.8:0') + test_address('8.8.8.8:1') + test_address('8.8.8.8:42') + test_address('8.8.8.8:888') + test_address('8.8.8.8:1337') + test_address('8.8.8.8:65535') + test_address('8.8.8.8:65536', False) + test_address('8.8.8.8:-1', False) def test_os_check_fails(Pihole): From 8090071effa7534ed6c985e8ebee666a4aae1f6d Mon Sep 17 00:00:00 2001 From: jbzdarkid Date: Wed, 17 Mar 2021 18:09:03 -0700 Subject: [PATCH 20/21] Update regex to disallow leading zeros Also updated a comment to point to a non-experimental RFC. Signed-off-by: jbzdarkid --- automated install/basic-install.sh | 8 +++++--- test/test_automated_install.py | 8 ++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4c9352b8..78bc0882 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1018,9 +1018,11 @@ valid_ip() { local stat=1 # Regex matching one IPv4 component, i.e. an integer from 0 to 255. - local ipv4elem="(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9]{1,2})"; - # Regex matching an optional port beginning with # matching optional port number starting '#' with range of 1 to 65536 - local portelem="(:(6553[0-5]|655[0-2][0-9]|65[0-4][0-9]{2}|6[0-4][0-9]{3}|[1-5][0-9]{4}|[0-9]{1,4}))?"; + # See https://tools.ietf.org/html/rfc1340 + local ipv4elem="(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]?|0)"; + # Regex matching an optional port beginning with : from 0 to 65535 + # See https://tools.ietf.org/html/rfc1340#page-33 + local portelem="(:(6553[0-5]|655[0-2][0-9]|65[0-4][0-9]{2}|6[0-4][0-9]{3}|[1-5][0-9]{4}|[1-9][0-9]{0,3}|0))?"; # Build a full IPv4 regex from the above subexpressions local regex="^${ipv4elem}\.${ipv4elem}\.${ipv4elem}\.${ipv4elem}${portelem}$" diff --git a/test/test_automated_install.py b/test/test_automated_install.py index 20dadc26..1266f46b 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -555,6 +555,14 @@ def test_validate_ip(Pihole): test_address('8.8.8.8:65535') test_address('8.8.8.8:65536', False) test_address('8.8.8.8:-1', False) + test_address('00.0.0.0', False) + test_address('010.0.0.0', False) + test_address('001.0.0.0', False) + test_address('0.0.0.0:00', False) + test_address('0.0.0.0:01', False) + test_address('0.0.0.0:001', False) + test_address('0.0.0.0:0001', False) + test_address('0.0.0.0:00001', False) def test_os_check_fails(Pihole): From 6198165df8d3231187dc7301bcafb2cf0655565f Mon Sep 17 00:00:00 2001 From: Jaime Baez Date: Fri, 26 Mar 2021 20:24:38 +0100 Subject: [PATCH 21/21] Fix error trying to access undefined variables on splash page. Signed-off-by: Jaime Baez --- advanced/index.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/index.php b/advanced/index.php index eea44462..8f33cf5d 100644 --- a/advanced/index.php +++ b/advanced/index.php @@ -58,20 +58,20 @@ if ($serverName === "pi.hole" // When directly browsing via IP or authorized hostname // Render splash/landing page based off presence of $landPage file // Unset variables so as to not be included in $landPage or $splashPage - unset($serverName, $svPasswd, $svEmail, $authorizedHosts, $validExtTypes, $currentUrlExt, $viewPort); + unset($svPasswd, $svEmail, $authorizedHosts, $validExtTypes, $currentUrlExt); // If $landPage file is present if (is_file(getcwd()."/$landPage")) { + unset($serverName, $viewPort); // unset extra variables not to be included in $landpage include $landPage; exit(); } // If $landPage file was not present, Set Splash Page output - $splashPage = " + $splashPage = << $viewPort - ● $serverName @@ -84,7 +84,7 @@ if ($serverName === "pi.hole" - "; +EOT; exit($splashPage); } elseif ($currentUrlExt === "js") { // Serve Pi-hole JavaScript for blocked domains requesting JS