Merge pull request #4405 from pi-hole/development

Pi-hole v5.6

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+- package-ecosystem: github-actions
+  directory: "/"
+  schedule:
+    interval: weekly
+    day: saturday
+    time: "10:00"
+  open-pull-requests-limit: 10
+  target-branch: developement

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,40 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches:
+      - master
+      - development
+  pull_request:
+    branches:
+      - master
+      - development
+  schedule:
+    - cron: '32 11 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    steps:
+    -
+      name: Checkout repository
+      uses: actions/checkout@v2
+    # Initializes the CodeQL tools for scanning.
+    -
+      name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: 'python'
+    -
+      name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+    -
+      name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/test.yml

@@ -5,21 +5,44 @@ on:
     types: [opened, synchronize, reopened, ready_for_review]
 
 jobs:
+  smoke-test:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    steps:
+    -
+      name: Checkout repository
+      uses: actions/checkout@v2
+    -
+      name: Run Smoke Tests
+      run: |
+        # Ensure scripts in repository are executable
+        IFS=$'\n';
+        for f in $(find . -name '*.sh'); do if [[ ! -x $f ]]; then echo "$f is not executable" && FAIL=1; fi ;done
+        unset IFS;
+        # If FAIL is 1 then we fail.
+        [[ $FAIL == 1 ]] && exit 1 || echo "Smoke Tests Passed"
+
   distro-test:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
+    needs: smoke-test
     strategy:
       matrix:
-        distro: [debian_9, debian_10, debian_11, ubuntu_16, ubuntu_18, ubuntu_20, ubuntu_21, centos_7, centos_8, fedora_32, fedora_33]
+        distro: [debian_9, debian_10, debian_11, ubuntu_16, ubuntu_18, ubuntu_20, ubuntu_21, centos_7, centos_8, fedora_33, fedora_34]
     env:
       DISTRO: ${{matrix.distro}}
     steps:
-    - uses: actions/checkout@v1
+    -
-    - name: Set up Python 3.7
+      name: Checkout repository
+      uses: actions/checkout@v2
+    -
+      name: Set up Python 3.7
       uses: actions/setup-python@v2
       with:
         python-version: 3.7
-    - name: Install dependencies
+    -
+      name: Install dependencies
       run: pip install -r test/requirements.txt
-    - name: Test with tox
+    -
+      name: Test with tox
       run: tox -c test/tox.${DISTRO}.ini

.gitignore

@@ -7,70 +7,5 @@ __pycache__
 .tox
 .eggs
 *.egg-info
-
+.idea/
-
-# Created by https://www.gitignore.io/api/jetbrains+iml
-
-### JetBrains+iml ###
-# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
-# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
-
-# All idea files, with exceptions
-.idea
-!.idea/codeStyles/*
-!.idea/codeStyleSettings.xml
-
-
-# Sensitive or high-churn files:
-.idea/**/dataSources/
-.idea/**/dataSources.ids
-.idea/**/dataSources.xml
-.idea/**/dataSources.local.xml
-.idea/**/sqlDataSources.xml
-.idea/**/dynamic.xml
-.idea/**/uiDesigner.xml
-
-# Gradle:
-.idea/**/gradle.xml
-.idea/**/libraries
-
-# CMake
-cmake-build-debug/
-
-# Mongo Explorer plugin:
-.idea/**/mongoSettings.xml
-
-## File-based project format:
-*.iws
-
-## Plugin-specific files:
-
-# IntelliJ
-/out/
-
-# mpeltonen/sbt-idea plugin
-.idea_modules/
-
-# JIRA plugin
-atlassian-ide-plugin.xml
-
-# Cursive Clojure plugin
-.idea/replstate.xml
-
-# Ruby plugin and RubyMine
-/.rakeTasks
-
-# Crashlytics plugin (for Android Studio and IntelliJ)
-com_crashlytics_export_strings.xml
-crashlytics.properties
-crashlytics-build.properties
-fabric.properties
-
-### JetBrains+iml Patch ###
-# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023
-
 *.iml
-.idea/misc.xml
-*.ipr
-
-# End of https://www.gitignore.io/api/jetbrains+iml

.idea/codeStyleSettings.xml

@@ -1,25 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-    <component name="ProjectCodeStyleSettingsManager">
-        <option name="PER_PROJECT_SETTINGS">
-            <value>
-                <option name="OTHER_INDENT_OPTIONS">
-                    <value>
-                        <option name="INDENT_SIZE" value="2" />
-                        <option name="CONTINUATION_INDENT_SIZE" value="8" />
-                        <option name="TAB_SIZE" value="2" />
-                        <option name="USE_TAB_CHARACTER" value="false" />
-                        <option name="SMART_TABS" value="false" />
-                        <option name="LABEL_INDENT_SIZE" value="0" />
-                        <option name="LABEL_INDENT_ABSOLUTE" value="false" />
-                        <option name="USE_RELATIVE_INDENTS" value="false" />
-                    </value>
-                </option>
-                <MarkdownNavigatorCodeStyleSettings>
-                    <option name="RIGHT_MARGIN" value="72" />
-                </MarkdownNavigatorCodeStyleSettings>
-            </value>
-        </option>
-        <option name="USE_PER_PROJECT_SETTINGS" value="true" />
-    </component>
-</project>

.idea/codeStyles/Project.xml

@@ -1,7 +0,0 @@
-<component name="ProjectCodeStyleConfiguration">
-  <code_scheme name="Project" version="173">
-    <MarkdownNavigatorCodeStyleSettings>
-      <option name="RIGHT_MARGIN" value="72" />
-    </MarkdownNavigatorCodeStyleSettings>
-  </code_scheme>
-</component>

.idea/codeStyles/codeStyleConfig.xml

@@ -1,5 +0,0 @@
-<component name="ProjectCodeStyleConfiguration">
-  <state>
-    <option name="USE_PER_PROJECT_SETTINGS" value="true" />
-  </state>
-</component>

README.md

@@ -11,9 +11,9 @@
 </p>
 <!-- markdownlint-enable MD033 -->
 
-The Pi-hole® is a [DNS sinkhole](https://en.wikipedia.org/wiki/DNS_Sinkhole) that protects your devices from unwanted content, without installing any client-side software.
+The Pi-hole® is a [DNS sinkhole](https://en.wikipedia.org/wiki/DNS_Sinkhole) that protects your devices from unwanted content without installing any client-side software.
 
-- **Easy-to-install**: our versatile installer walks you through the process, and takes less than ten minutes
+- **Easy-to-install**: our versatile installer walks you through the process and takes less than ten minutes
 - **Resolute**: content is blocked in _non-browser locations_, such as ad-laden mobile apps and smart TVs
 - **Responsive**: seamlessly speeds up the feel of everyday browsing by caching DNS queries
 - **Lightweight**: runs smoothly with [minimal hardware and software requirements](https://docs.pi-hole.net/main/prerequisites/)
@@ -22,7 +22,7 @@ The Pi-hole® is a [DNS sinkhole](https://en.wikipedia.org/wiki/DNS_Sinkhole) th
 - **Versatile**: can optionally function as a [DHCP server](https://discourse.pi-hole.net/t/how-do-i-use-pi-holes-built-in-dhcp-server-and-why-would-i-want-to/3026), ensuring *all* your devices are protected automatically
 - **Scalable**: [capable of handling hundreds of millions of queries](https://pi-hole.net/2017/05/24/how-much-traffic-can-pi-hole-handle/) when installed on server-grade hardware
 - **Modern**: blocks ads over both IPv4 and IPv6
-- **Free**: open source software which helps ensure _you_ are the sole person in control of your privacy
+- **Free**: open source software that helps ensure _you_ are the sole person in control of your privacy
 
 -----
 
@@ -57,21 +57,21 @@ Please refer to the [Pi-hole docker repo](https://github.com/pi-hole/docker-pi-h
 
 Once the installer has been run, you will need to [configure your router to have **DHCP clients use Pi-hole as their DNS server**](https://discourse.pi-hole.net/t/how-do-i-configure-my-devices-to-use-pi-hole-as-their-dns-server/245) which ensures that all devices connecting to your network will have content blocked without any further intervention.
 
-If your router does not support setting the DNS server, you can [use Pi-hole's built-in DHCP server](https://discourse.pi-hole.net/t/how-do-i-use-pi-holes-built-in-dhcp-server-and-why-would-i-want-to/3026); just be sure to disable DHCP on your router first (if it has that feature available).
+If your router does not support setting the DNS server, you can [use Pi-hole's built-in DHCP server](https://discourse.pi-hole.net/t/how-do-i-use-pi-holes-built-in-dhcp-server-and-why-would-i-want-to/3026); be sure to disable DHCP on your router first (if it has that feature available).
 
-As a last resort, you can always manually set each device to use Pi-hole as their DNS server.
+As a last resort, you can manually set each device to use Pi-hole as their DNS server.
 
 -----
 
-## Pi-hole is free, but powered by your support
+## Pi-hole is free but powered by your support
 
-There are many reoccurring costs involved with maintaining free, open source, and privacy-respecting software; expenses which [our volunteer developers](https://github.com/orgs/pi-hole/people) pitch in to cover out-of-pocket. This is just one example of how strongly we feel about our software, as well as the importance of keeping it maintained.
+There are many reoccurring costs involved with maintaining free, open source, and privacy-respecting software; expenses which [our volunteer developers](https://github.com/orgs/pi-hole/people) pitch in to cover out-of-pocket. This is just one example of how strongly we feel about our software and the importance of keeping it maintained.
 
 Make no mistake: **your support is absolutely vital to help keep us innovating!**
 
 ### [Donations](https://pi-hole.net/donate)
 
-Sending a donation using our Sponsor Button is **extremely helpful** in offsetting a portion of our monthly expenses and rewarding our dedicated development team:
+Donating using our Sponsor Button is **extremely helpful** in offsetting a portion of our monthly expenses:
 
 ### Alternative support
 
@@ -83,13 +83,13 @@ If you'd rather not donate (_which is okay!_), there are other ways you can help
 - [Digital Ocean](https://www.digitalocean.com/?refcode=344d234950e1) _affiliate link_
 - [Stickermule](https://www.stickermule.com/unlock?ref_id=9127301701&utm_medium=link&utm_source=invite) _earn a $10 credit after your first purchase_
 - [Amazon US](http://www.amazon.com/exec/obidos/redirect-home/pihole09-20) _affiliate link_
-- Spreading the word about our software, and how you have benefited from it
+- Spreading the word about our software and how you have benefited from it
 
 ### Contributing via GitHub
 
 We welcome _everyone_ to contribute to issue reports, suggest new features, and create pull requests.
 
-If you have something to add - anything from a typo through to a whole new feature, we're happy to check it out! Just make sure to fill out our template when submitting your request; the questions that it asks will help the volunteers quickly understand what you're aiming to achieve.
+If you have something to add - anything from a typo through to a whole new feature, we're happy to check it out! Just make sure to fill out our template when submitting your request; the questions it asks will help the volunteers quickly understand what you're aiming to achieve.
 
 You'll find that the [install script](https://github.com/pi-hole/pi-hole/blob/master/automated%20install/basic-install.sh) and the [debug script](https://github.com/pi-hole/pi-hole/blob/master/advanced/Scripts/piholeDebug.sh) have an abundance of comments, which will help you better understand how Pi-hole works. They're also a valuable resource to those who want to learn how to write scripts or code a program! We encourage anyone who likes to tinker to read through it and submit a pull request for us to review.
 
@@ -97,9 +97,9 @@ You'll find that the [install script](https://github.com/pi-hole/pi-hole/blob/ma
 
 ## Getting in touch with us
 
-While we are primarily reachable on our [Discourse User Forum](https://discourse.pi-hole.net/), we can also be found on a variety of social media outlets.
+While we are primarily reachable on our [Discourse User Forum](https://discourse.pi-hole.net/), we can also be found on various social media outlets.
 
-**Please be sure to check the FAQ's** before starting a new discussion. Many user questions already have answers and can be solved without any additional assistance.
+**Please be sure to check the FAQs** before starting a new discussion, as we do not have the spare time to reply to every request for assistance.
 
 - [Frequently Asked Questions](https://discourse.pi-hole.net/c/faqs)
 - [Feature Requests](https://discourse.pi-hole.net/c/feature-requests?order=votes)
@@ -125,15 +125,15 @@ Some of the statistics you can integrate include:
 - Queries cached
 - Unique clients
 
-The API can be accessed via [`telnet`](https://github.com/pi-hole/FTL), the Web (`admin/api.php`) and Command Line (`pihole -c -j`). You can find out [more details over here](https://discourse.pi-hole.net/t/pi-hole-api/1863).
+Access the API via [`telnet`](https://github.com/pi-hole/FTL), the Web (`admin/api.php`) and Command Line (`pihole -c -j`). You can find out [more details over here](https://discourse.pi-hole.net/t/pi-hole-api/1863).
 
 ### The Command Line Interface
 
-The [pihole](https://docs.pi-hole.net/core/pihole-command/) command has all the functionality necessary to be able to fully administer the Pi-hole, without the need of the Web Interface. It's fast, user-friendly, and auditable by anyone with an understanding of `bash`.
+The [pihole](https://docs.pi-hole.net/core/pihole-command/) command has all the functionality necessary to fully administer the Pi-hole, without the need of the Web Interface. It's fast, user-friendly, and auditable by anyone with an understanding of `bash`.
 
 Some notable features include:
 
-- [Whitelisting, Blacklisting and Regex](https://docs.pi-hole.net/core/pihole-command/#whitelisting-blacklisting-and-regex)
+- [Whitelisting, Blacklisting, and Regex](https://docs.pi-hole.net/core/pihole-command/#whitelisting-blacklisting-and-regex)
 - [Debugging utility](https://docs.pi-hole.net/core/pihole-command/#debugger)
 - [Viewing the live log file](https://docs.pi-hole.net/core/pihole-command/#tail)
 - [Updating Ad Lists](https://docs.pi-hole.net/core/pihole-command/#gravity)
@@ -149,7 +149,7 @@ This [optional dashboard](https://github.com/pi-hole/AdminLTE) allows you to vie
 
 Some notable features include:
 
-- Mobile friendly interface
+- Mobile-friendly interface
 - Password protection
 - Detailed graphs and doughnut charts
 - Top lists of domains and clients

advanced/Scripts/database_migration/gravity-db.sh

@@ -122,4 +122,10 @@ upgrade_gravityDB(){
 		sqlite3 "${database}" < "${scriptPath}/13_to_14.sql"
 		version=14
 	fi
+	if [[ "$version" == "14" ]]; then
+	# Changes the vw_adlist created in 5_to_6
+	echo -e "  ${INFO} Upgrading gravity database from version 14 to 15"
+	sqlite3 "${database}" < "${scriptPath}/14_to_15.sql"
+	version=15
+fi
 }

advanced/Scripts/database_migration/gravity/14_to_15.sql

@@ -0,0 +1,15 @@
+.timeout 30000
+
+PRAGMA FOREIGN_KEYS=OFF;
+
+BEGIN TRANSACTION;
+DROP VIEW vw_adlist;
+
+CREATE VIEW vw_adlist AS SELECT DISTINCT address, id
+    FROM adlist
+    WHERE enabled = 1
+    ORDER BY id;
+
+UPDATE info SET value = 15 WHERE property = 'version';
+
+COMMIT;

advanced/Scripts/list.sh

@@ -23,7 +23,7 @@ fi
 # have changed
 gravityDBfile="${GRAVITYDB}"
 
-reload=false
+noReloadRequested=false
 addmode=true
 verbose=true
 wildcard=false
@@ -35,6 +35,7 @@ typeId=""
 comment=""
 declare -i domaincount
 domaincount=0
+reload=false
 
 colfile="/opt/pihole/COL_TABLE"
 source ${colfile}
@@ -268,7 +269,7 @@ while (( "$#" )); do
         "--white-wild" | "white-wild" ) typeId=2; wildcard=true;;
         "--wild" | "wildcard" ) typeId=3; wildcard=true;;
         "--regex" | "regex"   ) typeId=3;;
-        "-nr"| "--noreload"  ) reload=false;;
+        "-nr"| "--noreload"  ) noReloadRequested=true;;
         "-d" | "--delmode"   ) addmode=false;;
         "-q" | "--quiet"     ) verbose=false;;
         "-h" | "--help"      ) helpFunc;;
@@ -294,6 +295,6 @@ if $web; then
 echo "DONE"
 fi
 
-if [[ "${reload}" != false ]]; then
+if [[ ${reload} == true && ${noReloadRequested} == false ]]; then
     pihole restartdns reload-lists
 fi

advanced/Scripts/piholeDebug.sh

@@ -585,6 +585,27 @@ processor_check() {
     fi
 }
 
+disk_usage() {
+    local file_system
+    local hide
+
+    echo_current_diagnostic "Disk usage"
+    mapfile -t file_system < <(df -h)
+
+    # Some lines of df might contain sensitive information like usernames and passwords.
+    # E.g. curlftpfs filesystems (https://www.looklinux.com/mount-ftp-share-on-linux-using-curlftps/)
+    # We are not interested in those lines so we collect keyword, to remove them from the output
+    # Additinal keywords can be added, separated by "|"
+    hide="curlftpfs"
+
+    # only show those lines not containg a sensitive phrase
+    for line in "${file_system[@]}"; do
+      if [[ ! $line =~ $hide ]]; then
+        log_write "   ${line}"
+      fi
+    done
+}
+
 parse_setup_vars() {
     echo_current_diagnostic "Setup variables"
     # If the file exists,
@@ -1238,10 +1259,10 @@ show_messages() {
 }
 
 analyze_gravity_list() {
-    echo_current_diagnostic "Gravity List and Database"
+    echo_current_diagnostic "Gravity Database"
 
     local gravity_permissions
-    gravity_permissions=$(ls -ld "${PIHOLE_GRAVITY_DB_FILE}")
+    gravity_permissions=$(ls -lhd "${PIHOLE_GRAVITY_DB_FILE}")
     log_write "${COL_GREEN}${gravity_permissions}${COL_NC}"
 
     show_db_entries "Info table" "SELECT property,value FROM info" "20 40"
@@ -1320,7 +1341,7 @@ analyze_pihole_log() {
   OLD_IFS="$IFS"
   # Get the lines that are in the file(s) and store them in an array for parsing later
   IFS=$'\r\n'
-  pihole_log_permissions=$(ls -ld "${PIHOLE_LOG}")
+  pihole_log_permissions=$(ls -lhd "${PIHOLE_LOG}")
   log_write "${COL_GREEN}${pihole_log_permissions}${COL_NC}"
   mapfile -t pihole_log_head < <(head -n 20 ${PIHOLE_LOG})
   log_write "   ${COL_CYAN}-----head of $(basename ${PIHOLE_LOG})------${COL_NC}"
@@ -1421,6 +1442,7 @@ diagnose_operating_system
 check_selinux
 check_firewalld
 processor_check
+disk_usage
 check_networking
 check_name_resolution
 check_dhcp_servers

advanced/Scripts/webpage.sh

@@ -709,10 +709,25 @@ AddCustomDNSAddress() {
 
     ip="${args[2]}"
     host="${args[3]}"
-	echo "${ip} ${host}" >> "${dnscustomfile}"
+    reload="${args[4]}"
 
-    # Restart dnsmasq to load new custom DNS entries
+    validHost="$(checkDomain "${host}")"
+    if [[ -n "${validHost}" ]]; then
+        if valid_ip "${ip}" || valid_ip6 "${ip}" ; then
+            echo "${ip} ${validHost}" >> "${dnscustomfile}"
+        else
+            echo -e "  ${CROSS} Invalid IP has been passed"
+            exit 1
+        fi
+    else
+        echo "  ${CROSS} Invalid Domain passed!"
+        exit 1
+    fi
+
+    # Restart dnsmasq to load new custom DNS entries only if $reload not false
+    if [[ ! $reload == "false" ]]; then
       RestartDNS
+    fi
 }
 
 RemoveCustomDNSAddress() {
@@ -720,16 +735,25 @@ RemoveCustomDNSAddress() {
 
     ip="${args[2]}"
     host="${args[3]}"
+    reload="${args[4]}"
 
+    validHost="$(checkDomain "${host}")"
+    if [[ -n "${validHost}" ]]; then
         if valid_ip "${ip}" || valid_ip6 "${ip}" ; then
-        sed -i "/^${ip} ${host}$/d" "${dnscustomfile}"
+            sed -i "/^${ip} ${validHost}$/d" "${dnscustomfile}"
         else
             echo -e "  ${CROSS} Invalid IP has been passed"
             exit 1
         fi
+        else
+            echo "  ${CROSS} Invalid Domain passed!"
+            exit 1
+    fi
 
-    # Restart dnsmasq to update removed custom DNS entries
+    # Restart dnsmasq to load new custom DNS entries only if reload is not false
+    if [[ ! $reload == "false" ]]; then
       RestartDNS
+    fi
 }
 
 AddCustomCNAMERecord() {
@@ -737,11 +761,25 @@ AddCustomCNAMERecord() {
 
     domain="${args[2]}"
     target="${args[3]}"
+    reload="${args[4]}"
 
-    echo "cname=${domain},${target}" >> "${dnscustomcnamefile}"
+    validDomain="$(checkDomain "${domain}")"
-
+    if [[ -n "${validDomain}" ]]; then
-    # Restart dnsmasq to load new custom CNAME records
+        validTarget="$(checkDomain "${target}")"
+        if [[ -n "${validTarget}" ]]; then
+          echo "cname=${validDomain},${validTarget}" >> "${dnscustomcnamefile}"
+        else
+          echo "  ${CROSS} Invalid Target Passed!"
+          exit 1
+        fi
+    else
+        echo "  ${CROSS} Invalid Domain passed!"
+        exit 1
+    fi
+    # Restart dnsmasq to load new custom CNAME records only if reload is not false
+    if [[ ! $reload == "false" ]]; then
       RestartDNS
+    fi
 }
 
 RemoveCustomCNAMERecord() {
@@ -749,11 +787,12 @@ RemoveCustomCNAMERecord() {
 
     domain="${args[2]}"
     target="${args[3]}"
+    reload="${args[4]}"
 
     validDomain="$(checkDomain "${domain}")"
     if [[ -n "${validDomain}" ]]; then
         validTarget="$(checkDomain "${target}")"
-        if [[ -n "${validDomain}" ]]; then
+        if [[ -n "${validTarget}" ]]; then
             sed -i "/cname=${validDomain},${validTarget}$/d" "${dnscustomcnamefile}"
         else
           echo "  ${CROSS} Invalid Target Passed!"
@@ -764,8 +803,10 @@ RemoveCustomCNAMERecord() {
         exit 1
     fi
 
-    # Restart dnsmasq to update removed custom CNAME records
+    # Restart dnsmasq to update removed custom CNAME records only if $reload not false
+    if [[ ! $reload == "false" ]]; then
       RestartDNS
+    fi
 }
 
 main() {

advanced/Scripts/wildcard_regex_converter.sh

@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-# Pi-hole: A black hole for Internet advertisements
-# (c) 2017 Pi-hole, LLC (https://pi-hole.net)
-# Network-wide ad blocking via your own hardware.
-#
-# Provides an automated migration subroutine to convert Pi-hole v3.x wildcard domains to Pi-hole v4.x regex filters
-#
-# This file is copyright under the latest version of the EUPL.
-# Please see LICENSE file for your rights under this license.
-
-# regexFile set in gravity.sh
-
-wildcardFile="/etc/dnsmasq.d/03-pihole-wildcard.conf"
-
-convert_wildcard_to_regex() {
-    if [ ! -f "${wildcardFile}" ]; then
-        return
-    fi
-    local addrlines domains uniquedomains
-    # Obtain wildcard domains from old file
-    addrlines="$(grep -oE "/.*/" ${wildcardFile})"
-    # Strip "/" from domain names and convert "." to regex-compatible "\."
-    domains="$(sed 's/\///g;s/\./\\./g' <<< "${addrlines}")"
-    # Remove repeated domains (may have been inserted two times due to A and AAAA blocking)
-    uniquedomains="$(uniq <<< "${domains}")"
-    # Automatically generate regex filters and remove old wildcards file
-    awk '{print "(^|\\.)"$0"$"}' <<< "${uniquedomains}" >> "${regexFile:?}" && rm "${wildcardFile}"
-}

advanced/Templates/gravity.db.sql

@@ -57,7 +57,7 @@ CREATE TABLE info
 	value TEXT NOT NULL
 );
 
-INSERT INTO "info" VALUES('version','14');
+INSERT INTO "info" VALUES('version','15');
 
 CREATE TABLE domain_audit
 (
@@ -143,12 +143,10 @@ CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id
     LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
     WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1);
 
-CREATE VIEW vw_adlist AS SELECT DISTINCT address, adlist.id AS id
+CREATE VIEW vw_adlist AS SELECT DISTINCT address, id
     FROM adlist
-    LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = adlist.id
+    WHERE enabled = 1
-    LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
+    ORDER BY id;
-    WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1)
-    ORDER BY adlist.id;
 
 CREATE TRIGGER tr_domainlist_add AFTER INSERT ON domainlist
     BEGIN

advanced/Templates/pihole-FTL.service

@@ -24,9 +24,13 @@ start() {
     touch /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole-FTL.log /var/log/pihole.log /etc/pihole/dhcp.leases
     # Ensure that permissions are set so that pihole-FTL can edit all necessary files
     chown pihole:pihole /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole-FTL.log /var/log/pihole.log /etc/pihole/dhcp.leases /run/pihole /etc/pihole
-    chmod 0644 /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole-FTL.log /var/log/pihole.log /etc/pihole/dhcp.leases /etc/pihole/macvendor.db
+    chmod 0644 /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole-FTL.log /var/log/pihole.log /etc/pihole/dhcp.leases
+    # Ensure that permissions are set so that pihole-FTL can edit the files. We ignore errors as the file may not (yet) exist
+    chmod -f 0644 /etc/pihole/macvendor.db
     # Chown database files to the user FTL runs as. We ignore errors as the files may not (yet) exist
     chown -f pihole:pihole /etc/pihole/pihole-FTL.db /etc/pihole/gravity.db /etc/pihole/macvendor.db
+    # Chown database file permissions so that the pihole group (web interface) can edit the file. We ignore errors as the files may not (yet) exist
+    chmod -f 0664 /etc/pihole/pihole-FTL.db
     if setcap CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_NICE,CAP_IPC_LOCK,CAP_CHOWN+eip "/usr/bin/pihole-FTL"; then
       su -s /bin/sh -c "/usr/bin/pihole-FTL" pihole
     else

advanced/lighttpd.conf.debian

@@ -85,5 +85,12 @@ $HTTP["url"] =~ "^/admin/\.(.*)" {
     url.access-deny = ("")
 }
 
+# allow teleporter iframe on settings page
+$HTTP["url"] =~ "/teleporter\.php$" {
+    $HTTP["referer"] =~ "/admin/settings\.php" {
+        setenv.add-response-header = ( "X-Frame-Options" => "SAMEORIGIN" )
+    }
+}
+
 # Default expire header
 expire.url = ( "" => "access plus 0 seconds" )

advanced/lighttpd.conf.fedora

@@ -93,5 +93,12 @@ $HTTP["url"] =~ "^/admin/\.(.*)" {
     url.access-deny = ("")
 }
 
+# allow teleporter iframe on settings page
+$HTTP["url"] =~ "/teleporter\.php$" {
+    $HTTP["referer"] =~ "/admin/settings\.php" {
+        setenv.add-response-header = ( "X-Frame-Options" => "SAMEORIGIN" )
+    }
+}
+
 # Default expire header
 expire.url = ( "" => "access plus 0 seconds" )

automated install/basic-install.sh

@@ -34,27 +34,26 @@ export PATH+=':/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
 
 # List of supported DNS servers
 DNS_SERVERS=$(cat << EOM
-Google (ECS);8.8.8.8;8.8.4.4;2001:4860:4860:0:0:0:0:8888;2001:4860:4860:0:0:0:0:8844
+Google (ECS, DNSSEC);8.8.8.8;8.8.4.4;2001:4860:4860:0:0:0:0:8888;2001:4860:4860:0:0:0:0:8844
 OpenDNS (ECS, DNSSEC);208.67.222.222;208.67.220.220;2620:119:35::35;2620:119:53::53
 Level3;4.2.2.1;4.2.2.2;;
 Comodo;8.26.56.26;8.20.247.20;;
-DNS.WATCH;84.200.69.80;84.200.70.40;2001:1608:10:25:0:0:1c04:b12f;2001:1608:10:25:0:0:9249:d69b
+DNS.WATCH (DNSSEC);84.200.69.80;84.200.70.40;2001:1608:10:25:0:0:1c04:b12f;2001:1608:10:25:0:0:9249:d69b
 Quad9 (filtered, DNSSEC);9.9.9.9;149.112.112.112;2620:fe::fe;2620:fe::9
 Quad9 (unfiltered, no DNSSEC);9.9.9.10;149.112.112.10;2620:fe::10;2620:fe::fe:10
-Quad9 (filtered + ECS);9.9.9.11;149.112.112.11;2620:fe::11;2620:fe::fe:11
+Quad9 (filtered, ECS, DNSSEC);9.9.9.11;149.112.112.11;2620:fe::11;2620:fe::fe:11
-Cloudflare;1.1.1.1;1.0.0.1;2606:4700:4700::1111;2606:4700:4700::1001
+Cloudflare (DNSSEC);1.1.1.1;1.0.0.1;2606:4700:4700::1111;2606:4700:4700::1001
 EOM
 )
 
 # Location for final installation log storage
-installLogLoc=/etc/pihole/install.log
+installLogLoc="/etc/pihole/install.log"
 # This is an important file as it contains information specific to the machine it's being installed on
-setupVars=/etc/pihole/setupVars.conf
+setupVars="/etc/pihole/setupVars.conf"
 # Pi-hole uses lighttpd as a Web server, and this is the config file for it
-# shellcheck disable=SC2034
+lighttpdConfig="/etc/lighttpd/lighttpd.conf"
-lighttpdConfig=/etc/lighttpd/lighttpd.conf
 # This is a file used for the colorized output
-coltable=/opt/pihole/COL_TABLE
+coltable="/opt/pihole/COL_TABLE"
 
 # Root of the web server
 webroot="/var/www/html"
@@ -262,10 +261,10 @@ os_check() {
 
 # Compatibility
 package_manager_detect() {
-# If apt-get is installed, then we know it's part of the Debian family
+# First check to see if apt-get is installed.
 if is_command apt-get ; then
     # Set some global variables here
-    # We don't set them earlier since the family might be Red Hat, so these values would be different
+    # We don't set them earlier since the installed package manager might be rpm, so these values would be different
     PKG_MANAGER="apt-get"
     # A variable to store the command used to update the package cache
     UPDATE_PKG_CACHE="${PKG_MANAGER} update"
@@ -273,68 +272,27 @@ if is_command apt-get ; then
     PKG_INSTALL=("${PKG_MANAGER}" -qq --no-install-recommends install)
     # grep -c will return 1 if there are no matches. This is an acceptable condition, so we OR TRUE to prevent set -e exiting the script.
     PKG_COUNT="${PKG_MANAGER} -s -o Debug::NoLocking=true upgrade | grep -c ^Inst || true"
-    # Update package cache. This is required already here to assure apt-cache calls have package lists available.
+    # Update package cache
     update_package_cache || exit 1
-    # Debian 7 doesn't have iproute2 so check if it's available first
-    if apt-cache show iproute2 > /dev/null 2>&1; then
-        iproute_pkg="iproute2"
-    # Otherwise, check if iproute is available
-    elif apt-cache show iproute > /dev/null 2>&1; then
-        iproute_pkg="iproute"
-    # Else print error and exit
-    else
-        printf "  %b Aborting installation: iproute2 and iproute packages were not found in APT repository.\\n" "${CROSS}"
-        exit 1
-    fi
     # Check for and determine version number (major and minor) of current php install
+    local phpVer="php"
     if is_command php ; then
         printf "  %b Existing PHP installation detected : PHP version %s\\n" "${INFO}" "$(php <<< "<?php echo PHP_VERSION ?>")"
         printf -v phpInsMajor "%d" "$(php <<< "<?php echo PHP_MAJOR_VERSION ?>")"
         printf -v phpInsMinor "%d" "$(php <<< "<?php echo PHP_MINOR_VERSION ?>")"
-        # Is installed php version 7.0 or greater
-        if [ "${phpInsMajor}" -ge 7 ]; then
-            phpInsNewer=true
-        fi
-    fi
-    # Several other packages depend on the version of PHP. If PHP is not installed, or an insufficient version,
-    # those packages should fall back to the default (latest?)
-    if [[ "$phpInsNewer" != true ]]; then
-        # Prefer the php metapackage if it's there
-        if apt-cache show php > /dev/null 2>&1; then
-            phpVer="php"
-        # Else fall back on the php5 package if it's there
-        elif apt-cache show php5 > /dev/null 2>&1; then
-            phpVer="php5"
-        # Else print error and exit
-        else
-            printf "  %b Aborting installation: No PHP packages were found in APT repository.\\n" "${CROSS}"
-            exit 1
-        fi
-    else
-        # Else, PHP is already installed at a version beyond v7.0, so the additional packages
-        # should match version with the current PHP version.
         phpVer="php$phpInsMajor.$phpInsMinor"
     fi
-    # We also need the correct version for `php-sqlite` (which differs across distros)
-    if apt-cache show "${phpVer}-sqlite3" > /dev/null 2>&1; then
-        phpSqlite="sqlite3"
-    elif apt-cache show "${phpVer}-sqlite" > /dev/null 2>&1; then
-        phpSqlite="sqlite"
-    else
-        printf "  %b Aborting installation: No SQLite PHP module was found in APT repository.\\n" "${CROSS}"
-        exit 1
-    fi
     # Packages required to perfom the os_check (stored as an array)
     OS_CHECK_DEPS=(grep dnsutils)
     # Packages required to run this install script (stored as an array)
-    INSTALLER_DEPS=(git "${iproute_pkg}" whiptail ca-certificates)
+    INSTALLER_DEPS=(git iproute2 whiptail ca-certificates)
     # Packages required to run Pi-hole (stored as an array)
-    PIHOLE_DEPS=(cron curl iputils-ping lsof netcat psmisc sudo unzip idn2 sqlite3 libcap2-bin dns-root-data libcap2)
+    PIHOLE_DEPS=(cron curl iputils-ping lsof psmisc sudo unzip idn2 sqlite3 libcap2-bin dns-root-data libcap2)
     # Packages required for the Web admin interface (stored as an array)
     # It's useful to separate this from Pi-hole, since the two repos are also setup separately
-    PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-${phpSqlite}" "${phpVer}-xml" "${phpVer}-intl")
+    PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-sqlite3" "${phpVer}-xml" "${phpVer}-intl")
     # Prior to PHP8.0, JSON functionality is provided as dedicated module, required by Pi-hole AdminLTE: https://www.php.net/manual/json.installation.php
-    if [[ "${phpInsNewer}" != true || "${phpInsMajor}" -lt 8 ]]; then
+    if [[ -z "${phpInsMajor}" || "${phpInsMajor}" -lt 8 ]]; then
         PIHOLE_WEB_DEPS+=("${phpVer}-json")
     fi
     # The Web server user,
@@ -360,7 +318,7 @@ if is_command apt-get ; then
         return 0
     }
 
-# If apt-get is not found, check for rpm to see if it's a Red Hat family OS
+# If apt-get is not found, check for rpm.
 elif is_command rpm ; then
     # Then check if dnf or yum is the package manager
     if is_command dnf ; then
@@ -369,12 +327,12 @@ elif is_command rpm ; then
         PKG_MANAGER="yum"
     fi
 
-    # These variable names match the ones in the Debian family. See above for an explanation of what they are for.
+    # These variable names match the ones for apt-get. See above for an explanation of what they are for.
     PKG_INSTALL=("${PKG_MANAGER}" install -y)
     PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l"
     OS_CHECK_DEPS=(grep bind-utils)
     INSTALLER_DEPS=(git iproute newt procps-ng which chkconfig ca-certificates)
-    PIHOLE_DEPS=(cronie curl findutils nmap-ncat sudo unzip libidn2 psmisc sqlite libcap lsof)
+    PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc sqlite libcap lsof)
     PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo php-xml php-json php-intl)
     LIGHTTPD_USER="lighttpd"
     LIGHTTPD_GROUP="lighttpd"
@@ -382,8 +340,8 @@ elif is_command rpm ; then
 
 # If neither apt-get or yum/dnf package managers were found
 else
-    # it's not an OS we can support,
+    # we cannot install required packages
-    printf "  %b OS distribution not supported\\n" "${CROSS}"
+    printf "  %b No supported package manager found\\n" "${CROSS}"
     # so exit the installer
     exit
 fi
@@ -556,7 +514,7 @@ update_repo() {
     git stash --all --quiet &> /dev/null || true # Okay for stash failure
     git clean --quiet --force -d || true # Okay for already clean directory
     # Pull the latest commits
-    git pull --quiet &> /dev/null || return $?
+    git pull --no-rebase --quiet &> /dev/null || return $?
     # Check current branch. If it is master, then reset to the latest available tag.
     # In case extra commits have been added after tagging/release (i.e in case of metadata updates/README.MD tweaks)
     curBranch=$(git rev-parse --abbrev-ref HEAD)
@@ -758,9 +716,8 @@ testIPv6() {
     fi
 }
 
-# A dialog for showing the user about IPv6 blocking
+find_IPv6_information() {
-useIPv6dialog() {
+    # Detects IPv6 address used for communication to WAN addresses.
-    # Determine the IPv6 address used for blocking
     IPV6_ADDRESSES=($(ip -6 address | grep 'scope global' | awk '{print $2}'))
 
     # For each address in the array above, determine the type of IPv6 address it is
@@ -780,76 +737,34 @@ useIPv6dialog() {
         # set the IPv6 address to the ULA address
         IPV6_ADDRESS="${ULA_ADDRESS}"
         # Show this info to the user
-        printf "  %b Found IPv6 ULA address, using it for blocking IPv6 ads\\n" "${INFO}"
+        printf "  %b Found IPv6 ULA address\\n" "${INFO}"
     # Otherwise, if the GUA_ADDRESS has a value,
     elif [[ ! -z "${GUA_ADDRESS}" ]]; then
         # Let the user know
-        printf "  %b Found IPv6 GUA address, using it for blocking IPv6 ads\\n" "${INFO}"
+        printf "  %b Found IPv6 GUA address\\n" "${INFO}"
         # And assign it to the global variable
         IPV6_ADDRESS="${GUA_ADDRESS}"
     # If none of those work,
     else
-        # explain that IPv6 blocking will not be used
+        printf "  %b Unable to find IPv6 ULA/GUA address\\n" "${INFO}"
-        printf "  %b Unable to find IPv6 ULA/GUA address, IPv6 adblocking will not be enabled\\n" "${INFO}"
         # So set the variable to be empty
         IPV6_ADDRESS=""
     fi
-
-    # If the IPV6_ADDRESS contains a value
-    if [[ ! -z "${IPV6_ADDRESS}" ]]; then
-        # Display that IPv6 is supported and will be used
-        whiptail --msgbox --backtitle "IPv6..." --title "IPv6 Supported" "$IPV6_ADDRESS will be used to block ads." "${r}" "${c}"
-    fi
 }
 
-# A function to check if we should use IPv4 and/or IPv6 for blocking ads
+# A function to collect IPv4 and IPv6 information of the device
-use4andor6() {
+collect_v4andv6_information() {
-    # Named local variables
-    local useIPv4
-    local useIPv6
-    # Let user choose IPv4 and/or IPv6 via a checklist
-    cmd=(whiptail --separate-output --checklist "Select Protocols (press space to toggle selection)" "${r}" "${c}" 2)
-    # In an array, show the options available:
-    # IPv4 (on by default)
-    options=(IPv4 "Block ads over IPv4" on
-    # or IPv6 (on by default if available)
-    IPv6 "Block ads over IPv6" on)
-    # In a variable, show the choices available; exit if Cancel is selected
-    choices=$("${cmd[@]}" "${options[@]}" 2>&1 >/dev/tty) || { printf "  %bCancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; exit 1; }
-    # For each choice available,
-    for choice in ${choices}
-    do
-        # Set the values to true
-        case ${choice} in
-        IPv4  )   useIPv4=true;;
-        IPv6  )   useIPv6=true;;
-        esac
-    done
-    # If IPv4 is to be used,
-    if [[ "${useIPv4}" ]]; then
-        # Run our function to get the information we need
     find_IPv4_information
+    # Echo the information to the user
+    printf "  %b IPv4 address: %s\\n" "${INFO}" "${IPV4_ADDRESS}"
+    # if `dhcpcd` is used offer to set this as static IP for the device
     if [[ -f "/etc/dhcpcd.conf" ]]; then
             # configure networking via dhcpcd
             getStaticIPv4Settings
             setDHCPCD
     fi
-    fi
+    find_IPv6_information
-    # If IPv6 is to be used,
-    if [[ "${useIPv6}" ]]; then
-        # Run our function to get this information
-        useIPv6dialog
-    fi
-    # Echo the information to the user
-    printf "  %b IPv4 address: %s\\n" "${INFO}" "${IPV4_ADDRESS}"
     printf "  %b IPv6 address: %s\\n" "${INFO}" "${IPV6_ADDRESS}"
-    # If neither protocol is selected,
-    if [[ ! "${useIPv4}" ]] && [[ ! "${useIPv6}" ]]; then
-        # Show an error in red
-        printf "  %bError: Neither IPv4 or IPv6 selected%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"
-        # and exit with an error
-        exit 1
-    fi
 }
 
 getStaticIPv4Settings() {
@@ -1397,18 +1312,18 @@ installConfigs() {
             # make it and set the owners
             install -d -m 755 -o "${USER}" -g root /etc/lighttpd
         # Otherwise, if the config file already exists
-        elif [[ -f "/etc/lighttpd/lighttpd.conf" ]]; then
+        elif [[ -f "${lighttpdConfig}" ]]; then
             # back up the original
-            mv /etc/lighttpd/lighttpd.conf /etc/lighttpd/lighttpd.conf.orig
+            mv "${lighttpdConfig}"{,.orig}
         fi
         # and copy in the config file Pi-hole needs
-        install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} /etc/lighttpd/lighttpd.conf
+        install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} "${lighttpdConfig}"
         # Make sure the external.conf file exists, as lighttpd v1.4.50 crashes without it
         touch /etc/lighttpd/external.conf
         chmod 644 /etc/lighttpd/external.conf
         # If there is a custom block page in the html/pihole directory, replace 404 handler in lighttpd config
         if [[ -f "${PI_HOLE_BLOCKPAGE_DIR}/custom.php" ]]; then
-            sed -i 's/^\(server\.error-handler-404\s*=\s*\).*$/\1"pihole\/custom\.php"/' /etc/lighttpd/lighttpd.conf
+            sed -i 's/^\(server\.error-handler-404\s*=\s*\).*$/\1"pihole\/custom\.php"/' "${lighttpdConfig}"
         fi
         # Make the directories if they do not exist and set the owners
         mkdir -p /run/lighttpd
@@ -1555,9 +1470,6 @@ disable_resolved_stublistener() {
 }
 
 update_package_cache() {
-    # Running apt-get update/upgrade with minimal output can cause some issues with
-    # requiring user input (e.g password for phpmyadmin see #218)
-
     # Update package cache on apt based OSes. Do this every time since
     # it's quick and packages can be updated at any time.
 
@@ -1622,6 +1534,8 @@ install_dependent_packages() {
         # If there's anything to install, install everything in the list.
         if [[ "${#installArray[@]}" -gt 0 ]]; then
             test_dpkg_lock
+            # Running apt-get install with minimal output can cause some issues with
+            # requiring user input (e.g password for phpmyadmin see #218)
             printf "  %b Processing %s install(s) for: %s, please wait...\\n" "${INFO}" "${PKG_MANAGER}" "${installArray[*]}"
             printf '%*s\n' "$columns" '' | tr " " -;
             "${PKG_INSTALL[@]}" "${installArray[@]}"
@@ -1800,7 +1714,7 @@ finalExports() {
     # If the setup variable file exists,
     if [[ -e "${setupVars}" ]]; then
         # update the variables in the file
-        sed -i.update.bak '/PIHOLE_INTERFACE/d;/IPV4_ADDRESS/d;/IPV6_ADDRESS/d;/PIHOLE_DNS_1\b/d;/PIHOLE_DNS_2\b/d;/QUERY_LOGGING/d;/INSTALL_WEB_SERVER/d;/INSTALL_WEB_INTERFACE/d;/LIGHTTPD_ENABLED/d;/CACHE_SIZE/d;' "${setupVars}"
+        sed -i.update.bak '/PIHOLE_INTERFACE/d;/IPV4_ADDRESS/d;/IPV6_ADDRESS/d;/PIHOLE_DNS_1\b/d;/PIHOLE_DNS_2\b/d;/QUERY_LOGGING/d;/INSTALL_WEB_SERVER/d;/INSTALL_WEB_INTERFACE/d;/LIGHTTPD_ENABLED/d;/CACHE_SIZE/d;/DNS_FQDN_REQUIRED/d;/DNS_BOGUS_PRIV/d;' "${setupVars}"
     fi
     # echo the information to the user
     {
@@ -1814,8 +1728,8 @@ finalExports() {
     echo "INSTALL_WEB_INTERFACE=${INSTALL_WEB_INTERFACE}"
     echo "LIGHTTPD_ENABLED=${LIGHTTPD_ENABLED}"
     echo "CACHE_SIZE=${CACHE_SIZE}"
-    echo "DNS_FQDN_REQUIRED=true"
+    echo "DNS_FQDN_REQUIRED=${DNS_FQDN_REQUIRED:-true}"
-    echo "DNS_BOGUS_PRIV=true"
+    echo "DNS_BOGUS_PRIV=${DNS_BOGUS_PRIV:-true}"
     }>> "${setupVars}"
     chmod 644 "${setupVars}"
 
@@ -1863,27 +1777,6 @@ installLogrotate() {
     printf "%b  %b %s\\n" "${OVER}" "${TICK}" "${str}"
 }
 
-# At some point in the future this list can be pruned, for now we'll need it to ensure updates don't break.
-# Refactoring of install script has changed the name of a couple of variables. Sort them out here.
-accountForRefactor() {
-    sed -i 's/piholeInterface/PIHOLE_INTERFACE/g' "${setupVars}"
-    sed -i 's/IPv4_address/IPV4_ADDRESS/g' "${setupVars}"
-    sed -i 's/IPv4addr/IPV4_ADDRESS/g' "${setupVars}"
-    sed -i 's/IPv6_address/IPV6_ADDRESS/g' "${setupVars}"
-    sed -i 's/piholeIPv6/IPV6_ADDRESS/g' "${setupVars}"
-    sed -i 's/piholeDNS1/PIHOLE_DNS_1/g' "${setupVars}"
-    sed -i 's/piholeDNS2/PIHOLE_DNS_2/g' "${setupVars}"
-    sed -i 's/^INSTALL_WEB=/INSTALL_WEB_INTERFACE=/' "${setupVars}"
-    # Add 'INSTALL_WEB_SERVER', if its not been applied already: https://github.com/pi-hole/pi-hole/pull/2115
-    if ! grep -q '^INSTALL_WEB_SERVER=' ${setupVars}; then
-        local webserver_installed=false
-        if grep -q '^INSTALL_WEB_INTERFACE=true' ${setupVars}; then
-            webserver_installed=true
-        fi
-        echo -e "INSTALL_WEB_SERVER=$webserver_installed" >> "${setupVars}"
-    fi
-}
-
 # Install base files and web interface
 installPihole() {
     # If the user wants to install the Web interface,
@@ -1914,10 +1807,6 @@ installPihole() {
             fi
         fi
     fi
-    # For updates and unattended install.
-    if [[ "${useUpdateVars}" == true ]]; then
-        accountForRefactor
-    fi
     # Install base files and web interface
     if ! installScripts; then
         printf "  %b Failure in dependent script copy function.\\n" "${CROSS}"
@@ -2002,7 +1891,7 @@ displayFinalMessage() {
     if [[ "${#1}" -gt 0 ]] ; then
         # set the password to the first argument.
         pwstring="$1"
-    elif [[ $(grep 'WEBPASSWORD' -c /etc/pihole/setupVars.conf) -gt 0 ]]; then
+    elif [[ $(grep 'WEBPASSWORD' -c "${setupVars}") -gt 0 ]]; then
         # Else if the password exists from previous setup, we'll load it later
         pwstring="unchanged"
     else
@@ -2025,8 +1914,6 @@ IPv6:	${IPV6_ADDRESS:-"Not Configured"}
 
 If you have not done so already, the above IP should be set to static.
 
-The install log is in /etc/pihole.
-
 ${additional}" "${r}" "${c}"
 }
 
@@ -2139,7 +2026,7 @@ checkout_pull_branch() {
     # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git)
     chmod -R a+rX "${directory}"
 
-    git_pull=$(git pull || return 1)
+    git_pull=$(git pull --no-rebase || return 1)
 
     if [[ "$git_pull" == *"up-to-date"* ]]; then
         printf "  %b %s\\n" "${INFO}" "${git_pull}"
@@ -2586,8 +2473,8 @@ main() {
         setDNS
         # Give the user a choice of blocklists to include in their install. Or not.
         chooseBlocklists
-        # Let the user decide if they want to block ads over IPv4 and/or IPv6
+        # find IPv4 and IPv6 information of the device
-        use4andor6
+        collect_v4andv6_information
         # Let the user decide if they want the web interface to be installed automatically
         setAdminFlag
         # Let the user decide if they want query logging enabled...
@@ -2660,7 +2547,7 @@ main() {
         # Add password to web UI if there is none
         pw=""
         # If no password is set,
-        if [[ $(grep 'WEBPASSWORD' -c /etc/pihole/setupVars.conf) == 0 ]] ; then
+        if [[ $(grep 'WEBPASSWORD' -c "${setupVars}") == 0 ]] ; then
             # generate a random password
             pw=$(tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c 8)
             # shellcheck disable=SC1091

gravity.sh

@@ -15,8 +15,6 @@ export LC_ALL=C
 
 coltable="/opt/pihole/COL_TABLE"
 source "${coltable}"
-regexconverter="/opt/pihole/wildcard_regex_converter.sh"
-source "${regexconverter}"
 # shellcheck disable=SC1091
 source "/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh"
 

test/_fedora_34.Dockerfile

@@ -1,4 +1,4 @@
-FROM fedora:32
+FROM fedora:34
 
 ENV GITDIR /etc/.pihole
 ENV SCRIPTDIR /opt/pihole

test/test_automated_install.py

@@ -11,18 +11,18 @@ from .conftest import (
 )
 
 
-def test_supported_operating_system(Pihole):
+def test_supported_package_manager(Pihole):
     '''
-    confirm installer exists on unsupported distribution
+    confirm installer exits when no supported package manager found
     '''
-    # break supported package managers to emulate an unsupported distribution
+    # break supported package managers
     Pihole.run('rm -rf /usr/bin/apt-get')
     Pihole.run('rm -rf /usr/bin/rpm')
     package_manager_detect = Pihole.run('''
     source /opt/pihole/basic-install.sh
     package_manager_detect
     ''')
-    expected_stdout = cross_box + ' OS distribution not supported'
+    expected_stdout = cross_box + ' No supported package manager found'
     assert expected_stdout in package_manager_detect.stdout
     # assert package_manager_detect.rc == 1
 
@@ -421,10 +421,9 @@ def test_IPv6_only_link_local(Pihole):
     )
     detectPlatform = Pihole.run('''
     source /opt/pihole/basic-install.sh
-    useIPv6dialog
+    find_IPv6_information
     ''')
-    expected_stdout = ('Unable to find IPv6 ULA/GUA address, '
+    expected_stdout = ('Unable to find IPv6 ULA/GUA address')
-                       'IPv6 adblocking will not be enabled')
     assert expected_stdout in detectPlatform.stdout
 
 
@@ -445,9 +444,9 @@ def test_IPv6_only_ULA(Pihole):
     )
     detectPlatform = Pihole.run('''
     source /opt/pihole/basic-install.sh
-    useIPv6dialog
+    find_IPv6_information
     ''')
-    expected_stdout = 'Found IPv6 ULA address, using it for blocking IPv6 ads'
+    expected_stdout = 'Found IPv6 ULA address'
     assert expected_stdout in detectPlatform.stdout
 
 
@@ -468,9 +467,9 @@ def test_IPv6_only_GUA(Pihole):
     )
     detectPlatform = Pihole.run('''
     source /opt/pihole/basic-install.sh
-    useIPv6dialog
+    find_IPv6_information
     ''')
-    expected_stdout = 'Found IPv6 GUA address, using it for blocking IPv6 ads'
+    expected_stdout = 'Found IPv6 GUA address'
     assert expected_stdout in detectPlatform.stdout
 
 
@@ -492,9 +491,9 @@ def test_IPv6_GUA_ULA_test(Pihole):
     )
     detectPlatform = Pihole.run('''
     source /opt/pihole/basic-install.sh
-    useIPv6dialog
+    find_IPv6_information
     ''')
-    expected_stdout = 'Found IPv6 ULA address, using it for blocking IPv6 ads'
+    expected_stdout = 'Found IPv6 ULA address'
     assert expected_stdout in detectPlatform.stdout
 
 
@@ -516,9 +515,9 @@ def test_IPv6_ULA_GUA_test(Pihole):
     )
     detectPlatform = Pihole.run('''
     source /opt/pihole/basic-install.sh
-    useIPv6dialog
+    find_IPv6_information
     ''')
-    expected_stdout = 'Found IPv6 ULA address, using it for blocking IPv6 ads'
+    expected_stdout = 'Found IPv6 ULA address'
     assert expected_stdout in detectPlatform.stdout
 
 

test/tox.fedora_34.ini

@@ -4,5 +4,5 @@ envlist = py37
 [testenv]
 whitelist_externals = docker
 deps = -rrequirements.txt
-commands = docker build -f _fedora_32.Dockerfile -t pytest_pihole:test_container ../
+commands = docker build -f _fedora_34.Dockerfile -t pytest_pihole:test_container ../
            pytest {posargs:-vv -n auto} ./test_automated_install.py ./test_centos_fedora_common_support.py ./test_fedora_support.py