Merge branch 'development' into use_utils

.github/workflows/stale.yml

@@ -13,7 +13,7 @@ jobs:
       issues: write
 
     steps:
-      - uses: actions/stale@v5.1.1
+      - uses: actions/stale@v5.2.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           days-before-stale: 30

.github/workflows/test.yml

@@ -43,7 +43,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        distro: [debian_10, debian_11, ubuntu_18, ubuntu_20, ubuntu_22, centos_8, fedora_34]
+        distro: [debian_10, debian_11, ubuntu_20, ubuntu_22, centos_8, fedora_34]
     env:
       DISTRO: ${{matrix.distro}}
     steps:
@@ -51,10 +51,10 @@ jobs:
       name: Checkout repository
       uses: actions/checkout@v3.0.2
     -
-      name: Set up Python 3.8
+      name: Set up Python 3.10
       uses: actions/setup-python@v4.2.0
       with:
-        python-version: 3.8
+        python-version: '3.10'
     -
       name: Install dependencies
       run: pip install -r test/requirements.txt

advanced/Scripts/piholeDebug.sh

@@ -41,6 +41,9 @@ else
     #OVER="\r\033[K"
 fi
 
+# shellcheck disable=SC1091
+. /etc/pihole/versions
+
 OBFUSCATED_PLACEHOLDER="<DOMAIN OBFUSCATED>"
 
 # FAQ URLs for use in showing the debug log
@@ -465,8 +468,8 @@ diagnose_operating_system() {
     # Display the current test that is running
     echo_current_diagnostic "Operating system"
 
-    # If the PIHOLE_DOCKER_TAG variable is set, include this information in the debug output
+    # If DOCKER_VERSION is set (Sourced from /etc/pihole/versions at start of script), include this information in the debug output
-    [ -n "${PIHOLE_DOCKER_TAG}" ] && log_write "${INFO} Pi-hole Docker Container: ${PIHOLE_DOCKER_TAG}"
+    [ -n "${DOCKER_VERSION}" ] && log_write "${INFO} Pi-hole Docker Container: ${DOCKER_VERSION}"
 
     # If there is a /etc/*release file, it's probably a supported operating system, so we can
     if ls /etc/*release 1> /dev/null 2>&1; then
@@ -802,7 +805,7 @@ check_networking() {
     ping_gateway "6"
     # Skip the following check if installed in docker container. Unpriv'ed containers do not have access to the information required
     # to resolve the service name listening - and the container should not start if there was a port conflict anyway
-    [ -z "${PIHOLE_DOCKER_TAG}" ] && check_required_ports
+    [ -z "${DOCKER_VERSION}" ] && check_required_ports
 }
 
 check_x_headers() {
@@ -812,7 +815,7 @@ check_x_headers() {
     # Similarly, it will show "X-Pi-hole: The Pi-hole Web interface is working!" if you view the header returned
     # when accessing the dashboard (i.e curl -I pi.hole/admin/)
     # server is operating correctly
-    echo_current_diagnostic "Dashboard and block page"
+    echo_current_diagnostic "Dashboard headers"
     # Use curl -I to get the header and parse out just the X-Pi-hole one
     local full_curl_output_dashboard
     local dashboard
@@ -822,7 +825,7 @@ check_x_headers() {
     local dashboard_working
     dashboard_working="X-Pi-hole: The Pi-hole Web interface is working!"
 
-    # Same logic applies to the dashboard as above, if the X-Header matches what a working system should have,
+    # If the X-Header matches what a working system should have,
     if [[ $dashboard == "$dashboard_working" ]]; then
         # then we can show a success
         log_write "$TICK Web interface X-Header: ${COL_GREEN}${dashboard}${COL_NC}"

advanced/Scripts/updatecheck.sh

@@ -8,23 +8,6 @@
 # This file is copyright under the latest version of the EUPL.
 # Please see LICENSE file for your rights under this license.
 
-# Credit: https://stackoverflow.com/a/46324904
-function json_extract() {
-    local key=$1
-    local json=$2
-
-    local string_regex='"([^"\]|\\.)*"'
-    local number_regex='-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?'
-    local value_regex="${string_regex}|${number_regex}|true|false|null"
-    local pair_regex="\"${key}\"[[:space:]]*:[[:space:]]*(${value_regex})"
-
-    if [[ ${json} =~ ${pair_regex} ]]; then
-        echo $(sed 's/^"\|"$//g' <<< "${BASH_REMATCH[1]}")
-    else
-        return 1
-    fi
-}
-
 function get_local_branch() {
     # Return active branch
     cd "${1}" 2> /dev/null || return 1
@@ -61,19 +44,19 @@ if [[ "$2" == "remote" ]]; then
         sleep 30
     fi
 
-    GITHUB_CORE_VERSION="$(json_extract tag_name "$(curl -s 'https://api.github.com/repos/pi-hole/pi-hole/releases/latest' 2> /dev/null)")"
+    GITHUB_CORE_VERSION="$(curl -s 'https://api.github.com/repos/pi-hole/pi-hole/releases/latest' 2> /dev/null | jq --raw-output .tag_name)"
     addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_CORE_VERSION" "${GITHUB_CORE_VERSION}"
 
     if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then
-        GITHUB_WEB_VERSION="$(json_extract tag_name "$(curl -s 'https://api.github.com/repos/pi-hole/AdminLTE/releases/latest' 2> /dev/null)")"
+        GITHUB_WEB_VERSION="$(curl -s 'https://api.github.com/repos/pi-hole/AdminLTE/releases/latest' 2> /dev/null | jq --raw-output .tag_name)"
         addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_VERSION" "${GITHUB_WEB_VERSION}"
     fi
 
-    GITHUB_FTL_VERSION="$(json_extract tag_name "$(curl -s 'https://api.github.com/repos/pi-hole/FTL/releases/latest' 2> /dev/null)")"
+    GITHUB_FTL_VERSION="$(curl -s 'https://api.github.com/repos/pi-hole/FTL/releases/latest' 2> /dev/null | jq --raw-output .tag_name)"
     addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_FTL_VERSION" "${GITHUB_FTL_VERSION}"
 
     if [[ "${PIHOLE_DOCKER_TAG}" ]]; then
-        GITHUB_DOCKER_VERSION="$(json_extract tag_name "$(curl -s 'https://api.github.com/repos/pi-hole/docker-pi-hole/releases/latest' 2> /dev/null)")"
+        GITHUB_DOCKER_VERSION="$(curl -s 'https://api.github.com/repos/pi-hole/docker-pi-hole/releases/latest' 2> /dev/null | jq --raw-output .tag_name)"
         addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_DOCKER_VERSION" "${GITHUB_DOCKER_VERSION}"
     fi
 
@@ -101,4 +84,9 @@ else
     FTL_VERSION="$(pihole-FTL version)"
     addOrEditKeyValPair "${VERSION_FILE}" "FTL_VERSION" "${FTL_VERSION}"
 
+    # PIHOLE_DOCKER_TAG is set as env variable only on docker installations
+    if [[ "${PIHOLE_DOCKER_TAG}" ]]; then
+        addOrEditKeyValPair "${VERSION_FILE}" "DOCKER_VERSION" "${PIHOLE_DOCKER_TAG}"
+    fi
+
 fi

advanced/lighttpd.conf.debian

@@ -32,6 +32,8 @@ server.errorlog             = "/var/log/lighttpd/error-pihole.log"
 server.pid-file             = "/run/lighttpd.pid"
 server.username             = "www-data"
 server.groupname            = "www-data"
+# For lighttpd version 1.4.46 or above, the port can be overwritten in `/etc/lighttpd/external.conf` using the := operator
+# e.g. server.port := 8000
 server.port                 = 80
 accesslog.filename          = "/var/log/lighttpd/access-pihole.log"
 accesslog.format            = "%{%s}t|%V|%r|%s|%b"

advanced/lighttpd.conf.fedora

@@ -33,6 +33,8 @@ server.errorlog             = "/var/log/lighttpd/error-pihole.log"
 server.pid-file             = "/run/lighttpd.pid"
 server.username             = "lighttpd"
 server.groupname            = "lighttpd"
+# For lighttpd version 1.4.46 or above, the port can be overwritten in `/etc/lighttpd/external.conf` using the := operator
+# e.g. server.port := 8000
 server.port                 = 80
 accesslog.filename          = "/var/log/lighttpd/access-pihole.log"
 accesslog.format            = "%{%s}t|%V|%r|%s|%b"

automated install/basic-install.sh

@@ -331,7 +331,7 @@ package_manager_detect() {
         # Packages required to run this install script (stored as an array)
         INSTALLER_DEPS=(git iproute2 dialog ca-certificates)
         # Packages required to run Pi-hole (stored as an array)
-        PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 libcap2-bin dns-root-data libcap2 netcat-openbsd procps)
+        PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq)
         # Packages required for the Web admin interface (stored as an array)
         # It's useful to separate this from Pi-hole, since the two repos are also setup separately
         PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-sqlite3" "${phpVer}-xml" "${phpVer}-intl")
@@ -361,7 +361,7 @@ package_manager_detect() {
         PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l || true"
         OS_CHECK_DEPS=(grep bind-utils)
         INSTALLER_DEPS=(git dialog iproute newt procps-ng which chkconfig ca-certificates)
-        PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc libcap nmap-ncat)
+        PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc libcap nmap-ncat jq)
         PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo php-xml php-json php-intl)
         LIGHTTPD_USER="lighttpd"
         LIGHTTPD_GROUP="lighttpd"

pihole

@@ -104,6 +104,9 @@ restartDNS() {
   local svcOption svc str output status pid icon FTL_PID_FILE
   svcOption="${1:-restart}"
 
+  # get the current path to the pihole-FTL.pid
+  FTL_PID_FILE="$(getFTLPIDFile)"
+
   # Determine if we should reload or restart
   if [[ "${svcOption}" =~ "reload-lists" ]]; then
     # Reloading of the lists has been requested
@@ -111,9 +114,6 @@ restartDNS() {
     # Note 2: We cannot use killall here as it does
     #         not know about real-time signals
 
-    # get the current path to the pihole-FTL.pid
-    FTL_PID_FILE="$(getFTLPIDFile)"
-
     pid="$(getFTLPID ${FTL_PID_FILE})"
     if [[ "$pid" -eq "-1" ]]; then
       svc="true"

test/_ubuntu_18.Dockerfile

@@ -1,17 +0,0 @@
-FROM buildpack-deps:bionic-scm
-
-ENV GITDIR /etc/.pihole
-ENV SCRIPTDIR /opt/pihole
-
-RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole
-ADD . $GITDIR
-RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/
-ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR
-
-RUN true && \
-    chmod +x $SCRIPTDIR/*
-
-ENV SKIP_INSTALL true
-ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net
-
-#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/setup.py

@@ -1,6 +1,7 @@
 from setuptools import setup
 
 setup(
+    py_modules=[],
     setup_requires=['pytest-runner'],
     tests_require=['pytest'],
 )

test/test_any_automated_install.py

@@ -881,13 +881,17 @@ def test_FTL_binary_installed_and_responsive_no_errors(host):
     source /opt/pihole/basic-install.sh
     create_pihole_user
     funcOutput=$(get_binary_name)
+    echo "development" > /etc/pihole/ftlbranch
     binary="pihole-FTL${funcOutput##*pihole-FTL}"
     theRest="${funcOutput%pihole-FTL*}"
     FTLdetect "${binary}" "${theRest}"
-    pihole-FTL version
+    ''')
+    version_check = host.run('''
+    VERSION=$(pihole-FTL version)
+    echo ${VERSION:0:1}
     ''')
     expected_stdout = 'v'
-    assert expected_stdout in installed_binary.stdout
+    assert expected_stdout in version_check.stdout
 
 
 def test_IPv6_only_link_local(host):

test/tox.centos_8.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py38
+envlist = py3
 
 [testenv]
 whitelist_externals = docker

test/tox.debian_10.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py38
+envlist = py3
 
 [testenv]
 whitelist_externals = docker

test/tox.debian_11.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py38
+envlist = py3
 
 [testenv]
 whitelist_externals = docker

test/tox.fedora_34.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py38
+envlist = py3
 
 [testenv]
 whitelist_externals = docker

test/tox.ubuntu_18.ini

@@ -1,8 +0,0 @@
-[tox]
-envlist = py38
-
-[testenv]
-whitelist_externals = docker
-deps = -rrequirements.txt
-commands = docker build -f _ubuntu_18.Dockerfile -t pytest_pihole:test_container ../
-           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py

test/tox.ubuntu_20.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py38
+envlist = py3
 
 [testenv]
 whitelist_externals = docker

test/tox.ubuntu_22.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py38
+envlist = py3
 
 [testenv]
 whitelist_externals = docker