From 9f31ab8a6fd2dfe361c2383f4b59648eb9cd6e3f Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 1 Jan 2023 12:34:40 +0000 Subject: [PATCH 001/413] Debug log does not need to check php/lighttpd Signed-off-by: Adam Warner --- advanced/Scripts/piholeDebug.sh | 63 +++++++-------------------------- 1 file changed, 12 insertions(+), 51 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index fa1cebbb..7462cba3 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -64,10 +64,8 @@ PIHOLE_SCRIPTS_DIRECTORY="/opt/pihole" BIN_DIRECTORY="/usr/local/bin" RUN_DIRECTORY="/run" LOG_DIRECTORY="/var/log/pihole" -WEB_SERVER_LOG_DIRECTORY="/var/log/lighttpd" -WEB_SERVER_CONFIG_DIRECTORY="/etc/lighttpd" -WEB_SERVER_CONFIG_DIRECTORY_FEDORA="${WEB_SERVER_CONFIG_DIRECTORY}/conf.d" -WEB_SERVER_CONFIG_DIRECTORY_DEBIAN="${WEB_SERVER_CONFIG_DIRECTORY}/conf-enabled" +#WEB_SERVER_LOG_DIRECTORY="/var/log/lighttpd" #TODO: FTL access log? +#WEB_SERVER_CONFIG_DIRECTORY="/etc/lighttpd" #TODO: FTL access log? HTML_DIRECTORY="/var/www/html" WEB_GIT_DIRECTORY="${HTML_DIRECTORY}/admin" SHM_DIRECTORY="/dev/shm" @@ -77,10 +75,8 @@ ETC="/etc" # https://discourse.pi-hole.net/t/what-files-does-pi-hole-use/1684 PIHOLE_CRON_FILE="${CRON_D_DIRECTORY}/pihole" -WEB_SERVER_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/lighttpd.conf" -WEB_SERVER_CUSTOM_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/external.conf" -WEB_SERVER_PIHOLE_CONFIG_FILE_DEBIAN="${WEB_SERVER_CONFIG_DIRECTORY_DEBIAN}/15-pihole-admin.conf" -WEB_SERVER_PIHOLE_CONFIG_FILE_FEDORA="${WEB_SERVER_CONFIG_DIRECTORY_FEDORA}/pihole-admin.conf" +#WEB_SERVER_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/lighttpd.conf" +#WEB_SERVER_CUSTOM_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/external.conf" PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" @@ -131,21 +127,19 @@ PIHOLE_LOG_GZIPS="${LOG_DIRECTORY}/pihole.log.[0-9].*" PIHOLE_DEBUG_LOG="${LOG_DIRECTORY}/pihole_debug.log" PIHOLE_FTL_LOG="$(get_ftl_conf_value "LOGFILE" "${LOG_DIRECTORY}/FTL.log")" -PIHOLE_WEB_SERVER_ACCESS_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/access-pihole.log" -PIHOLE_WEB_SERVER_ERROR_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/error-pihole.log" +# PIHOLE_WEB_SERVER_ACCESS_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/access-pihole.log" #TODO: FTL Error log? +# PIHOLE_WEB_SERVER_ERROR_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/error-pihole.log" #TODO: FTL Error log? RESOLVCONF="${ETC}/resolv.conf" DNSMASQ_CONF="${ETC}/dnsmasq.conf" # Store Pi-hole's processes in an array for easy use and parsing -PIHOLE_PROCESSES=( "lighttpd" "pihole-FTL" ) +PIHOLE_PROCESSES=( "pihole-FTL" ) # Store the required directories in an array so it can be parsed through REQUIRED_FILES=("${PIHOLE_CRON_FILE}" -"${WEB_SERVER_CONFIG_FILE}" -"${WEB_SERVER_CUSTOM_CONFIG_FILE}" -"${WEB_SERVER_PIHOLE_CONFIG_FILE_DEBIAN}" -"${WEB_SERVER_PIHOLE_CONFIG_FILE_FEDORA}" +# "${WEB_SERVER_CONFIG_FILE}" +# "${WEB_SERVER_CUSTOM_CONFIG_FILE}" "${PIHOLE_INSTALL_LOG_FILE}" "${PIHOLE_RAW_BLOCKLIST_FILES}" "${PIHOLE_LOCAL_HOSTS_FILE}" @@ -351,39 +345,6 @@ check_component_versions() { check_ftl_version } - -get_program_version() { - local program_name="${1}" - # Create a local variable so this function can be safely reused - local program_version - echo_current_diagnostic "${program_name} version" - # Evaluate the program we are checking, if it is any of the ones below, show the version - case "${program_name}" in - "lighttpd") program_version="$(${program_name} -v 2> /dev/null | head -n1 | cut -d '/' -f2 | cut -d ' ' -f1)" - ;; - "php") program_version="$(${program_name} -v 2> /dev/null | head -n1 | cut -d '-' -f1 | cut -d ' ' -f2)" - ;; - # If a match is not found, show an error - *) echo "Unrecognized program"; - esac - # If the program does not have a version (the variable is empty) - if [[ -z "${program_version}" ]]; then - # Display and error - log_write "${CROSS} ${COL_RED}${program_name} version could not be detected.${COL_NC}" - else - # Otherwise, display the version - log_write "${INFO} ${program_version}" - fi -} - -# These are the most critical dependencies of Pi-hole, so we check for them -# and their versions, using the functions above. -check_critical_program_versions() { - # Use the function created earlier and bundle them into one function that checks all the version numbers - get_program_version "lighttpd" - get_program_version "php" -} - os_check() { # This function gets a list of supported OS versions from a TXT record at versions.pi-hole.net # and determines whether or not the script is running on one of those systems @@ -737,10 +698,10 @@ compare_port_to_service_assigned() { check_required_ports() { echo_current_diagnostic "Ports in use" - # Since Pi-hole needs 53, 80, and 4711, check what they are being used by + # Since Pi-hole needs 53 and 4711, check what they are being used by # so we can detect any issues local resolver="pihole-FTL" - local web_server="lighttpd" + local web_server="pihole-FTL" local ftl="pihole-FTL" # Create an array for these ports in use ports_in_use=() @@ -1520,7 +1481,7 @@ initialize_debug # available to the other functions source_setup_variables check_component_versions -check_critical_program_versions +# check_critical_program_versions diagnose_operating_system check_selinux check_firewalld From 31ee15200d89ffd27fc16819ecebf3c360345993 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 1 Jan 2023 12:45:14 +0000 Subject: [PATCH 002/413] gut the install script of references to web server/lighttpd Signed-off-by: Adam Warner --- automated install/basic-install.sh | 233 ----------------------------- 1 file changed, 233 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 24fe9e54..3615804c 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -58,8 +58,6 @@ EOM installLogLoc="/etc/pihole/install.log" # This is an important file as it contains information specific to the machine it's being installed on setupVars="/etc/pihole/setupVars.conf" -# Pi-hole uses lighttpd as a Web server, and this is the config file for it -lighttpdConfig="/etc/lighttpd/lighttpd.conf" # This is a file used for the colorized output coltable="/opt/pihole/COL_TABLE" @@ -110,13 +108,11 @@ c=70 # The runUnattended flag is one example of this reconfigure=false runUnattended=false -INSTALL_WEB_SERVER=true # Check arguments for the undocumented flags for var in "$@"; do case "$var" in "--reconfigure" ) reconfigure=true;; "--unattended" ) runUnattended=true;; - "--disable-install-webserver" ) INSTALL_WEB_SERVER=false;; esac done @@ -308,42 +304,12 @@ package_manager_detect() { PKG_COUNT="${PKG_MANAGER} -s -o Debug::NoLocking=true upgrade | grep -c ^Inst || true" # Update package cache update_package_cache || exit 1 - # Check for and determine version number (major and minor) of current php install - local phpVer="php" - if is_command php ; then - phpVer="$(php <<< "")" - # Check if the first character of the string is numeric - if [[ ${phpVer:0:1} =~ [1-9] ]]; then - printf " %b Existing PHP installation detected : PHP version %s\\n" "${INFO}" "${phpVer}" - printf -v phpInsMajor "%d" "$(php <<< "")" - printf -v phpInsMinor "%d" "$(php <<< "")" - phpVer="php$phpInsMajor.$phpInsMinor" - else - printf " %b No valid PHP installation detected!\\n" "${CROSS}" - printf " %b PHP version : %s\\n" "${INFO}" "${phpVer}" - printf " %b Aborting installation.\\n" "${CROSS}" - exit 1 - fi - fi # Packages required to perform the os_check (stored as an array) OS_CHECK_DEPS=(grep dnsutils) # Packages required to run this install script (stored as an array) INSTALLER_DEPS=(git iproute2 dialog ca-certificates) # Packages required to run Pi-hole (stored as an array) PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) - # Packages required for the Web admin interface (stored as an array) - # It's useful to separate this from Pi-hole, since the two repos are also setup separately - PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-sqlite3" "${phpVer}-xml" "${phpVer}-intl") - # Prior to PHP8.0, JSON functionality is provided as dedicated module, required by Pi-hole AdminLTE: https://www.php.net/manual/json.installation.php - if [[ -z "${phpInsMajor}" || "${phpInsMajor}" -lt 8 ]]; then - PIHOLE_WEB_DEPS+=("${phpVer}-json") - fi - # The Web server user, - LIGHTTPD_USER="www-data" - # group, - LIGHTTPD_GROUP="www-data" - # and config file - LIGHTTPD_CFG="lighttpd.conf.debian" # If apt-get is not found, check for rpm. elif is_command rpm ; then @@ -361,25 +327,6 @@ package_manager_detect() { OS_CHECK_DEPS=(grep bind-utils) INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates) PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc libcap nmap-ncat jq) - PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo php-xml php-json php-intl) - LIGHTTPD_USER="lighttpd" - LIGHTTPD_GROUP="lighttpd" - LIGHTTPD_CFG="lighttpd.conf.fedora" - - # If the host OS is centos (or a derivative), epel is required for lighttpd - if ! grep -qiE 'fedora|fedberry' /etc/redhat-release; then - if rpm -qa | grep -qi 'epel'; then - printf " %b EPEL repository already installed\\n" "${TICK}" - else - local RH_RELEASE EPEL_PKG - # EPEL not already installed, add it based on the release version - RH_RELEASE=$(grep -oP '(?<= )[0-9]+(?=\.?)' /etc/redhat-release) - EPEL_PKG="https://dl.fedoraproject.org/pub/epel/epel-release-latest-${RH_RELEASE}.noarch.rpm" - printf " %b Enabling EPEL package repository (https://fedoraproject.org/wiki/EPEL)\\n" "${INFO}" - "${PKG_INSTALL[@]}" "${EPEL_PKG}" - printf " %b Installed %s\\n" "${TICK}" "${EPEL_PKG}" - fi - fi # If neither apt-get or yum/dnf package managers were found else @@ -1121,7 +1068,6 @@ setAdminFlag() { printf " %b Not installing Admin Web Interface\\n" "${INFO}" # Set the flag to not install the web interface INSTALL_WEB_INTERFACE=false - INSTALL_WEB_SERVER=false ;; "${DIALOG_ESC}") # User pressed @@ -1129,43 +1075,6 @@ setAdminFlag() { exit 1 ;; esac - - # If the user wants to install the Web admin interface (i.e. it has not been deselected above) and did not deselect the web server via command-line argument - if [[ "${INSTALL_WEB_INTERFACE}" == true && "${INSTALL_WEB_SERVER}" == true ]]; then - # Get list of required PHP modules, excluding base package (common) and handler (cgi) - local i php_modules - for i in "${PIHOLE_WEB_DEPS[@]}"; do [[ $i == 'php'* && $i != *'-common' && $i != *'-cgi' ]] && php_modules+=" ${i#*-}"; done - dialog --no-shadow --keep-tite \ - --backtitle "Pi-hole Installation" \ - --title "Web Server" \ - --yesno "\\n\\nA web server is required for the Admin Web Interface.\ -\\n\\nDo you want to install lighttpd and the required PHP modules?\ -\\n\\nNB: If you disable this, and, do not have an existing web server \ -and required PHP modules (${php_modules# }) installed, the web interface \ -will not function. Additionally the web server user needs to be member of \ -the \"pihole\" group for full functionality." \ - "${r}" "${c}" && result=0 || result=$? - - case ${result} in - "${DIALOG_OK}") - # If they chose yes, - printf " %b Installing lighttpd\\n" "${INFO}" - # Set the flag to install the web server - INSTALL_WEB_SERVER=true - ;; - "${DIALOG_CANCEL}") - # If they chose no, - printf " %b Not installing lighttpd\\n" "${INFO}" - # Set the flag to not install the web server - INSTALL_WEB_SERVER=false - ;; - "${DIALOG_ESC}") - # User pressed - printf " %b Escape pressed, exiting installer at web server choice.%b\\n" "${COL_LIGHT_RED}" "${COL_NC}" - exit 1 - ;; - esac - fi } # A function to display a list of example blocklists for users to select @@ -1397,79 +1306,6 @@ installConfigs() { fi install -T -m 0755 "${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole-FTL-prestart.sh" "${PI_HOLE_INSTALL_DIR}/pihole-FTL-prestart.sh" install -T -m 0755 "${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole-FTL-poststop.sh" "${PI_HOLE_INSTALL_DIR}/pihole-FTL-poststop.sh" - - # If the user chose to install the dashboard, - if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - # set permissions on /etc/lighttpd/lighttpd.conf so pihole user (other) can read the file - chmod o+x /etc/lighttpd - chmod o+r "${lighttpdConfig}" - - # Ensure /run/lighttpd exists and is owned by lighttpd user - # Needed for the php socket - mkdir -p /run/lighttpd - chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} /run/lighttpd - - if grep -q -F "OVERWRITTEN BY PI-HOLE" "${lighttpdConfig}"; then - # Attempt to preserve backwards compatibility with older versions - install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} "${lighttpdConfig}" - # Make the directories if they do not exist and set the owners - mkdir -p /var/cache/lighttpd/compress - chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} /var/cache/lighttpd/compress - mkdir -p /var/cache/lighttpd/uploads - chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} /var/cache/lighttpd/uploads - fi - # Copy the config file to include for pihole admin interface - if [[ -d "/etc/lighttpd/conf.d" ]]; then - install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/pihole-admin.conf /etc/lighttpd/conf.d/pihole-admin.conf - if grep -q -F 'include "/etc/lighttpd/conf.d/pihole-admin.conf"' "${lighttpdConfig}"; then - : - else - echo 'include "/etc/lighttpd/conf.d/pihole-admin.conf"' >> "${lighttpdConfig}" - fi - # Avoid some warnings trace from lighttpd, which might break tests - conf=/etc/lighttpd/conf.d/pihole-admin.conf - if lighttpd -f "${lighttpdConfig}" -tt 2>&1 | grep -q -F "WARNING: unknown config-key: dir-listing\."; then - echo '# Avoid some warnings trace from lighttpd, which might break tests' >> $conf - echo 'server.modules += ( "mod_dirlisting" )' >> $conf - fi - if lighttpd -f "${lighttpdConfig}" -tt 2>&1 | grep -q -F "warning: please use server.use-ipv6"; then - echo '# Avoid some warnings trace from lighttpd, which might break tests' >> $conf - echo 'server.use-ipv6 := "disable"' >> $conf - fi - elif [[ -d "/etc/lighttpd/conf-available" ]]; then - conf=/etc/lighttpd/conf-available/15-pihole-admin.conf - install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/pihole-admin.conf $conf - - # Get the version number of lighttpd - version=$(dpkg-query -f='${Version}\n' --show lighttpd) - # Test if that version is greater than or euqal to 1.4.56 - if dpkg --compare-versions "$version" "ge" "1.4.56"; then - # If it is, then we don't need to disable the modules - # (server.modules duplication is ignored in lighttpd 1.4.56+) - : - else - # disable server.modules += ( ... ) in $conf to avoid module dups - if awk '!/^server\.modules/{print}' $conf > $conf.$$ && mv $conf.$$ $conf; then - : - else - rm $conf.$$ - fi - fi - - chmod 644 $conf - if is_command lighty-enable-mod ; then - lighty-enable-mod pihole-admin access accesslog redirect fastcgi setenv > /dev/null || true - else - # Otherwise, show info about installing them - printf " %b Warning: 'lighty-enable-mod' utility not found\\n" "${INFO}" - printf " Please ensure fastcgi is enabled if you experience issues\\n" - fi - else - # lighttpd config include dir not found - printf " %b Warning: lighttpd config include dir not found\\n" "${INFO}" - printf " Please manually install pihole-admin.conf\\n" - fi - fi } install_manpage() { @@ -1717,29 +1553,6 @@ install_dependent_packages() { return 0 } -# Install the Web interface dashboard -installPiholeWeb() { - # Install Sudoers file - local str="Installing sudoer file" - printf "\\n %b %s..." "${INFO}" "${str}" - # Make the .d directory if it doesn't exist, - install -d -m 755 /etc/sudoers.d/ - # and copy in the pihole sudoers file - install -m 0640 ${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole.sudo /etc/sudoers.d/pihole - # Add lighttpd user (OS dependent) to sudoers file - echo "${LIGHTTPD_USER} ALL=NOPASSWD: ${PI_HOLE_BIN_DIR}/pihole" >> /etc/sudoers.d/pihole - - # If the Web server user is lighttpd, - if [[ "$LIGHTTPD_USER" == "lighttpd" ]]; then - # Allow executing pihole via sudo with Fedora - # Usually /usr/local/bin ${PI_HOLE_BIN_DIR} is not permitted as directory for sudoable programs - echo "Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin:${PI_HOLE_BIN_DIR}" >> /etc/sudoers.d/pihole - fi - # Set the strict permissions on the file - chmod 0440 /etc/sudoers.d/pihole - printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" -} - # Installs a cron file installCron() { # Install the cron job @@ -1838,9 +1651,7 @@ finalExports() { addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_1" "${PIHOLE_DNS_1}" addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_2" "${PIHOLE_DNS_2}" addOrEditKeyValPair "${setupVars}" "QUERY_LOGGING" "${QUERY_LOGGING}" - addOrEditKeyValPair "${setupVars}" "INSTALL_WEB_SERVER" "${INSTALL_WEB_SERVER}" addOrEditKeyValPair "${setupVars}" "INSTALL_WEB_INTERFACE" "${INSTALL_WEB_INTERFACE}" - addOrEditKeyValPair "${setupVars}" "LIGHTTPD_ENABLED" "${LIGHTTPD_ENABLED}" addOrEditKeyValPair "${setupVars}" "CACHE_SIZE" "${CACHE_SIZE}" addOrEditKeyValPair "${setupVars}" "DNS_FQDN_REQUIRED" "${DNS_FQDN_REQUIRED:-true}" addOrEditKeyValPair "${setupVars}" "DNS_BOGUS_PRIV" "${DNS_BOGUS_PRIV:-true}" @@ -1904,25 +1715,6 @@ installLogrotate() { # Install base files and web interface installPihole() { - # If the user wants to install the Web interface, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - if [[ ! -d "${webroot}" ]]; then - # make the Web directory if necessary - install -d -m 0755 ${webroot} - fi - - if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - # Set the owner and permissions - chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} ${webroot} - chmod 0775 ${webroot} - # Repair permissions if webroot is not world readable - chmod a+rx /var/www - chmod a+rx ${webroot} - # Give lighttpd access to the pihole group so the web interface can - # manage the gravity.db database - usermod -a -G pihole ${LIGHTTPD_USER} - fi - fi # Install base files and web interface if ! installScripts; then printf " %b Failure in dependent script copy function.\\n" "${CROSS}" @@ -2624,27 +2416,12 @@ main() { # Install the Core dependencies local dep_install_list=("${PIHOLE_DEPS[@]}") - if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - # And, if the setting says so, install the Web admin interface dependencies - dep_install_list+=("${PIHOLE_WEB_DEPS[@]}") - fi # Install packages used by the actual software printf " %b Checking for / installing Required dependencies for Pi-hole software...\\n" "${INFO}" install_dependent_packages "${dep_install_list[@]}" unset dep_install_list - # On some systems, lighttpd is not enabled on first install. We need to enable it here if the user - # has chosen to install the web interface, else the LIGHTTPD_ENABLED check will fail - if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - enable_service lighttpd - fi - # Determine if lighttpd is correctly enabled - if check_service_active "lighttpd"; then - LIGHTTPD_ENABLED=true - else - LIGHTTPD_ENABLED=false - fi # Create the pihole user create_pihole_user @@ -2685,16 +2462,6 @@ main() { # but before starting or resarting the dnsmasq or ftl services disable_resolved_stublistener - # If the Web server was installed, - if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - if [[ "${LIGHTTPD_ENABLED}" == true ]]; then - restart_service lighttpd - enable_service lighttpd - else - printf " %b Lighttpd is disabled, skipping service restart\\n" "${INFO}" - fi - fi - printf " %b Restarting services...\\n" "${INFO}" # Start services From c520b293264f1761cd008e73f4ff3d5d22db8822 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 1 Jan 2023 12:47:00 +0000 Subject: [PATCH 003/413] some more lighttpd refs (tests can come later) Signed-off-by: Adam Warner --- advanced/Scripts/piholeDebug.sh | 57 ++++++++++----------- advanced/lighttpd.conf.debian | 73 --------------------------- advanced/lighttpd.conf.fedora | 87 --------------------------------- advanced/pihole-admin.conf | 82 ------------------------------- 4 files changed, 26 insertions(+), 273 deletions(-) delete mode 100644 advanced/lighttpd.conf.debian delete mode 100644 advanced/lighttpd.conf.fedora delete mode 100644 advanced/pihole-admin.conf diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 7462cba3..9f895aab 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -64,8 +64,6 @@ PIHOLE_SCRIPTS_DIRECTORY="/opt/pihole" BIN_DIRECTORY="/usr/local/bin" RUN_DIRECTORY="/run" LOG_DIRECTORY="/var/log/pihole" -#WEB_SERVER_LOG_DIRECTORY="/var/log/lighttpd" #TODO: FTL access log? -#WEB_SERVER_CONFIG_DIRECTORY="/etc/lighttpd" #TODO: FTL access log? HTML_DIRECTORY="/var/www/html" WEB_GIT_DIRECTORY="${HTML_DIRECTORY}/admin" SHM_DIRECTORY="/dev/shm" @@ -75,9 +73,6 @@ ETC="/etc" # https://discourse.pi-hole.net/t/what-files-does-pi-hole-use/1684 PIHOLE_CRON_FILE="${CRON_D_DIRECTORY}/pihole" -#WEB_SERVER_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/lighttpd.conf" -#WEB_SERVER_CUSTOM_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/external.conf" - PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" @@ -762,34 +757,34 @@ check_networking() { [ -z "${DOCKER_VERSION}" ] && check_required_ports } -check_x_headers() { - # The X-Headers allow us to determine from the command line if the Web - # lighttpd.conf has a directive to show "X-Pi-hole: A black hole for Internet advertisements." - # in the header of any Pi-holed domain - # Similarly, it will show "X-Pi-hole: The Pi-hole Web interface is working!" if you view the header returned - # when accessing the dashboard (i.e curl -I pi.hole/admin/) - # server is operating correctly - echo_current_diagnostic "Dashboard headers" - # Use curl -I to get the header and parse out just the X-Pi-hole one - local full_curl_output_dashboard - local dashboard - full_curl_output_dashboard="$(curl -Is localhost/admin/)" - dashboard=$(echo "${full_curl_output_dashboard}" | awk '/X-Pi-hole/' | tr -d '\r') - # Store what the X-Header should be in variables for comparison later - local dashboard_working - dashboard_working="X-Pi-hole: The Pi-hole Web interface is working!" +# check_x_headers() { +# # The X-Headers allow us to determine from the command line if the Web +# # lighttpd.conf has a directive to show "X-Pi-hole: A black hole for Internet advertisements." +# # in the header of any Pi-holed domain +# # Similarly, it will show "X-Pi-hole: The Pi-hole Web interface is working!" if you view the header returned +# # when accessing the dashboard (i.e curl -I pi.hole/admin/) +# # server is operating correctly +# echo_current_diagnostic "Dashboard headers" +# # Use curl -I to get the header and parse out just the X-Pi-hole one +# local full_curl_output_dashboard +# local dashboard +# full_curl_output_dashboard="$(curl -Is localhost/admin/)" +# dashboard=$(echo "${full_curl_output_dashboard}" | awk '/X-Pi-hole/' | tr -d '\r') +# # Store what the X-Header should be in variables for comparison later +# local dashboard_working +# dashboard_working="X-Pi-hole: The Pi-hole Web interface is working!" - # If the X-Header matches what a working system should have, - if [[ $dashboard == "$dashboard_working" ]]; then - # then we can show a success - log_write "$TICK Web interface X-Header: ${COL_GREEN}${dashboard}${COL_NC}" - else - # Otherwise, it's a failure since the X-Headers either don't exist or have been modified in some way - log_write "$CROSS Web interface X-Header: ${COL_RED}X-Header does not match or could not be retrieved.${COL_NC}" +# # If the X-Header matches what a working system should have, +# if [[ $dashboard == "$dashboard_working" ]]; then +# # then we can show a success +# log_write "$TICK Web interface X-Header: ${COL_GREEN}${dashboard}${COL_NC}" +# else +# # Otherwise, it's a failure since the X-Headers either don't exist or have been modified in some way +# log_write "$CROSS Web interface X-Header: ${COL_RED}X-Header does not match or could not be retrieved.${COL_NC}" - log_write "${COL_RED}${full_curl_output_dashboard}${COL_NC}" - fi -} +# log_write "${COL_RED}${full_curl_output_dashboard}${COL_NC}" +# fi +# } dig_at() { # We need to test if Pi-hole can properly resolve domain names diff --git a/advanced/lighttpd.conf.debian b/advanced/lighttpd.conf.debian deleted file mode 100644 index f31f7bcd..00000000 --- a/advanced/lighttpd.conf.debian +++ /dev/null @@ -1,73 +0,0 @@ -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Lighttpd config for Pi-hole -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. - -################################################################################################### -# IF THIS HEADER EXISTS, THE FILE WILL BE OVERWRITTEN BY PI-HOLE'S UPDATE PROCEDURE. # -# ANY CHANGES MADE TO THIS FILE WILL BE LOST ON THE NEXT UPDATE UNLESS YOU REMOVE THIS HEADER # -# # -# ENSURE THAT YOU DO NOT REMOVE THE REQUIRED LINE: # -# # -# include "/etc/lighttpd/conf-enabled/*.conf" # -# # -################################################################################################### - -server.modules = ( - "mod_access", - "mod_auth", - "mod_expire", - "mod_redirect", - "mod_setenv", - "mod_rewrite" -) - -server.document-root = "/var/www/html" -server.upload-dirs = ( "/var/cache/lighttpd/uploads" ) -server.errorlog = "/var/log/lighttpd/error-pihole.log" -server.pid-file = "/run/lighttpd.pid" -server.username = "www-data" -server.groupname = "www-data" -# For lighttpd version 1.4.46 or above, the port can be overwritten in `/etc/lighttpd/external.conf` using the := operator -# e.g. server.port := 8000 -server.port = 80 - -# Allow streaming response -# reference: https://redmine.lighttpd.net/projects/lighttpd/wiki/Server_stream-response-bodyDetails -server.stream-response-body = 1 -#ssl.read-ahead = "disable" - -index-file.names = ( "index.php", "index.html", "index.lighttpd.html" ) -url.access-deny = ( "~", ".inc", ".md", ".yml", ".ini" ) -static-file.exclude-extensions = ( ".php", ".pl", ".fcgi" ) - -mimetype.assign = ( - ".ico" => "image/x-icon", - ".jpeg" => "image/jpeg", - ".jpg" => "image/jpeg", - ".png" => "image/png", - ".svg" => "image/svg+xml", - ".css" => "text/css; charset=utf-8", - ".html" => "text/html; charset=utf-8", - ".js" => "text/javascript; charset=utf-8", - ".json" => "application/json; charset=utf-8", - ".map" => "application/json; charset=utf-8", - ".txt" => "text/plain; charset=utf-8", - ".eot" => "application/vnd.ms-fontobject", - ".otf" => "font/otf", - ".ttc" => "font/collection", - ".ttf" => "font/ttf", - ".woff" => "font/woff", - ".woff2" => "font/woff2" -) - -# Add user chosen options held in (optional) external file -include "external*.conf" - -# default listening port for IPv6 falls back to the IPv4 port -include_shell "/usr/share/lighttpd/use-ipv6.pl " + server.port -include "/etc/lighttpd/conf-enabled/*.conf" diff --git a/advanced/lighttpd.conf.fedora b/advanced/lighttpd.conf.fedora deleted file mode 100644 index e09d7760..00000000 --- a/advanced/lighttpd.conf.fedora +++ /dev/null @@ -1,87 +0,0 @@ -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Lighttpd config for Pi-hole -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. - -################################################################################################### -# IF THIS HEADER EXISTS, THE FILE WILL BE OVERWRITTEN BY PI-HOLE'S UPDATE PROCEDURE. # -# ANY CHANGES MADE TO THIS FILE WILL BE LOST ON THE NEXT UPDATE UNLESS YOU REMOVE THIS HEADER # -# # -# ENSURE THAT YOU DO NOT REMOVE THE REQUIRED LINE: # -# # -# include "/etc/lighttpd/conf.d/pihole-admin.conf" # -# # -################################################################################################### - -server.modules = ( - "mod_access", - "mod_auth", - "mod_expire", - "mod_fastcgi", - "mod_accesslog", - "mod_redirect", - "mod_setenv", - "mod_rewrite" -) - -server.document-root = "/var/www/html" -server.upload-dirs = ( "/var/cache/lighttpd/uploads" ) -server.errorlog = "/var/log/lighttpd/error-pihole.log" -server.pid-file = "/run/lighttpd.pid" -server.username = "lighttpd" -server.groupname = "lighttpd" -# For lighttpd version 1.4.46 or above, the port can be overwritten in `/etc/lighttpd/external.conf` using the := operator -# e.g. server.port := 8000 -server.port = 80 - -# Allow streaming response -# reference: https://redmine.lighttpd.net/projects/lighttpd/wiki/Server_stream-response-bodyDetails -server.stream-response-body = 1 -#ssl.read-ahead = "disable" - -index-file.names = ( "index.php", "index.html", "index.lighttpd.html" ) -url.access-deny = ( "~", ".inc", ".md", ".yml", ".ini" ) -static-file.exclude-extensions = ( ".php", ".pl", ".fcgi" ) - -mimetype.assign = ( - ".ico" => "image/x-icon", - ".jpeg" => "image/jpeg", - ".jpg" => "image/jpeg", - ".png" => "image/png", - ".svg" => "image/svg+xml", - ".css" => "text/css; charset=utf-8", - ".html" => "text/html; charset=utf-8", - ".js" => "text/javascript; charset=utf-8", - ".json" => "application/json; charset=utf-8", - ".map" => "application/json; charset=utf-8", - ".txt" => "text/plain; charset=utf-8", - ".eot" => "application/vnd.ms-fontobject", - ".otf" => "font/otf", - ".ttc" => "font/collection", - ".ttf" => "font/ttf", - ".woff" => "font/woff", - ".woff2" => "font/woff2" -) - -# Add user chosen options held in (optional) external file -include "external*.conf" - -# default listening port for IPv6 falls back to the IPv4 port -#include_shell "/usr/share/lighttpd/use-ipv6.pl " + server.port -#include_shell "/usr/share/lighttpd/create-mime.assign.pl" -#include_shell "/usr/share/lighttpd/include-conf-enabled.pl" - -fastcgi.server = ( - ".php" => ( - "localhost" => ( - "socket" => "/tmp/php-fastcgi.socket", - "bin-path" => "/usr/bin/php-cgi" - ) - ) -) - -include "/etc/lighttpd/conf.d/pihole-admin.conf" diff --git a/advanced/pihole-admin.conf b/advanced/pihole-admin.conf deleted file mode 100644 index 0bb6eac9..00000000 --- a/advanced/pihole-admin.conf +++ /dev/null @@ -1,82 +0,0 @@ -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Lighttpd config for Pi-hole -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. - -############################################################################### -# FILE AUTOMATICALLY OVERWRITTEN BY PI-HOLE INSTALL/UPDATE PROCEDURE. # -# ANY CHANGES MADE TO THIS FILE AFTER INSTALL WILL BE LOST ON THE NEXT UPDATE # -############################################################################### - -server.errorlog := "/var/log/lighttpd/error-pihole.log" - -$HTTP["url"] =~ "^/admin/" { - server.document-root = "/var/www/html" - server.stream-response-body = 1 - accesslog.filename = "/var/log/lighttpd/access-pihole.log" - accesslog.format = "%{%s}t|%h|%V|%r|%s|%b" - - fastcgi.server = ( - ".php" => ( - "localhost" => ( - "socket" => "/run/lighttpd/pihole-php-fastcgi.socket", - "bin-path" => "/usr/bin/php-cgi", - "min-procs" => 1, - "max-procs" => 1, - "bin-environment" => ( - "PHP_FCGI_CHILDREN" => "4", - "PHP_FCGI_MAX_REQUESTS" => "10000", - ), - "bin-copy-environment" => ( - "PATH", "SHELL", "USER" - ), - "broken-scriptfilename" => "enable", - ) - ) - ) - - # X-Pi-hole is a response header for debugging using curl -I - # X-Frame-Options prevents clickjacking attacks and helps ensure your content is not embedded into other sites via < frame >, < iframe > or < object >. - # X-XSS-Protection sets the configuration for the cross-site scripting filters built into most browsers. This is important because it tells the browser to block the response if a malicious script has been inserted from a user input. (deprecated; disabled) - # X-Content-Type-Options stops a browser from trying to MIME-sniff the content type and forces it to stick with the declared content-type. This is important because the browser will only load external resources if their content-type matches what is expected, and not malicious hidden code. - # Content-Security-Policy tells the browser where resources are allowed to be loaded and if it’s allowed to parse/run inline styles or Javascript. This is important because it prevents content injection attacks, such as Cross Site Scripting (XSS). - # X-Permitted-Cross-Domain-Policies is an XML document that grants a web client, such as Adobe Flash Player or Adobe Acrobat (though not necessarily limited to these), permission to handle data across domains. - # Referrer-Policy allows control/restriction of the amount of information present in the referral header for links away from your page—the URL path or even if the header is sent at all. - setenv.add-response-header = ( - "X-Pi-hole" => "The Pi-hole Web interface is working!", - "X-Frame-Options" => "DENY", - "X-XSS-Protection" => "0", - "X-Content-Type-Options" => "nosniff", - "Content-Security-Policy" => "default-src 'self' 'unsafe-inline';", - "X-Permitted-Cross-Domain-Policies" => "none", - "Referrer-Policy" => "same-origin" - ) - - # Block . files from being served, such as .git, .github, .gitignore - $HTTP["url"] =~ "^/admin/\." { - url.access-deny = ("") - } - - # allow teleporter and API qr code iframe on settings page - $HTTP["url"] =~ "/(teleporter|api_token)\.php$" { - $HTTP["referer"] =~ "/admin/settings\.php" { - setenv.set-response-header = ( "X-Frame-Options" => "SAMEORIGIN" ) - } - } -} -else $HTTP["url"] == "/admin" { - url.redirect = ("" => "/admin/") -} - -$HTTP["host"] == "pi.hole" { - $HTTP["url"] == "/" { - url.redirect = ("" => "/admin/") - } -} - -# (keep this on one line for basic-install.sh filtering during install) -server.modules += ( "mod_access", "mod_accesslog", "mod_redirect", "mod_fastcgi", "mod_setenv" ) From d2d11959289fedfc0490921269a35e864742cf82 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 1 Jan 2023 12:49:07 +0000 Subject: [PATCH 004/413] Needs tweaking, offer chance to change web interface port Signed-off-by: Adam Warner --- automated install/basic-install.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 3615804c..d9e22e08 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -92,6 +92,7 @@ IPV6_ADDRESS=${IPV6_ADDRESS} # Give settings their default values. These may be changed by prompts later in the script. QUERY_LOGGING=true INSTALL_WEB_INTERFACE=true +WEBPORT=8080 PRIVACY_LEVEL=0 CACHE_SIZE=10000 @@ -1062,6 +1063,14 @@ setAdminFlag() { printf " %b Installing Admin Web Interface\\n" "${INFO}" # Set the flag to install the web interface INSTALL_WEB_INTERFACE=true + + # Web port TODO: Below whiptail copy pasted from a previous go at this. needs converting to dialog + # Ask for the IPv4 address + WEBPORT=$(whiptail --backtitle "Setting web interface port" --title "Web Port" --inputbox "By default, pihole-FTL listens for http traffic on port 8080. If you wish to change the port, you may do so now. You can also do it later by editing /etc/pihole/pihole-FTL.conf" "${r}" "${c}" "${WEBPORT}" 3>&1 1>&2 2>&3) || \ + # Canceling IPv4 settings window + { echo -e " ${COL_LIGHT_RED}Cancel was selected, exiting installer${COL_NC}"; exit 1; } + printf " %b The Web interface will be accessible on port: %s\\n" "${INFO}" "${WEBPORT}" + ;; "${DIALOG_CANCEL}") # If they chose no, @@ -1662,6 +1671,9 @@ finalExports() { # Set the privacy level addOrEditKeyValPair "${FTL_CONFIG_FILE}" "PRIVACYLEVEL" "${PRIVACY_LEVEL}" + # Set the web port + addOrEditKeyValPair "${FTL_CONFIG_FILE}" "WEBPORT" "${PRIVACY_LEVEL}" + # Bring in the current settings and the functions to manipulate them source "${setupVars}" # shellcheck source=advanced/Scripts/webpage.sh @@ -1819,7 +1831,7 @@ displayFinalMessage() { # If the user wants to install the dashboard, if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then # Store a message in a variable and display it - additional="View the web interface at http://pi.hole/admin or http://${IPV4_ADDRESS%/*}/admin\\n\\nYour Admin Webpage login password is ${pwstring}" + additional="View the web interface at http://pi.hole/admin:${WEBPORT} or http://${IPV4_ADDRESS%/*}:${WEBPORT}/admin\\n\\nYour Admin Webpage login password is ${pwstring}" fi # Final completion message to user @@ -2524,7 +2536,7 @@ main() { if [[ "${useUpdateVars}" == false ]]; then # If the Web interface was installed, if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - printf " %b View the web interface at http://pi.hole/admin or http://%s/admin\\n\\n" "${INFO}" "${IPV4_ADDRESS%/*}" + printf " %b View the web interface at http://pi.hole:${WEBPORT}/admin or http://%s/admin\\n\\n" "${INFO}" "${IPV4_ADDRESS%/*}:${WEBPORT}" fi # Explain to the user how to use Pi-hole as their DNS server printf " %b You may now configure your devices to use the Pi-hole as their DNS server\\n" "${INFO}" From 31f16510e2e469b48780625ed45d0f4312f9a9da Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 13 Jan 2023 21:23:50 +0000 Subject: [PATCH 005/413] Clear out some no longer needed ftl/dnsmasq config code Signed-off-by: Adam Warner --- advanced/01-pihole.conf | 35 -- advanced/06-rfc6761.conf | 42 -- advanced/dnsmasq.conf.original | 648 ----------------------------- automated install/basic-install.sh | 114 +---- 4 files changed, 20 insertions(+), 819 deletions(-) delete mode 100644 advanced/01-pihole.conf delete mode 100644 advanced/06-rfc6761.conf delete mode 100644 advanced/dnsmasq.conf.original diff --git a/advanced/01-pihole.conf b/advanced/01-pihole.conf deleted file mode 100644 index 677910f6..00000000 --- a/advanced/01-pihole.conf +++ /dev/null @@ -1,35 +0,0 @@ -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Dnsmasq config for Pi-hole's FTLDNS -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. - -############################################################################### -# FILE AUTOMATICALLY POPULATED BY PI-HOLE INSTALL/UPDATE PROCEDURE. # -# ANY CHANGES MADE TO THIS FILE AFTER INSTALL WILL BE LOST ON THE NEXT UPDATE # -# # -# IF YOU WISH TO CHANGE THE UPSTREAM SERVERS, CHANGE THEM IN: # -# /etc/pihole/setupVars.conf # -# # -# ANY OTHER CHANGES SHOULD BE MADE IN A SEPARATE CONFIG FILE # -# WITHIN /etc/dnsmasq.d/yourname.conf # -############################################################################### - -addn-hosts=/etc/pihole/local.list -addn-hosts=/etc/pihole/custom.list - -domain-needed - -localise-queries - -bogus-priv - -no-resolv - -log-queries -log-facility=/var/log/pihole/pihole.log - -log-async diff --git a/advanced/06-rfc6761.conf b/advanced/06-rfc6761.conf deleted file mode 100644 index fcdd0010..00000000 --- a/advanced/06-rfc6761.conf +++ /dev/null @@ -1,42 +0,0 @@ -# Pi-hole: A black hole for Internet advertisements -# (c) 2021 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# RFC 6761 config file for Pi-hole -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. - -############################################################################### -# FILE AUTOMATICALLY POPULATED BY PI-HOLE INSTALL/UPDATE PROCEDURE. # -# ANY CHANGES MADE TO THIS FILE AFTER INSTALL WILL BE LOST ON THE NEXT UPDATE # -# # -# CHANGES SHOULD BE MADE IN A SEPARATE CONFIG FILE # -# WITHIN /etc/dnsmasq.d/yourname.conf # -############################################################################### - -# RFC 6761: Caching DNS servers SHOULD recognize -# test, localhost, invalid -# names as special and SHOULD NOT attempt to look up NS records for them, or -# otherwise query authoritative DNS servers in an attempt to resolve these -# names. -server=/test/ -server=/localhost/ -server=/invalid/ - -# The same RFC requests something similar for -# 10.in-addr.arpa. 21.172.in-addr.arpa. 27.172.in-addr.arpa. -# 16.172.in-addr.arpa. 22.172.in-addr.arpa. 28.172.in-addr.arpa. -# 17.172.in-addr.arpa. 23.172.in-addr.arpa. 29.172.in-addr.arpa. -# 18.172.in-addr.arpa. 24.172.in-addr.arpa. 30.172.in-addr.arpa. -# 19.172.in-addr.arpa. 25.172.in-addr.arpa. 31.172.in-addr.arpa. -# 20.172.in-addr.arpa. 26.172.in-addr.arpa. 168.192.in-addr.arpa. -# Pi-hole implements this via the dnsmasq option "bogus-priv" (see -# 01-pihole.conf) because this also covers IPv6. - -# OpenWRT furthermore blocks bind, local, onion domains -# see https://git.openwrt.org/?p=openwrt/openwrt.git;a=blob_plain;f=package/network/services/dnsmasq/files/rfc6761.conf;hb=HEAD -# and https://www.iana.org/assignments/special-use-domain-names/special-use-domain-names.xhtml -# We do not include the ".local" rule ourselves, see https://github.com/pi-hole/pi-hole/pull/4282#discussion_r689112972 -server=/bind/ -server=/onion/ diff --git a/advanced/dnsmasq.conf.original b/advanced/dnsmasq.conf.original deleted file mode 100644 index 4aa5a8bf..00000000 --- a/advanced/dnsmasq.conf.original +++ /dev/null @@ -1,648 +0,0 @@ -# Configuration file for dnsmasq. -# -# Format is one option per line, legal options are the same -# as the long options legal on the command line. See -# "/usr/sbin/dnsmasq --help" or "man 8 dnsmasq" for details. - -# Listen on this specific port instead of the standard DNS port -# (53). Setting this to zero completely disables DNS function, -# leaving only DHCP and/or TFTP. -#port=5353 - -# The following two options make you a better netizen, since they -# tell dnsmasq to filter out queries which the public DNS cannot -# answer, and which load the servers (especially the root servers) -# unnecessarily. If you have a dial-on-demand link they also stop -# these requests from bringing up the link unnecessarily. - -# Never forward plain names (without a dot or domain part) -#domain-needed -# Never forward addresses in the non-routed address spaces. -#bogus-priv - -# Uncomment these to enable DNSSEC validation and caching: -# (Requires dnsmasq to be built with DNSSEC option.) -#conf-file=%%PREFIX%%/share/dnsmasq/trust-anchors.conf -#dnssec - -# Replies which are not DNSSEC signed may be legitimate, because the domain -# is unsigned, or may be forgeries. Setting this option tells dnsmasq to -# check that an unsigned reply is OK, by finding a secure proof that a DS -# record somewhere between the root and the domain does not exist. -# The cost of setting this is that even queries in unsigned domains will need -# one or more extra DNS queries to verify. -#dnssec-check-unsigned - -# Uncomment this to filter useless windows-originated DNS requests -# which can trigger dial-on-demand links needlessly. -# Note that (amongst other things) this blocks all SRV requests, -# so don't use it if you use eg Kerberos, SIP, XMMP or Google-talk. -# This option only affects forwarding, SRV records originating for -# dnsmasq (via srv-host= lines) are not suppressed by it. -#filterwin2k - -# Change this line if you want dns to get its upstream servers from -# somewhere other that /etc/resolv.conf -#resolv-file= - -# By default, dnsmasq will send queries to any of the upstream -# servers it knows about and tries to favor servers to are known -# to be up. Uncommenting this forces dnsmasq to try each query -# with each server strictly in the order they appear in -# /etc/resolv.conf -#strict-order - -# If you don't want dnsmasq to read /etc/resolv.conf or any other -# file, getting its servers from this file instead (see below), then -# uncomment this. -#no-resolv - -# If you don't want dnsmasq to poll /etc/resolv.conf or other resolv -# files for changes and re-read them then uncomment this. -#no-poll - -# Add other name servers here, with domain specs if they are for -# non-public domains. -#server=/localnet/192.168.0.1 - -# Example of routing PTR queries to nameservers: this will send all -# address->name queries for 192.168.3/24 to nameserver 10.1.2.3 -#server=/3.168.192.in-addr.arpa/10.1.2.3 - -# Add local-only domains here, queries in these domains are answered -# from /etc/hosts or DHCP only. -#local=/localnet/ - -# Add domains which you want to force to an IP address here. -# The example below send any host in double-click.net to a local -# web-server. -#address=/double-click.net/127.0.0.1 - -# --address (and --server) work with IPv6 addresses too. -#address=/www.thekelleys.org.uk/fe80::20d:60ff:fe36:f83 - -# Add the IPs of all queries to yahoo.com, google.com, and their -# subdomains to the vpn and search ipsets: -#ipset=/yahoo.com/google.com/vpn,search - -# You can control how dnsmasq talks to a server: this forces -# queries to 10.1.2.3 to be routed via eth1 -# server=10.1.2.3@eth1 - -# and this sets the source (ie local) address used to talk to -# 10.1.2.3 to 192.168.1.1 port 55 (there must be a interface with that -# IP on the machine, obviously). -# server=10.1.2.3@192.168.1.1#55 - -# If you want dnsmasq to change uid and gid to something other -# than the default, edit the following lines. -#user= -#group= - -# If you want dnsmasq to listen for DHCP and DNS requests only on -# specified interfaces (and the loopback) give the name of the -# interface (eg eth0) here. -# Repeat the line for more than one interface. -#interface= -# Or you can specify which interface _not_ to listen on -#except-interface= -# Or which to listen on by address (remember to include 127.0.0.1 if -# you use this.) -#listen-address= -# If you want dnsmasq to provide only DNS service on an interface, -# configure it as shown above, and then use the following line to -# disable DHCP and TFTP on it. -#no-dhcp-interface= - -# On systems which support it, dnsmasq binds the wildcard address, -# even when it is listening on only some interfaces. It then discards -# requests that it shouldn't reply to. This has the advantage of -# working even when interfaces come and go and change address. If you -# want dnsmasq to really bind only the interfaces it is listening on, -# uncomment this option. About the only time you may need this is when -# running another nameserver on the same machine. -#bind-interfaces - -# If you don't want dnsmasq to read /etc/hosts, uncomment the -# following line. -#no-hosts -# or if you want it to read another file, as well as /etc/hosts, use -# this. -#addn-hosts=/etc/banner_add_hosts - -# Set this (and domain: see below) if you want to have a domain -# automatically added to simple names in a hosts-file. -#expand-hosts - -# Set the domain for dnsmasq. this is optional, but if it is set, it -# does the following things. -# 1) Allows DHCP hosts to have fully qualified domain names, as long -# as the domain part matches this setting. -# 2) Sets the "domain" DHCP option thereby potentially setting the -# domain of all systems configured by DHCP -# 3) Provides the domain part for "expand-hosts" -#domain=thekelleys.org.uk - -# Set a different domain for a particular subnet -#domain=wireless.thekelleys.org.uk,192.168.2.0/24 - -# Same idea, but range rather then subnet -#domain=reserved.thekelleys.org.uk,192.68.3.100,192.168.3.200 - -# Uncomment this to enable the integrated DHCP server, you need -# to supply the range of addresses available for lease and optionally -# a lease time. If you have more than one network, you will need to -# repeat this for each network on which you want to supply DHCP -# service. -#dhcp-range=192.168.0.50,192.168.0.150,12h - -# This is an example of a DHCP range where the netmask is given. This -# is needed for networks we reach the dnsmasq DHCP server via a relay -# agent. If you don't know what a DHCP relay agent is, you probably -# don't need to worry about this. -#dhcp-range=192.168.0.50,192.168.0.150,255.255.255.0,12h - -# This is an example of a DHCP range which sets a tag, so that -# some DHCP options may be set only for this network. -#dhcp-range=set:red,192.168.0.50,192.168.0.150 - -# Use this DHCP range only when the tag "green" is set. -#dhcp-range=tag:green,192.168.0.50,192.168.0.150,12h - -# Specify a subnet which can't be used for dynamic address allocation, -# is available for hosts with matching --dhcp-host lines. Note that -# dhcp-host declarations will be ignored unless there is a dhcp-range -# of some type for the subnet in question. -# In this case the netmask is implied (it comes from the network -# configuration on the machine running dnsmasq) it is possible to give -# an explicit netmask instead. -#dhcp-range=192.168.0.0,static - -# Enable DHCPv6. Note that the prefix-length does not need to be specified -# and defaults to 64 if missing/ -#dhcp-range=1234::2, 1234::500, 64, 12h - -# Do Router Advertisements, BUT NOT DHCP for this subnet. -#dhcp-range=1234::, ra-only - -# Do Router Advertisements, BUT NOT DHCP for this subnet, also try and -# add names to the DNS for the IPv6 address of SLAAC-configured dual-stack -# hosts. Use the DHCPv4 lease to derive the name, network segment and -# MAC address and assume that the host will also have an -# IPv6 address calculated using the SLAAC algorithm. -#dhcp-range=1234::, ra-names - -# Do Router Advertisements, BUT NOT DHCP for this subnet. -# Set the lifetime to 46 hours. (Note: minimum lifetime is 2 hours.) -#dhcp-range=1234::, ra-only, 48h - -# Do DHCP and Router Advertisements for this subnet. Set the A bit in the RA -# so that clients can use SLAAC addresses as well as DHCP ones. -#dhcp-range=1234::2, 1234::500, slaac - -# Do Router Advertisements and stateless DHCP for this subnet. Clients will -# not get addresses from DHCP, but they will get other configuration information. -# They will use SLAAC for addresses. -#dhcp-range=1234::, ra-stateless - -# Do stateless DHCP, SLAAC, and generate DNS names for SLAAC addresses -# from DHCPv4 leases. -#dhcp-range=1234::, ra-stateless, ra-names - -# Do router advertisements for all subnets where we're doing DHCPv6 -# Unless overridden by ra-stateless, ra-names, et al, the router -# advertisements will have the M and O bits set, so that the clients -# get addresses and configuration from DHCPv6, and the A bit reset, so the -# clients don't use SLAAC addresses. -#enable-ra - -# Supply parameters for specified hosts using DHCP. There are lots -# of valid alternatives, so we will give examples of each. Note that -# IP addresses DO NOT have to be in the range given above, they just -# need to be on the same network. The order of the parameters in these -# do not matter, it's permissible to give name, address and MAC in any -# order. - -# Always allocate the host with Ethernet address 11:22:33:44:55:66 -# The IP address 192.168.0.60 -#dhcp-host=11:22:33:44:55:66,192.168.0.60 - -# Always set the name of the host with hardware address -# 11:22:33:44:55:66 to be "fred" -#dhcp-host=11:22:33:44:55:66,fred - -# Always give the host with Ethernet address 11:22:33:44:55:66 -# the name fred and IP address 192.168.0.60 and lease time 45 minutes -#dhcp-host=11:22:33:44:55:66,fred,192.168.0.60,45m - -# Give a host with Ethernet address 11:22:33:44:55:66 or -# 12:34:56:78:90:12 the IP address 192.168.0.60. Dnsmasq will assume -# that these two Ethernet interfaces will never be in use at the same -# time, and give the IP address to the second, even if it is already -# in use by the first. Useful for laptops with wired and wireless -# addresses. -#dhcp-host=11:22:33:44:55:66,12:34:56:78:90:12,192.168.0.60 - -# Give the machine which says its name is "bert" IP address -# 192.168.0.70 and an infinite lease -#dhcp-host=bert,192.168.0.70,infinite - -# Always give the host with client identifier 01:02:02:04 -# the IP address 192.168.0.60 -#dhcp-host=id:01:02:02:04,192.168.0.60 - -# Always give the host with client identifier "marjorie" -# the IP address 192.168.0.60 -#dhcp-host=id:marjorie,192.168.0.60 - -# Enable the address given for "judge" in /etc/hosts -# to be given to a machine presenting the name "judge" when -# it asks for a DHCP lease. -#dhcp-host=judge - -# Never offer DHCP service to a machine whose Ethernet -# address is 11:22:33:44:55:66 -#dhcp-host=11:22:33:44:55:66,ignore - -# Ignore any client-id presented by the machine with Ethernet -# address 11:22:33:44:55:66. This is useful to prevent a machine -# being treated differently when running under different OS's or -# between PXE boot and OS boot. -#dhcp-host=11:22:33:44:55:66,id:* - -# Send extra options which are tagged as "red" to -# the machine with Ethernet address 11:22:33:44:55:66 -#dhcp-host=11:22:33:44:55:66,set:red - -# Send extra options which are tagged as "red" to -# any machine with Ethernet address starting 11:22:33: -#dhcp-host=11:22:33:*:*:*,set:red - -# Give a fixed IPv6 address and name to client with -# DUID 00:01:00:01:16:d2:83:fc:92:d4:19:e2:d8:b2 -# Note the MAC addresses CANNOT be used to identify DHCPv6 clients. -# Note also the they [] around the IPv6 address are obligatory. -#dhcp-host=id:00:01:00:01:16:d2:83:fc:92:d4:19:e2:d8:b2, fred, [1234::5] - -# Ignore any clients which are not specified in dhcp-host lines -# or /etc/ethers. Equivalent to ISC "deny unknown-clients". -# This relies on the special "known" tag which is set when -# a host is matched. -#dhcp-ignore=tag:!known - -# Send extra options which are tagged as "red" to any machine whose -# DHCP vendorclass string includes the substring "Linux" -#dhcp-vendorclass=set:red,Linux - -# Send extra options which are tagged as "red" to any machine one -# of whose DHCP userclass strings includes the substring "accounts" -#dhcp-userclass=set:red,accounts - -# Send extra options which are tagged as "red" to any machine whose -# MAC address matches the pattern. -#dhcp-mac=set:red,00:60:8C:*:*:* - -# If this line is uncommented, dnsmasq will read /etc/ethers and act -# on the ethernet-address/IP pairs found there just as if they had -# been given as --dhcp-host options. Useful if you keep -# MAC-address/host mappings there for other purposes. -#read-ethers - -# Send options to hosts which ask for a DHCP lease. -# See RFC 2132 for details of available options. -# Common options can be given to dnsmasq by name: -# run "dnsmasq --help dhcp" to get a list. -# Note that all the common settings, such as netmask and -# broadcast address, DNS server and default route, are given -# sane defaults by dnsmasq. You very likely will not need -# any dhcp-options. If you use Windows clients and Samba, there -# are some options which are recommended, they are detailed at the -# end of this section. - -# Override the default route supplied by dnsmasq, which assumes the -# router is the same machine as the one running dnsmasq. -#dhcp-option=3,1.2.3.4 - -# Do the same thing, but using the option name -#dhcp-option=option:router,1.2.3.4 - -# Override the default route supplied by dnsmasq and send no default -# route at all. Note that this only works for the options sent by -# default (1, 3, 6, 12, 28) the same line will send a zero-length option -# for all other option numbers. -#dhcp-option=3 - -# Set the NTP time server addresses to 192.168.0.4 and 10.10.0.5 -#dhcp-option=option:ntp-server,192.168.0.4,10.10.0.5 - -# Send DHCPv6 option. Note [] around IPv6 addresses. -#dhcp-option=option6:dns-server,[1234::77],[1234::88] - -# Send DHCPv6 option for namservers as the machine running -# dnsmasq and another. -#dhcp-option=option6:dns-server,[::],[1234::88] - -# Ask client to poll for option changes every six hours. (RFC4242) -#dhcp-option=option6:information-refresh-time,6h - -# Set the NTP time server address to be the same machine as -# is running dnsmasq -#dhcp-option=42,0.0.0.0 - -# Set the NIS domain name to "welly" -#dhcp-option=40,welly - -# Set the default time-to-live to 50 -#dhcp-option=23,50 - -# Set the "all subnets are local" flag -#dhcp-option=27,1 - -# Send the etherboot magic flag and then etherboot options (a string). -#dhcp-option=128,e4:45:74:68:00:00 -#dhcp-option=129,NIC=eepro100 - -# Specify an option which will only be sent to the "red" network -# (see dhcp-range for the declaration of the "red" network) -# Note that the tag: part must precede the option: part. -#dhcp-option = tag:red, option:ntp-server, 192.168.1.1 - -# The following DHCP options set up dnsmasq in the same way as is specified -# for the ISC dhcpcd in -# http://www.samba.org/samba/ftp/docs/textdocs/DHCP-Server-Configuration.txt -# adapted for a typical dnsmasq installation where the host running -# dnsmasq is also the host running samba. -# you may want to uncomment some or all of them if you use -# Windows clients and Samba. -#dhcp-option=19,0 # option ip-forwarding off -#dhcp-option=44,0.0.0.0 # set netbios-over-TCP/IP nameserver(s) aka WINS server(s) -#dhcp-option=45,0.0.0.0 # netbios datagram distribution server -#dhcp-option=46,8 # netbios node type - -# Send an empty WPAD option. This may be REQUIRED to get windows 7 to behave. -#dhcp-option=252,"\n" - -# Send RFC-3397 DNS domain search DHCP option. WARNING: Your DHCP client -# probably doesn't support this...... -#dhcp-option=option:domain-search,eng.apple.com,marketing.apple.com - -# Send RFC-3442 classless static routes (note the netmask encoding) -#dhcp-option=121,192.168.1.0/24,1.2.3.4,10.0.0.0/8,5.6.7.8 - -# Send vendor-class specific options encapsulated in DHCP option 43. -# The meaning of the options is defined by the vendor-class so -# options are sent only when the client supplied vendor class -# matches the class given here. (A substring match is OK, so "MSFT" -# matches "MSFT" and "MSFT 5.0"). This example sets the -# mtftp address to 0.0.0.0 for PXEClients. -#dhcp-option=vendor:PXEClient,1,0.0.0.0 - -# Send microsoft-specific option to tell windows to release the DHCP lease -# when it shuts down. Note the "i" flag, to tell dnsmasq to send the -# value as a four-byte integer - that's what microsoft wants. See -# http://technet2.microsoft.com/WindowsServer/en/library/a70f1bb7-d2d4-49f0-96d6-4b7414ecfaae1033.mspx?mfr=true -#dhcp-option=vendor:MSFT,2,1i - -# Send the Encapsulated-vendor-class ID needed by some configurations of -# Etherboot to allow is to recognize the DHCP server. -#dhcp-option=vendor:Etherboot,60,"Etherboot" - -# Send options to PXELinux. Note that we need to send the options even -# though they don't appear in the parameter request list, so we need -# to use dhcp-option-force here. -# See http://syslinux.zytor.com/pxe.php#special for details. -# Magic number - needed before anything else is recognized -#dhcp-option-force=208,f1:00:74:7e -# Configuration file name -#dhcp-option-force=209,configs/common -# Path prefix -#dhcp-option-force=210,/tftpboot/pxelinux/files/ -# Reboot time. (Note 'i' to send 32-bit value) -#dhcp-option-force=211,30i - -# Set the boot filename for netboot/PXE. You will only need -# this is you want to boot machines over the network and you will need -# a TFTP server; either dnsmasq's built in TFTP server or an -# external one. (See below for how to enable the TFTP server.) -#dhcp-boot=pxelinux.0 - -# The same as above, but use custom tftp-server instead machine running dnsmasq -#dhcp-boot=pxelinux,server.name,192.168.1.100 - -# Boot for Etherboot gPXE. The idea is to send two different -# filenames, the first loads gPXE, and the second tells gPXE what to -# load. The dhcp-match sets the gpxe tag for requests from gPXE. -#dhcp-match=set:gpxe,175 # gPXE sends a 175 option. -#dhcp-boot=tag:!gpxe,undionly.kpxe -#dhcp-boot=mybootimage - -# Encapsulated options for Etherboot gPXE. All the options are -# encapsulated within option 175 -#dhcp-option=encap:175, 1, 5b # priority code -#dhcp-option=encap:175, 176, 1b # no-proxydhcp -#dhcp-option=encap:175, 177, string # bus-id -#dhcp-option=encap:175, 189, 1b # BIOS drive code -#dhcp-option=encap:175, 190, user # iSCSI username -#dhcp-option=encap:175, 191, pass # iSCSI password - -# Test for the architecture of a netboot client. PXE clients are -# supposed to send their architecture as option 93. (See RFC 4578) -#dhcp-match=peecees, option:client-arch, 0 #x86-32 -#dhcp-match=itanics, option:client-arch, 2 #IA64 -#dhcp-match=hammers, option:client-arch, 6 #x86-64 -#dhcp-match=mactels, option:client-arch, 7 #EFI x86-64 - -# Do real PXE, rather than just booting a single file, this is an -# alternative to dhcp-boot. -#pxe-prompt="What system shall I netboot?" -# or with timeout before first available action is taken: -#pxe-prompt="Press F8 for menu.", 60 - -# Available boot services. for PXE. -#pxe-service=x86PC, "Boot from local disk" - -# Loads /pxelinux.0 from dnsmasq TFTP server. -#pxe-service=x86PC, "Install Linux", pxelinux - -# Loads /pxelinux.0 from TFTP server at 1.2.3.4. -# Beware this fails on old PXE ROMS. -#pxe-service=x86PC, "Install Linux", pxelinux, 1.2.3.4 - -# Use bootserver on network, found my multicast or broadcast. -#pxe-service=x86PC, "Install windows from RIS server", 1 - -# Use bootserver at a known IP address. -#pxe-service=x86PC, "Install windows from RIS server", 1, 1.2.3.4 - -# If you have multicast-FTP available, -# information for that can be passed in a similar way using options 1 -# to 5. See page 19 of -# http://download.intel.com/design/archives/wfm/downloads/pxespec.pdf - - -# Enable dnsmasq's built-in TFTP server -#enable-tftp - -# Set the root directory for files available via FTP. -#tftp-root=/var/ftpd - -# Make the TFTP server more secure: with this set, only files owned by -# the user dnsmasq is running as will be send over the net. -#tftp-secure - -# This option stops dnsmasq from negotiating a larger blocksize for TFTP -# transfers. It will slow things down, but may rescue some broken TFTP -# clients. -#tftp-no-blocksize - -# Set the boot file name only when the "red" tag is set. -#dhcp-boot=tag:red,pxelinux.red-net - -# An example of dhcp-boot with an external TFTP server: the name and IP -# address of the server are given after the filename. -# Can fail with old PXE ROMS. Overridden by --pxe-service. -#dhcp-boot=/var/ftpd/pxelinux.0,boothost,192.168.0.3 - -# If there are multiple external tftp servers having a same name -# (using /etc/hosts) then that name can be specified as the -# tftp_servername (the third option to dhcp-boot) and in that -# case dnsmasq resolves this name and returns the resultant IP -# addresses in round robin fashion. This facility can be used to -# load balance the tftp load among a set of servers. -#dhcp-boot=/var/ftpd/pxelinux.0,boothost,tftp_server_name - -# Set the limit on DHCP leases, the default is 150 -#dhcp-lease-max=150 - -# The DHCP server needs somewhere on disk to keep its lease database. -# This defaults to a sane location, but if you want to change it, use -# the line below. -#dhcp-leasefile=/var/lib/misc/dnsmasq.leases - -# Set the DHCP server to authoritative mode. In this mode it will barge in -# and take over the lease for any client which broadcasts on the network, -# whether it has a record of the lease or not. This avoids long timeouts -# when a machine wakes up on a new network. DO NOT enable this if there's -# the slightest chance that you might end up accidentally configuring a DHCP -# server for your campus/company accidentally. The ISC server uses -# the same option, and this URL provides more information: -# http://www.isc.org/files/auth.html -#dhcp-authoritative - -# Run an executable when a DHCP lease is created or destroyed. -# The arguments sent to the script are "add" or "del", -# then the MAC address, the IP address and finally the hostname -# if there is one. -#dhcp-script=/bin/echo - -# Set the cachesize here. -#cache-size=150 - -# If you want to disable negative caching, uncomment this. -#no-negcache - -# Normally responses which come from /etc/hosts and the DHCP lease -# file have Time-To-Live set as zero, which conventionally means -# do not cache further. If you are happy to trade lower load on the -# server for potentially stale date, you can set a time-to-live (in -# seconds) here. -#local-ttl= - -# If you want dnsmasq to detect attempts by Verisign to send queries -# to unregistered .com and .net hosts to its sitefinder service and -# have dnsmasq instead return the correct NXDOMAIN response, uncomment -# this line. You can add similar lines to do the same for other -# registries which have implemented wildcard A records. -#bogus-nxdomain=64.94.110.11 - -# If you want to fix up DNS results from upstream servers, use the -# alias option. This only works for IPv4. -# This alias makes a result of 1.2.3.4 appear as 5.6.7.8 -#alias=1.2.3.4,5.6.7.8 -# and this maps 1.2.3.x to 5.6.7.x -#alias=1.2.3.0,5.6.7.0,255.255.255.0 -# and this maps 192.168.0.10->192.168.0.40 to 10.0.0.10->10.0.0.40 -#alias=192.168.0.10-192.168.0.40,10.0.0.0,255.255.255.0 - -# Change these lines if you want dnsmasq to serve MX records. - -# Return an MX record named "maildomain.com" with target -# servermachine.com and preference 50 -#mx-host=maildomain.com,servermachine.com,50 - -# Set the default target for MX records created using the localmx option. -#mx-target=servermachine.com - -# Return an MX record pointing to the mx-target for all local -# machines. -#localmx - -# Return an MX record pointing to itself for all local machines. -#selfmx - -# Change the following lines if you want dnsmasq to serve SRV -# records. These are useful if you want to serve ldap requests for -# Active Directory and other windows-originated DNS requests. -# See RFC 2782. -# You may add multiple srv-host lines. -# The fields are ,,,, -# If the domain part if missing from the name (so that is just has the -# service and protocol sections) then the domain given by the domain= -# config option is used. (Note that expand-hosts does not need to be -# set for this to work.) - -# A SRV record sending LDAP for the example.com domain to -# ldapserver.example.com port 389 -#srv-host=_ldap._tcp.example.com,ldapserver.example.com,389 - -# A SRV record sending LDAP for the example.com domain to -# ldapserver.example.com port 389 (using domain=) -#domain=example.com -#srv-host=_ldap._tcp,ldapserver.example.com,389 - -# Two SRV records for LDAP, each with different priorities -#srv-host=_ldap._tcp.example.com,ldapserver.example.com,389,1 -#srv-host=_ldap._tcp.example.com,ldapserver.example.com,389,2 - -# A SRV record indicating that there is no LDAP server for the domain -# example.com -#srv-host=_ldap._tcp.example.com - -# The following line shows how to make dnsmasq serve an arbitrary PTR -# record. This is useful for DNS-SD. (Note that the -# domain-name expansion done for SRV records _does_not -# occur for PTR records.) -#ptr-record=_http._tcp.dns-sd-services,"New Employee Page._http._tcp.dns-sd-services" - -# Change the following lines to enable dnsmasq to serve TXT records. -# These are used for things like SPF and zeroconf. (Note that the -# domain-name expansion done for SRV records _does_not -# occur for TXT records.) - -#Example SPF. -#txt-record=example.com,"v=spf1 a -all" - -#Example zeroconf -#txt-record=_http._tcp.example.com,name=value,paper=A4 - -# Provide an alias for a "local" DNS name. Note that this _only_ works -# for targets which are names from DHCP or /etc/hosts. Give host -# "bert" another name, bertrand -#cname=bertand,bert - -# For debugging purposes, log each DNS query as it passes through -# dnsmasq. -#log-queries - -# Log lots of extra information about DHCP transactions. -#log-dhcp - -# Include another lot of configuration options. -#conf-file=/etc/dnsmasq.more.conf -#conf-dir=/etc/dnsmasq.d - -# Include all the files in a directory except those ending in .bak -#conf-dir=/etc/dnsmasq.d,.bak - -# Include all files in a directory which end in .conf -#conf-dir=/etc/dnsmasq.d/*.conf diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index d9e22e08..7d899940 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1137,83 +1137,31 @@ installDefaultBlocklists() { echo "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" >> "${adlistFile}" } -# Check if /etc/dnsmasq.conf is from pi-hole. If so replace with an original and install new in .d directory -version_check_dnsmasq() { +remove_old_dnsmasq_ftl_configs() { # Local, named variables local dnsmasq_conf="/etc/dnsmasq.conf" - local dnsmasq_conf_orig="/etc/dnsmasq.conf.orig" - local dnsmasq_pihole_id_string="addn-hosts=/etc/pihole/gravity.list" - local dnsmasq_pihole_id_string2="# Dnsmasq config for Pi-hole's FTLDNS" - local dnsmasq_original_config="${PI_HOLE_LOCAL_REPO}/advanced/dnsmasq.conf.original" - local dnsmasq_pihole_01_source="${PI_HOLE_LOCAL_REPO}/advanced/01-pihole.conf" - local dnsmasq_pihole_01_target="/etc/dnsmasq.d/01-pihole.conf" - local dnsmasq_rfc6761_06_source="${PI_HOLE_LOCAL_REPO}/advanced/06-rfc6761.conf" - local dnsmasq_rfc6761_06_target="/etc/dnsmasq.d/06-rfc6761.conf" + local pihole_01="/etc/dnsmasq.d/01-pihole.conf" + local rfc6761_06="/etc/dnsmasq.d/06-rfc6761.conf" + local pihole_dhcp_02="/etc/dnsmasq.d/02-pihole-dhcp.conf" + + # pihole-FTL does some fancy stuff with config these days, and so we can remove some old config files + if [[ -f "${pihole_01}" ]]; then + rm "${pihole_01}" + fi + + if [[ -f "${rfc6761_06}" ]]; then + rm "${rfc6761_06}" + fi + + if [[ -f "${pihole_dhcp_02}" ]]; then + rm "${pihole_dhcp_02}" + fi # If the dnsmasq config file exists if [[ -f "${dnsmasq_conf}" ]]; then - printf " %b Existing dnsmasq.conf found..." "${INFO}" - # If a specific string is found within this file, we presume it's from older versions on Pi-hole, - if grep -q "${dnsmasq_pihole_id_string}" "${dnsmasq_conf}" || - grep -q "${dnsmasq_pihole_id_string2}" "${dnsmasq_conf}"; then - printf " it is from a previous Pi-hole install.\\n" - printf " %b Backing up dnsmasq.conf to dnsmasq.conf.orig..." "${INFO}" - # so backup the original file, - mv -f "${dnsmasq_conf}" "${dnsmasq_conf_orig}" - printf "%b %b Backing up dnsmasq.conf to dnsmasq.conf.orig...\\n" "${OVER}" "${TICK}" - printf " %b Restoring default dnsmasq.conf..." "${INFO}" - # and replace it with the default - install -D -m 644 -T "${dnsmasq_original_config}" "${dnsmasq_conf}" - printf "%b %b Restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" - else - # Otherwise, don't to anything - printf " it is not a Pi-hole file, leaving alone!\\n" - fi - else - # If a file cannot be found, - printf " %b No dnsmasq.conf found... restoring default dnsmasq.conf..." "${INFO}" - # restore the default one - install -D -m 644 -T "${dnsmasq_original_config}" "${dnsmasq_conf}" - printf "%b %b No dnsmasq.conf found... restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" + # Back it up - we will need to add a symlink to /etc/pihole/dnsmasq.conf later + mv "${dnsmasq_conf}" "${dnsmasq_conf}.old" fi - - printf " %b Installing %s..." "${INFO}" "${dnsmasq_pihole_01_target}" - # Check to see if dnsmasq directory exists (it may not due to being a fresh install and dnsmasq no longer being a dependency) - if [[ ! -d "/etc/dnsmasq.d" ]];then - install -d -m 755 "/etc/dnsmasq.d" - fi - # Copy the new Pi-hole DNS config file into the dnsmasq.d directory - install -D -m 644 -T "${dnsmasq_pihole_01_source}" "${dnsmasq_pihole_01_target}" - printf "%b %b Installed %s\n" "${OVER}" "${TICK}" "${dnsmasq_pihole_01_target}" - # Add settings with the GLOBAL DNS variables that we populated earlier - # First, set the interface to listen on - addOrEditKeyValPair "${dnsmasq_pihole_01_target}" "interface" "$PIHOLE_INTERFACE" - if [[ "${PIHOLE_DNS_1}" != "" ]]; then - # then add in the primary DNS server. - addOrEditKeyValPair "${dnsmasq_pihole_01_target}" "server" "$PIHOLE_DNS_1" - fi - # Ditto if DNS2 is not empty - if [[ "${PIHOLE_DNS_2}" != "" ]]; then - addKey "${dnsmasq_pihole_01_target}" "server=$PIHOLE_DNS_2" - fi - - # Set the cache size - addOrEditKeyValPair "${dnsmasq_pihole_01_target}" "cache-size" "$CACHE_SIZE" - - sed -i 's/^#conf-dir=\/etc\/dnsmasq.d$/conf-dir=\/etc\/dnsmasq.d/' "${dnsmasq_conf}" - - # If the user does not want to enable logging, - if [[ "${QUERY_LOGGING}" == false ]] ; then - # remove itfrom the DNS config file - removeKey "${dnsmasq_pihole_01_target}" "log-queries" - else - # Otherwise, enable it by adding the directive to the DNS config file - addKey "${dnsmasq_pihole_01_target}" "log-queries" - fi - - printf " %b Installing %s..." "${INFO}" "${dnsmasq_rfc6761_06_source}" - install -D -m 644 -T "${dnsmasq_rfc6761_06_source}" "${dnsmasq_rfc6761_06_target}" - printf "%b %b Installed %s\n" "${OVER}" "${TICK}" "${dnsmasq_rfc6761_06_target}" } # Clean an existing installation to prepare for upgrade/reinstall @@ -1272,7 +1220,7 @@ installScripts() { installConfigs() { printf "\\n %b Installing configs from %s...\\n" "${INFO}" "${PI_HOLE_LOCAL_REPO}" # Make sure Pi-hole's config files are in place - version_check_dnsmasq + remove_old_dnsmasq_ftl_configs # Install list of DNS servers # Format: Name;Primary IPv4;Secondary IPv4;Primary IPv6;Secondary IPv6 @@ -2077,28 +2025,6 @@ FTLinstall() { fi } -disable_dnsmasq() { - # dnsmasq can now be stopped and disabled if it exists - if is_command dnsmasq; then - if check_service_active "dnsmasq";then - printf " %b FTL can now resolve DNS Queries without dnsmasq running separately\\n" "${INFO}" - stop_service dnsmasq - disable_service dnsmasq - fi - fi - - # Backup existing /etc/dnsmasq.conf if present and ensure that - # /etc/dnsmasq.conf contains only "conf-dir=/etc/dnsmasq.d" - local conffile="/etc/dnsmasq.conf" - if [[ -f "${conffile}" ]]; then - printf " %b Backing up %s to %s.old\\n" "${INFO}" "${conffile}" "${conffile}" - mv "${conffile}" "${conffile}.old" - fi - # Create /etc/dnsmasq.conf - echo "conf-dir=/etc/dnsmasq.d" > "${conffile}" - chmod 644 "${conffile}" -} - get_binary_name() { # This gives the machine architecture which may be different from the OS architecture... local machine From 0e558e4c367132f2c525a27ec7bf19c2ffe10e6c Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 13 Jan 2023 21:30:01 +0000 Subject: [PATCH 006/413] Remove some code that installs the blockpage/sets up web directories Remove some code that disables dnsmasq if it is running Create dnsmasq.d directory if it does not exist --- automated install/basic-install.sh | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 7d899940..9d9eb8be 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1162,6 +1162,11 @@ remove_old_dnsmasq_ftl_configs() { # Back it up - we will need to add a symlink to /etc/pihole/dnsmasq.conf later mv "${dnsmasq_conf}" "${dnsmasq_conf}.old" fi + + # Create /etc/dnsmasq.d if it doesn't exist + if [[ ! -d "/etc/dnsmasq.d" ]]; then + mkdir "/etc/dnsmasq.d" + fi } # Clean an existing installation to prepare for upgrade/reinstall @@ -1380,7 +1385,7 @@ check_service_active() { fi } -# Systemd-resolved's DNSStubListener and dnsmasq can't share port 53. +# Systemd-resolved's DNSStubListener and ftl can't share port 53. disable_resolved_stublistener() { printf " %b Testing if systemd-resolved is enabled\\n" "${INFO}" # Check if Systemd-resolved's DNSStubListener is enabled and active on port 53 @@ -1389,7 +1394,7 @@ disable_resolved_stublistener() { printf " %b %b Testing if systemd-resolved DNSStub-Listener is active" "${OVER}" "${INFO}" if ( grep -E '#?DNSStubListener=yes' /etc/systemd/resolved.conf &> /dev/null ); then # Disable the DNSStubListener to unbind it from port 53 - # Note that this breaks dns functionality on host until dnsmasq/ftl are up and running + # Note that this breaks dns functionality on host until ftl are up and running printf "%b %b Disabling systemd-resolved DNSStubListener" "${OVER}" "${TICK}" # Make a backup of the original /etc/systemd/resolved.conf # (This will need to be restored on uninstallation) @@ -1695,21 +1700,13 @@ installPihole() { printf " %b Failure in dependent config copy function.\\n" "${CROSS}" exit 1 fi - # If the user wants to install the dashboard, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - # do so - installPiholeWeb - fi + # Install the cron file installCron # Install the logrotate file installLogrotate || true - # Check if dnsmasq is present. If so, disable it and back up any possible - # config file - disable_dnsmasq - # install a man page entry for pihole install_manpage @@ -2134,13 +2131,6 @@ FTLcheckUpdate() { local remoteSha1 local localSha1 - # if dnsmasq exists and is running at this point, force reinstall of FTL Binary - if is_command dnsmasq; then - if check_service_active "dnsmasq";then - return 0 - fi - fi - if [[ ! "${ftlBranch}" == "master" ]]; then #Check whether or not the binary for this FTL branch actually exists. If not, then there is no update! local path @@ -2397,7 +2387,7 @@ main() { # Check for and disable systemd-resolved-DNSStubListener before reloading resolved # DNSStubListener needs to remain in place for installer to download needed files, # so this change needs to be made after installation is complete, - # but before starting or resarting the dnsmasq or ftl services + # but before starting or resarting the ftl service disable_resolved_stublistener printf " %b Restarting services...\\n" "${INFO}" From e3ffec57622c45ed01e1a9bf8e1ee31159a35871 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 13 Jan 2023 22:23:33 +0000 Subject: [PATCH 007/413] Change when old configs are removed Signed-off-by: Adam Warner --- automated install/basic-install.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9d9eb8be..764503f2 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1224,8 +1224,7 @@ installScripts() { # Install the configs from PI_HOLE_LOCAL_REPO to their various locations installConfigs() { printf "\\n %b Installing configs from %s...\\n" "${INFO}" "${PI_HOLE_LOCAL_REPO}" - # Make sure Pi-hole's config files are in place - remove_old_dnsmasq_ftl_configs + # Install list of DNS servers # Format: Name;Primary IPv4;Secondary IPv4;Primary IPv6;Secondary IPv6 @@ -1695,6 +1694,8 @@ installPihole() { exit 1 fi + remove_old_dnsmasq_ftl_configs + # Install config files if ! installConfigs; then printf " %b Failure in dependent config copy function.\\n" "${CROSS}" From 632aead691f747036c8d78a1096d705cf26820e5 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 14 Jan 2023 23:11:42 +0000 Subject: [PATCH 008/413] add wrappers to utils.sh for setting FTL config & add tests Signed-off-by: Adam Warner --- advanced/Scripts/utils.sh | 23 ++++++++++++++++ test/test_any_utils.py | 55 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index f655e56c..c97ad0cf 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -145,3 +145,26 @@ getFTLPID() { FTL_PID=${FTL_PID:=-1} echo "${FTL_PID}" } + +####################### +# returns value from FTLs config file using pihole-FTL --config +# +# Takes one argument: key +# Example getFTLConfigValue dns.piholePTR +####################### +getFTLConfigValue(){ + pihole-FTL --config "${1}" +} + +####################### +# sets value in FTLs config file using pihole-FTL --config +# +# Takes two arguments: key and value +# Example setFTLConfigValue dns.piholePTR PI.HOLE +# +# Note, for complex values such as dnsmasq.upstreams, you should wrap the value in single quotes: +# setFTLConfigValue dnsmasq.upstreams '[ "8.8.8.8" , "8.8.4.4" ]' +####################### +setFTLConfigValue(){ + pihole-FTL --config "${1}" "${2}" +} diff --git a/test/test_any_utils.py b/test/test_any_utils.py index b3fabe6c..741b1127 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -170,3 +170,58 @@ def test_getFTLPIDFile_and_getFTLPID_custom(host): ) expected_stdout = "1234\n" assert expected_stdout == output.stdout + + +def test_setFTLConfigValue_getFTLConfigValue(host): + """ + Confirms setFTLConfigValue works + Requires FTL to be installed, so we do that first (taken from test_FTL_binary_installed_and_responsive_no_errors) + """ + host.run( + """ + source /opt/pihole/basic-install.sh + create_pihole_user + funcOutput=$(get_binary_name) + echo "new/http" > /etc/pihole/ftlbranch + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" + """ + ) + + output = host.run( + """ + source /opt/pihole/utils.sh + setFTLConfigValue "dnsmasq.upstreams" '["1.1.1.1"]' + """ + ) + + assert '[ "1.1.1.1" ]' in output.stdout + + +def test_getFTLConfigValue_getFTLConfigValue(host): + """ + Confirms getFTLConfigValue works (also assumes setFTLConfigValue works) + Requires FTL to be installed, so we do that first (taken from test_FTL_binary_installed_and_responsive_no_errors) + """ + host.run( + """ + source /opt/pihole/basic-install.sh + create_pihole_user + funcOutput=$(get_binary_name) + echo "new/http" > /etc/pihole/ftlbranch + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" + """ + ) + + output = host.run( + """ + source /opt/pihole/utils.sh + setFTLConfigValue "dnsmasq.upstreams" '["9.9.9.9"]' > /dev/null + getFTLConfigValue "dnsmasq.upstreams" + """ + ) + + assert '[ "9.9.9.9" ]' in output.stdout From 019be067d9ddaf5cf1e1304ddb53c69ce2237a5e Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 14 Jan 2023 23:15:40 +0000 Subject: [PATCH 009/413] Convert SetWebPassword to use new FTL config Signed-off-by: Adam Warner --- advanced/Scripts/webpage.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 7680590f..1282b1ee 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -77,7 +77,7 @@ deleteFTLsetting() { } changeFTLsetting() { - addOrEditKeyValPair "${FTLconf}" "${1}" "${2}" + pihole-FTL --config "${1}" "${2}" } add_dnsmasq_setting() { @@ -165,7 +165,7 @@ SetWebPassword() { echo "" if [ "${PASSWORD}" == "" ]; then - addOrEditKeyValPair "${setupVars}" "WEBPASSWORD" "" + setFTLConfigValue "api.pwhash" "" echo -e " ${TICK} Password Removed" exit 0 fi @@ -178,7 +178,7 @@ SetWebPassword() { # We do not wrap this in brackets, otherwise BASH will expand any appropriate syntax hash=$(HashPassword "$PASSWORD") # Save hash to file - addOrEditKeyValPair "${setupVars}" "WEBPASSWORD" "${hash}" + setFTLConfigValue "api.pwhash" "${hash}" echo -e " ${TICK} New password set" else echo -e " ${CROSS} Passwords don't match. Your password has not been changed" From b8acccde90502731e1a1226a45ff8e4c903d0421 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 15 Jan 2023 11:33:53 +0000 Subject: [PATCH 010/413] Update comments on the move of /etc/dnsmasq.old -> /etc/dnsmasq.old to better reflect why we are doing it Signed-off-by: Adam Warner --- automated install/basic-install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 764503f2..87ff6d83 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1159,7 +1159,8 @@ remove_old_dnsmasq_ftl_configs() { # If the dnsmasq config file exists if [[ -f "${dnsmasq_conf}" ]]; then - # Back it up - we will need to add a symlink to /etc/pihole/dnsmasq.conf later + # There should not be anything custom in here for Pi-hole users + # It is no longer needed, but we'll back it up instead of deleting it just in case mv "${dnsmasq_conf}" "${dnsmasq_conf}.old" fi From 8f7c82840728ac742e21eb0fd260243c6f60416f Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 15 Jan 2023 13:47:38 +0000 Subject: [PATCH 011/413] No need for test_installPiholeWeb_fresh_install_no_errors Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index c1288287..e15839f2 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -118,21 +118,6 @@ def test_selinux_not_detected(host): assert check_selinux.rc == 0 -def test_installPiholeWeb_fresh_install_no_errors(host): - """ - confirms all web page assets from Core repo are installed on a fresh build - """ - installWeb = host.run( - """ - umask 0027 - source /opt/pihole/basic-install.sh - installPiholeWeb - """ - ) - expected_stdout = tick_box + " Installing sudoer file" - assert expected_stdout in installWeb.stdout - - def get_directories_recursive(host, directory): if directory is None: return directory From 5e431210fd5a312939a4e204f0d4e32df199b744 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 15 Jan 2023 13:49:25 +0000 Subject: [PATCH 012/413] [REVISIT] Remove test_setupVars_saved_to_file for now - it may no longer be needed as we move towards getting rid of the file. Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 34 ------------------------------ 1 file changed, 34 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index e15839f2..059e3a8b 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -68,40 +68,6 @@ def test_setupVars_are_sourced_to_global_scope(host): assert "{}={}".format(k, v) in output -def test_setupVars_saved_to_file(host): - """ - confirm saved settings are written to a file for future updates to re-use - """ - # dedent works better with this and padding matching script below - set_setup_vars = "\n" - for k, v in SETUPVARS.items(): - set_setup_vars += " {}={}\n".format(k, v) - host.run(set_setup_vars) - - script = dedent( - """\ - set -e - echo start - TERM=xterm - source /opt/pihole/basic-install.sh - source /opt/pihole/utils.sh - {} - mkdir -p /etc/dnsmasq.d - version_check_dnsmasq - echo "" > /etc/pihole/pihole-FTL.conf - finalExports - cat /etc/pihole/setupVars.conf - """.format( - set_setup_vars - ) - ) - - output = run_script(host, script).stdout - - for k, v in SETUPVARS.items(): - assert "{}={}".format(k, v) in output - - def test_selinux_not_detected(host): """ confirms installer continues when SELinux configuration file does not exist From 414df5b3724d3a5f5f6cb54d140bb737b27034a5 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 15 Jan 2023 13:54:40 +0000 Subject: [PATCH 013/413] Remove no longer required checks in test_installPihole_fresh_install_readableFiles Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 43 ------------------------------ 1 file changed, 43 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 059e3a8b..ef99d57b 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -105,9 +105,6 @@ def test_installPihole_fresh_install_readableFiles(host): mock_command_2( "systemctl", { - "enable lighttpd": ("", "0"), - "restart lighttpd": ("", "0"), - "start lighttpd": ("", "0"), "enable pihole-FTL": ("", "0"), "restart pihole-FTL": ("", "0"), "start pihole-FTL": ("", "0"), @@ -123,7 +120,6 @@ def test_installPihole_fresh_install_readableFiles(host): setup_var_file = "cat < /etc/pihole/setupVars.conf\n" for k, v in SETUPVARS.items(): setup_var_file += "{}={}\n".format(k, v) - setup_var_file += "INSTALL_WEB_SERVER=true\n" setup_var_file += "INSTALL_WEB_INTERFACE=true\n" setup_var_file += "EOF\n" host.run(setup_var_file) @@ -194,23 +190,6 @@ def test_installPihole_fresh_install_readableFiles(host): check_setup = test_cmd.format("r", "/etc/pihole/setupVars.conf", piholeuser) actual_rc = host.run(check_setup).rc assert exit_status_success == actual_rc - # check dnsmasq files - # readable /etc/dnsmasq.conf - check_dnsmasqconf = test_cmd.format("r", "/etc/dnsmasq.conf", piholeuser) - actual_rc = host.run(check_dnsmasqconf).rc - assert exit_status_success == actual_rc - # readable /etc/dnsmasq.d/01-pihole.conf - check_dnsmasqconf = test_cmd.format("r", "/etc/dnsmasq.d", piholeuser) - actual_rc = host.run(check_dnsmasqconf).rc - assert exit_status_success == actual_rc - check_dnsmasqconf = test_cmd.format("x", "/etc/dnsmasq.d", piholeuser) - actual_rc = host.run(check_dnsmasqconf).rc - assert exit_status_success == actual_rc - check_dnsmasqconf = test_cmd.format( - "r", "/etc/dnsmasq.d/01-pihole.conf", piholeuser - ) - actual_rc = host.run(check_dnsmasqconf).rc - assert exit_status_success == actual_rc # check readable and executable /etc/init.d/pihole-FTL check_init = test_cmd.format("x", "/etc/init.d/pihole-FTL", piholeuser) actual_rc = host.run(check_init).rc @@ -218,28 +197,6 @@ def test_installPihole_fresh_install_readableFiles(host): check_init = test_cmd.format("r", "/etc/init.d/pihole-FTL", piholeuser) actual_rc = host.run(check_init).rc assert exit_status_success == actual_rc - # check readable /etc/lighttpd/lighttpd.conf - check_lighttpd = test_cmd.format("r", "/etc/lighttpd/lighttpd.conf", piholeuser) - actual_rc = host.run(check_lighttpd).rc - assert exit_status_success == actual_rc - # check readable /etc/lighttpd/conf*/pihole-admin.conf - check_lighttpd = test_cmd.format("r", "/etc/lighttpd/conf.d", piholeuser) - if host.run(check_lighttpd).rc == exit_status_success: - check_lighttpd = test_cmd.format( - "r", "/etc/lighttpd/conf.d/pihole-admin.conf", piholeuser - ) - actual_rc = host.run(check_lighttpd).rc - assert exit_status_success == actual_rc - else: - check_lighttpd = test_cmd.format( - "r", "/etc/lighttpd/conf-available", piholeuser - ) - if host.run(check_lighttpd).rc == exit_status_success: - check_lighttpd = test_cmd.format( - "r", "/etc/lighttpd/conf-available/15-pihole-admin.conf", piholeuser - ) - actual_rc = host.run(check_lighttpd).rc - assert exit_status_success == actual_rc # check readable and executable manpages if maninstalled is True: check_man = test_cmd.format("x", "/usr/local/share/man", piholeuser) From 9bf372ef43f314cfee4770640725376d40cc6681 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 15 Jan 2023 13:58:05 +0000 Subject: [PATCH 014/413] [REVISIT] Remove test_installPihole_fresh_install_readableBlockpage for now. We may be able to recylcle it later, but I have my doubts Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 225 ----------------------------- 1 file changed, 225 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index ef99d57b..66d00814 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -255,231 +255,6 @@ def test_installPihole_fresh_install_readableFiles(host): actual_rc = host.run(check_pihole).rc -@pytest.mark.parametrize("test_webpage", [True]) -def test_installPihole_fresh_install_readableBlockpage(host, test_webpage): - """ - confirms all web page assets from Core repo are readable - by $LIGHTTPD_USER on a fresh build - """ - piholeWebpage = [ - "127.0.0.1", - # "pi.hole" - ] - # dialog returns Cancel for user prompt - mock_command("dialog", {"*": ("", "0")}, host) - - # mock git pull - mock_command_passthrough("git", {"pull": ("", "0")}, host) - # mock systemctl to start lighttpd and FTL - ligthttpdcommand = dedent( - r'''\"\" - echo 'starting lighttpd with {}' - if [ command -v "apt-get" >/dev/null 2>&1 ]; then - LIGHTTPD_USER="www-data" - LIGHTTPD_GROUP="www-data" - else - LIGHTTPD_USER="lighttpd" - LIGHTTPD_GROUP="lighttpd" - fi - mkdir -p "{run}" - chown {usergroup} "{run}" - mkdir -p "{cache}" - chown {usergroup} "/var/cache" - chown {usergroup} "{cache}" - mkdir -p "{compress}" - chown {usergroup} "{compress}" - mkdir -p "{uploads}" - chown {usergroup} "{uploads}" - chmod 0777 /var - chmod 0777 /var/cache - chmod 0777 "{cache}" - find "{run}" -type d -exec chmod 0777 {chmodarg} \;; - find "{run}" -type f -exec chmod 0666 {chmodarg} \;; - find "{compress}" -type d -exec chmod 0777 {chmodarg} \;; - find "{compress}" -type f -exec chmod 0666 {chmodarg} \;; - find "{uploads}" -type d -exec chmod 0777 {chmodarg} \;; - find "{uploads}" -type f -exec chmod 0666 {chmodarg} \;; - /usr/sbin/lighttpd -tt -f '{config}' - /usr/sbin/lighttpd -f '{config}' - echo \"\"'''.format( - "{}", - usergroup="${{LIGHTTPD_USER}}:${{LIGHTTPD_GROUP}}", - chmodarg="{{}}", - config="/etc/lighttpd/lighttpd.conf", - run="/run/lighttpd", - cache="/var/cache/lighttpd", - uploads="/var/cache/lighttpd/uploads", - compress="/var/cache/lighttpd/compress", - ) - ) - FTLcommand = dedent( - '''\"\" - set -x - /etc/init.d/pihole-FTL restart - echo \"\"''' - ) - mock_command_run( - "systemctl", - { - "enable lighttpd": ("", "0"), - "restart lighttpd": (ligthttpdcommand.format("restart"), "0"), - "start lighttpd": (ligthttpdcommand.format("start"), "0"), - "enable pihole-FTL": ("", "0"), - "restart pihole-FTL": (FTLcommand, "0"), - "start pihole-FTL": (FTLcommand, "0"), - "*": ('echo "systemctl call with $@"', "0"), - }, - host, - ) - # create configuration file - setup_var_file = "cat < /etc/pihole/setupVars.conf\n" - for k, v in SETUPVARS.items(): - setup_var_file += "{}={}\n".format(k, v) - setup_var_file += "INSTALL_WEB_SERVER=true\n" - setup_var_file += "INSTALL_WEB_INTERFACE=true\n" - setup_var_file += "EOF\n" - host.run(setup_var_file) - installWeb = host.run( - """ - export TERM=xterm - export DEBIAN_FRONTEND=noninteractive - umask 0027 - runUnattended=true - useUpdateVars=true - source /opt/pihole/basic-install.sh > /dev/null - runUnattended=true - useUpdateVars=true - main - echo "LIGHTTPD_USER=${LIGHTTPD_USER}" - echo "webroot=${webroot}" - echo "INSTALL_WEB_INTERFACE=${INSTALL_WEB_INTERFACE}" - echo "INSTALL_WEB_SERVER=${INSTALL_WEB_SERVER}" - """ - ) - assert 0 == installWeb.rc - piholeuser = "pihole" - webuser = "" - user = re.findall(r"^\s*LIGHTTPD_USER=.*$", installWeb.stdout, re.MULTILINE) - for match in user: - webuser = match.replace("LIGHTTPD_USER=", "").strip() - webroot = "" - user = re.findall(r"^\s*webroot=.*$", installWeb.stdout, re.MULTILINE) - for match in user: - webroot = match.replace("webroot=", "").strip() - if not webroot.strip(): - webroot = "/var/www/html" - installWebInterface = True - interface = re.findall( - r"^\s*INSTALL_WEB_INTERFACE=.*$", installWeb.stdout, re.MULTILINE - ) - for match in interface: - testvalue = match.replace("INSTALL_WEB_INTERFACE=", "").strip().lower() - if not testvalue.strip(): - installWebInterface = testvalue == "true" - installWebServer = True - server = re.findall(r"^\s*INSTALL_WEB_SERVER=.*$", installWeb.stdout, re.MULTILINE) - for match in server: - testvalue = match.replace("INSTALL_WEB_SERVER=", "").strip().lower() - if not testvalue.strip(): - installWebServer = testvalue == "true" - # if webserver install was not requested - # at least pihole must be able to read files - if installWebServer is False: - webuser = piholeuser - exit_status_success = 0 - test_cmd = 'su --shell /bin/bash --command "test -{0} {1}" -p {2}' - # check files that need a running FTL to be created - # readable and writeable pihole-FTL.db - check_FTLconf = test_cmd.format("r", "/etc/pihole/pihole-FTL.db", piholeuser) - actual_rc = host.run(check_FTLconf).rc - assert exit_status_success == actual_rc - check_FTLconf = test_cmd.format("w", "/etc/pihole/pihole-FTL.db", piholeuser) - actual_rc = host.run(check_FTLconf).rc - assert exit_status_success == actual_rc - # check directories above $webroot for read and execute permission - check_var = test_cmd.format("r", "/var", webuser) - actual_rc = host.run(check_var).rc - assert exit_status_success == actual_rc - check_var = test_cmd.format("x", "/var", webuser) - actual_rc = host.run(check_var).rc - assert exit_status_success == actual_rc - check_www = test_cmd.format("r", "/var/www", webuser) - actual_rc = host.run(check_www).rc - assert exit_status_success == actual_rc - check_www = test_cmd.format("x", "/var/www", webuser) - actual_rc = host.run(check_www).rc - assert exit_status_success == actual_rc - check_html = test_cmd.format("r", "/var/www/html", webuser) - actual_rc = host.run(check_html).rc - assert exit_status_success == actual_rc - check_html = test_cmd.format("x", "/var/www/html", webuser) - actual_rc = host.run(check_html).rc - assert exit_status_success == actual_rc - # check directories below $webroot for read and execute permission - check_admin = test_cmd.format("r", webroot + "/admin", webuser) - actual_rc = host.run(check_admin).rc - assert exit_status_success == actual_rc - check_admin = test_cmd.format("x", webroot + "/admin", webuser) - actual_rc = host.run(check_admin).rc - assert exit_status_success == actual_rc - directories = get_directories_recursive(host, webroot + "/admin/") - for directory in directories: - check_pihole = test_cmd.format("r", directory, webuser) - actual_rc = host.run(check_pihole).rc - check_pihole = test_cmd.format("x", directory, webuser) - actual_rc = host.run(check_pihole).rc - findfiles = 'find "{}" -maxdepth 1 -type f -exec echo {{}} \\;;' - filelist = host.run(findfiles.format(directory)) - files = list(filter(bool, filelist.stdout.splitlines())) - for file in files: - check_pihole = test_cmd.format("r", file, webuser) - actual_rc = host.run(check_pihole).rc - # check web interface files - # change nameserver to pi-hole - # setting nameserver in /etc/resolv.conf to pi-hole does - # not work here because of the way docker uses this file - ns = host.run(r"sed -i 's/nameserver.*/nameserver 127.0.0.1/' /etc/resolv.conf") - pihole_is_ns = ns.rc == 0 - - def is_ip(address): - m = re.match(r"(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})", address) - return bool(m) - - if installWebInterface is True: - if test_webpage is True: - # check webpage for unreadable files - noPHPfopen = re.compile( - ( - r"PHP Error(%d+):\s+fopen([^)]+):\s+" - + r"failed to open stream: " - + r"Permission denied in" - ), - re.I, - ) - # using cURL option --dns-servers is not possible - status = ( - 'curl -s --head "{}" | ' - + "head -n 1 | " - + 'grep "HTTP/1.[01] [23].." > /dev/null' - ) - digcommand = r"dig A +short {} @127.0.0.1 | head -n 1" - pagecontent = 'curl --verbose -L "{}"' - for page in piholeWebpage: - testpage = "http://" + page + "/admin/" - resolvesuccess = True - if is_ip(page) is False: - dig = host.run(digcommand.format(page)) - testpage = "http://" + dig.stdout.strip() + "/admin/" - resolvesuccess = dig.rc == 0 - if resolvesuccess or pihole_is_ns: - # check HTTP status of blockpage - actual_rc = host.run(status.format(testpage)) - assert exit_status_success == actual_rc.rc - # check for PHP error - actual_output = host.run(pagecontent.format(testpage)) - assert noPHPfopen.match(actual_output.stdout) is None - - def test_update_package_cache_success_no_errors(host): """ confirms package cache was updated without any errors From bf16fe4a376c30a84f842ec569327ef5dc28b41d Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 15 Jan 2023 15:47:10 +0000 Subject: [PATCH 015/413] Remove the EPEL related tests on centos/fedora - no longer neccasery Signed-off-by: Adam Warner --- test/test_centos_common_support.py | 27 --------------------------- test/test_fedora_support.py | 15 --------------- test/tox.centos_8.ini | 2 +- test/tox.centos_9.ini | 2 +- test/tox.fedora_36.ini | 2 +- test/tox.fedora_37.ini | 2 +- 6 files changed, 4 insertions(+), 46 deletions(-) delete mode 100644 test/test_centos_common_support.py delete mode 100644 test/test_fedora_support.py diff --git a/test/test_centos_common_support.py b/test/test_centos_common_support.py deleted file mode 100644 index 871fee29..00000000 --- a/test/test_centos_common_support.py +++ /dev/null @@ -1,27 +0,0 @@ -import pytest -from .conftest import ( - tick_box, - info_box, - cross_box, - mock_command, -) - - -def test_enable_epel_repository_centos(host): - """ - confirms the EPEL package repository is enabled when installed on CentOS - """ - package_manager_detect = host.run( - """ - source /opt/pihole/basic-install.sh - package_manager_detect - """ - ) - expected_stdout = info_box + ( - " Enabling EPEL package repository " "(https://fedoraproject.org/wiki/EPEL)" - ) - assert expected_stdout in package_manager_detect.stdout - expected_stdout = tick_box + " Installed" - assert expected_stdout in package_manager_detect.stdout - epel_package = host.package("epel-release") - assert epel_package.is_installed diff --git a/test/test_fedora_support.py b/test/test_fedora_support.py deleted file mode 100644 index e7d31a5d..00000000 --- a/test/test_fedora_support.py +++ /dev/null @@ -1,15 +0,0 @@ -def test_epel_and_remi_not_installed_fedora(host): - """ - confirms installer does not attempt to install EPEL/REMI repositories - on Fedora - """ - package_manager_detect = host.run( - """ - source /opt/pihole/basic-install.sh - package_manager_detect - """ - ) - assert package_manager_detect.stdout == "" - - epel_package = host.package("epel-release") - assert not epel_package.is_installed diff --git a/test/tox.centos_8.ini b/test/tox.centos_8.ini index dca77c93..85ae1ffb 100644 --- a/test/tox.centos_8.ini +++ b/test/tox.centos_8.ini @@ -5,4 +5,4 @@ envlist = py3 allowlist_externals = docker deps = -rrequirements.txt commands = docker buildx build --load --progress plain -f _centos_8.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py diff --git a/test/tox.centos_9.ini b/test/tox.centos_9.ini index a69c336a..4c51aefa 100644 --- a/test/tox.centos_9.ini +++ b/test/tox.centos_9.ini @@ -5,4 +5,4 @@ envlist = py3 allowlist_externals = docker deps = -rrequirements.txt commands = docker buildx build --load --progress plain -f _centos_9.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py diff --git a/test/tox.fedora_36.ini b/test/tox.fedora_36.ini index 515487ed..1896a45f 100644 --- a/test/tox.fedora_36.ini +++ b/test/tox.fedora_36.ini @@ -5,4 +5,4 @@ envlist = py3 allowlist_externals = docker deps = -rrequirements.txt commands = docker buildx build --load --progress plain -f _fedora_36.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py diff --git a/test/tox.fedora_37.ini b/test/tox.fedora_37.ini index 2a8ef398..9c8752cc 100644 --- a/test/tox.fedora_37.ini +++ b/test/tox.fedora_37.ini @@ -5,4 +5,4 @@ envlist = py3 allowlist_externals = docker deps = -rrequirements.txt commands = docker buildx build --load --progress plain -f _fedora_37.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py From 36956103004a99f59aa2258ff0880745d535f354 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 24 Jan 2023 19:51:01 +0100 Subject: [PATCH 016/413] Allow running pihole -g without root (run as user pihole) Signed-off-by: DL6ER --- gravity.sh | 2 +- pihole | 20 ++++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index fe2097dd..5b950b28 100755 --- a/gravity.sh +++ b/gravity.sh @@ -912,7 +912,7 @@ gravity_ShowCount # Determine if DNS has been restarted by this instance of gravity if [[ -z "${dnsWasOffline:-}" ]]; then - "${PIHOLE_COMMAND}" restartdns reload + "${PIHOLE_COMMAND}" reloaddns fi gravity_Cleanup diff --git a/pihole b/pihole index 1d9ad82c..6796acc6 100755 --- a/pihole +++ b/pihole @@ -513,13 +513,13 @@ if [[ $# = 0 ]]; then fi # functions that do not require sudo power +need_root=1 case "${1}" in "-h" | "help" | "--help" ) helpFunc;; "-v" | "version" ) versionFunc "$@";; "-c" | "chronometer" ) chronometerFunc "$@";; "-q" | "query" ) queryFunc "$@";; "status" ) statusFunc "$2";; - "tricorder" ) tricorderFunc;; # we need to add all arguments that require sudo power to not trigger the * argument @@ -532,13 +532,14 @@ case "${1}" in "-f" | "flush" ) ;; "-up" | "updatePihole" ) ;; "-r" | "reconfigure" ) ;; - "-g" | "updateGravity" ) ;; "-l" | "logging" ) ;; "uninstall" ) ;; "enable" ) ;; "disable" ) ;; "-d" | "debug" ) ;; "restartdns" ) ;; + "-g" | "updateGravity" ) need_root=0;; + "reloaddns" ) need_root=0;; "-a" | "admin" ) ;; "checkout" ) ;; "updatechecker" ) ;; @@ -547,8 +548,8 @@ case "${1}" in * ) helpFunc;; esac -# Must be root to use this tool -if [[ ! $EUID -eq 0 ]];then +# Must be root to use this tool for most functions +if [[ ! $EUID -eq 0 && need_root -eq 1 ]];then if [[ -x "$(command -v sudo)" ]]; then exec sudo bash "$0" "$@" exit $? @@ -557,6 +558,16 @@ if [[ ! $EUID -eq 0 ]];then exit 1 fi fi +# Can also be user pihole for other functions +if [[ ${USER} != "pihole" && need_root -eq 0 ]];then + if [[ -x "$(command -v sudo)" ]]; then + exec sudo -u pihole bash "$0" "$@" + exit $? + else + echo -e " ${CROSS} sudo is needed to run pihole commands. Please run this script as root or install sudo." + exit 1 + fi +fi # Handle redirecting to specific functions based on arguments case "${1}" in @@ -576,6 +587,7 @@ case "${1}" in "enable" ) piholeEnable 1;; "disable" ) piholeEnable 0 "$2";; "restartdns" ) restartDNS "$2";; + "reloaddns" ) restartDNS "reload";; "-a" | "admin" ) webpageFunc "$@";; "checkout" ) piholeCheckoutFunc "$@";; "updatechecker" ) shift; updateCheckFunc "$@";; From 3cb3adc5ca5796c6b4f79b943eb5563c1c0972a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 3 Feb 2023 19:55:55 +0100 Subject: [PATCH 017/413] Fix setting webpassword via pihole -a -p MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 1282b1ee..53f36a20 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -165,7 +165,7 @@ SetWebPassword() { echo "" if [ "${PASSWORD}" == "" ]; then - setFTLConfigValue "api.pwhash" "" + setFTLConfigValue "webserver.api.pwhash" "" >/dev/null echo -e " ${TICK} Password Removed" exit 0 fi @@ -178,7 +178,7 @@ SetWebPassword() { # We do not wrap this in brackets, otherwise BASH will expand any appropriate syntax hash=$(HashPassword "$PASSWORD") # Save hash to file - setFTLConfigValue "api.pwhash" "${hash}" + setFTLConfigValue "webserver.api.pwhash" "${hash}" >/dev/null echo -e " ${TICK} New password set" else echo -e " ${CROSS} Passwords don't match. Your password has not been changed" From 0568a69d8346d31372a1f35acc9f246369064001 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 13:15:47 +0000 Subject: [PATCH 018/413] Use WEBPORT instead of PRIVACY_LEVEL to set the web port... Signed-off-by: Adam Warner --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 87ff6d83..bce7aa91 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1625,7 +1625,7 @@ finalExports() { addOrEditKeyValPair "${FTL_CONFIG_FILE}" "PRIVACYLEVEL" "${PRIVACY_LEVEL}" # Set the web port - addOrEditKeyValPair "${FTL_CONFIG_FILE}" "WEBPORT" "${PRIVACY_LEVEL}" + addOrEditKeyValPair "${FTL_CONFIG_FILE}" "WEBPORT" "${WEBPORT}" # Bring in the current settings and the functions to manipulate them source "${setupVars}" From dbc6b814b2381e9db3d0361dff7cc7b80ef0236f Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 15:15:55 +0000 Subject: [PATCH 019/413] Should not need to reloadDNS manually any more as FTL will detect that it needs to reload by itself Signed-off-by: Adam Warner --- gravity.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index 5b950b28..e914732a 100755 --- a/gravity.sh +++ b/gravity.sh @@ -910,11 +910,6 @@ chmod g+w "${piholeDir}" "${gravityDBfile}" # Compute numbers to be displayed gravity_ShowCount -# Determine if DNS has been restarted by this instance of gravity -if [[ -z "${dnsWasOffline:-}" ]]; then - "${PIHOLE_COMMAND}" reloaddns -fi - gravity_Cleanup echo "" From 2784b267ec6424ae1235bf4d0a52d07fe7ab8c11 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 16:40:38 +0000 Subject: [PATCH 020/413] pihole command: read values from pihole-FTL instead of setupvars.conf Signed-off-by: Adam Warner --- advanced/Scripts/utils.sh | 4 ++-- pihole | 33 ++++++++++----------------------- 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index c97ad0cf..8bab396a 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -153,7 +153,7 @@ getFTLPID() { # Example getFTLConfigValue dns.piholePTR ####################### getFTLConfigValue(){ - pihole-FTL --config "${1}" + pihole-FTL --config -q "${1}" } ####################### @@ -166,5 +166,5 @@ getFTLConfigValue(){ # setFTLConfigValue dnsmasq.upstreams '[ "8.8.8.8" , "8.8.4.4" ]' ####################### setFTLConfigValue(){ - pihole-FTL --config "${1}" "${2}" + pihole-FTL --config "${1}" "${2}" >/dev/null } diff --git a/pihole b/pihole index 6796acc6..0c91df35 100755 --- a/pihole +++ b/pihole @@ -11,10 +11,9 @@ readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" -# setupVars and PI_HOLE_BIN_DIR are not readonly here because in some functions (checkout), +# PI_HOLE_BIN_DIR is not readonly here because in some functions (checkout), # they might get set again when the installer is sourced. This causes an # error due to modifying a readonly variable. -setupVars="/etc/pihole/setupVars.conf" PI_HOLE_BIN_DIR="/usr/local/bin" readonly colfile="${PI_HOLE_SCRIPT_DIR}/COL_TABLE" @@ -193,7 +192,7 @@ Time: elif [[ "${1}" == "0" ]]; then # Disable Pi-hole - if grep -cq "BLOCKING_ENABLED=false" "${setupVars}"; then + if ! getFTLConfigValue dns.blocking.active; then echo -e " ${INFO} Blocking already disabled, nothing to do" exit 0 fi @@ -233,19 +232,19 @@ Time: fi local str="Pi-hole Disabled" - addOrEditKeyValPair "${setupVars}" "BLOCKING_ENABLED" "false" + setFTLConfigValue dns.blocking.active false fi else # Enable Pi-hole killall -q pihole-reenable - if grep -cq "BLOCKING_ENABLED=true" "${setupVars}"; then + if getFTLConfigValue dns.blocking.active; then echo -e " ${INFO} Blocking already enabled, nothing to do" exit 0 fi echo -e " ${INFO} Enabling blocking" local str="Pi-hole Enabled" - addOrEditKeyValPair "${setupVars}" "BLOCKING_ENABLED" "true" + setFTLConfigValue dns.blocking.active true fi restartDNS reload-lists @@ -267,8 +266,7 @@ Options: exit 0 elif [[ "${1}" == "off" ]]; then # Disable logging - removeKey /etc/dnsmasq.d/01-pihole.conf "log-queries" - addOrEditKeyValPair "${setupVars}" "QUERY_LOGGING" "false" + setFTLConfigValue dns.queryLogging false if [[ "${2}" != "noflush" ]]; then # Flush logs "${PI_HOLE_BIN_DIR}"/pihole -f @@ -277,8 +275,7 @@ Options: local str="Logging has been disabled!" elif [[ "${1}" == "on" ]]; then # Enable logging - addKey /etc/dnsmasq.d/01-pihole.conf "log-queries" - addOrEditKeyValPair "${setupVars}" "QUERY_LOGGING" "true" + setFTLConfigValue dns.queryLogging true echo -e " ${INFO} Enabling logging..." local str="Logging has been enabled!" else @@ -354,26 +351,16 @@ statusFunc() { fi # Determine if Pi-hole's blocking is enabled - if grep -q "BLOCKING_ENABLED=false" /etc/pihole/setupVars.conf; then - # A config is commented out - case "${1}" in - "web") echo 0;; - *) echo -e " ${CROSS} Pi-hole blocking is disabled";; - esac - elif grep -q "BLOCKING_ENABLED=true" /etc/pihole/setupVars.conf; then - # Configs are set + if getFTLConfigValue dns.blocking.active; then case "${1}" in "web") echo "$port";; *) echo -e " ${TICK} Pi-hole blocking is enabled";; esac else - # No configs were found case "${1}" in - "web") echo -2;; - *) echo -e " ${INFO} Pi-hole blocking will be enabled";; + "web") echo 0;; + *) echo -e " ${CROSS} Pi-hole blocking is disabled";; esac - # Enable blocking - "${PI_HOLE_BIN_DIR}"/pihole enable fi exit 0 } From cd17040f959551fbc3250a4565e5fe547f595073 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 17:20:52 +0000 Subject: [PATCH 021/413] setupVars.conf not existing is no longer a critical error Signed-off-by: Adam Warner --- gravity.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index e914732a..515927e8 100755 --- a/gravity.sh +++ b/gravity.sh @@ -46,10 +46,6 @@ curl_connect_timeout=10 setupVars="${piholeDir}/setupVars.conf" if [[ -f "${setupVars}" ]];then source "${setupVars}" -else - echo -e " ${COL_LIGHT_RED}Installation Failure: ${setupVars} does not exist! ${COL_NC} - Please run 'pihole -r', and choose the 'reconfigure' option to fix." - exit 1 fi # Set up tmp dir variable in case it's not configured From 0e8f285f4f670d07be8c8b40d15ebdad3c4afa75 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 18:39:48 +0000 Subject: [PATCH 022/413] Always assume that the web interface will be installed. We could revist this again in future if we decide that there should still be a choice to install the interface or not Signed-off-by: Adam Warner --- advanced/Scripts/piholeCheckout.sh | 27 +++---- advanced/Scripts/piholeDebug.sh | 15 +--- advanced/Scripts/update.sh | 24 +++--- advanced/Scripts/updatecheck.sh | 28 +++---- advanced/Scripts/version.sh | 24 ++---- automated install/basic-install.sh | 115 ++++++++--------------------- automated install/uninstall.sh | 4 - test/test_any_automated_install.py | 1 - 8 files changed, 71 insertions(+), 167 deletions(-) diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index cf57800c..5a2eebd7 100755 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -61,12 +61,11 @@ checkout() { echo -e " Please re-run install script from https://github.com/pi-hole/pi-hole${COL_NC}" exit 1; fi - if [[ "${INSTALL_WEB_INTERFACE}" == "true" ]]; then - if ! is_repo "${webInterfaceDir}" ; then - echo -e " ${COL_LIGHT_RED}Error: Web Admin repo is missing from system!" - echo -e " Please re-run install script from https://github.com/pi-hole/pi-hole${COL_NC}" - exit 1; - fi + + if ! is_repo "${webInterfaceDir}" ; then + echo -e " ${COL_LIGHT_RED}Error: Web Admin repo is missing from system!" + echo -e " Please re-run install script from https://github.com/pi-hole/pi-hole${COL_NC}" + exit 1; fi if [[ -z "${1}" ]]; then @@ -85,11 +84,9 @@ checkout() { echo "" echo -e " ${INFO} Pi-hole Core" fetch_checkout_pull_branch "${PI_HOLE_FILES_DIR}" "development" || { echo " ${CROSS} Unable to pull Core development branch"; exit 1; } - if [[ "${INSTALL_WEB_INTERFACE}" == "true" ]]; then - echo "" - echo -e " ${INFO} Web interface" - fetch_checkout_pull_branch "${webInterfaceDir}" "devel" || { echo " ${CROSS} Unable to pull Web development branch"; exit 1; } - fi + echo "" + echo -e " ${INFO} Web interface" + fetch_checkout_pull_branch "${webInterfaceDir}" "devel" || { echo " ${CROSS} Unable to pull Web development branch"; exit 1; } #echo -e " ${TICK} Pi-hole Core" local path @@ -101,10 +98,8 @@ checkout() { echo -e " ${INFO} Shortcut \"master\" detected - checking out master branches..." echo -e " ${INFO} Pi-hole core" fetch_checkout_pull_branch "${PI_HOLE_FILES_DIR}" "master" || { echo " ${CROSS} Unable to pull Core master branch"; exit 1; } - if [[ ${INSTALL_WEB_INTERFACE} == "true" ]]; then - echo -e " ${INFO} Web interface" - fetch_checkout_pull_branch "${webInterfaceDir}" "master" || { echo " ${CROSS} Unable to pull Web master branch"; exit 1; } - fi + echo -e " ${INFO} Web interface" + fetch_checkout_pull_branch "${webInterfaceDir}" "master" || { echo " ${CROSS} Unable to pull Web master branch"; exit 1; } #echo -e " ${TICK} Web Interface" local path path="master/${binary}" @@ -137,7 +132,7 @@ checkout() { exit 1 fi checkout_pull_branch "${PI_HOLE_FILES_DIR}" "${2}" - elif [[ "${1}" == "web" ]] && [[ "${INSTALL_WEB_INTERFACE}" == "true" ]] ; then + elif [[ "${1}" == "web" ]] ; then str="Fetching branches from ${webInterfaceGitUrl}" echo -ne " ${INFO} $str" if ! fully_fetch_repo "${webInterfaceDir}" ; then diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 9f895aab..e7c383b5 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -291,17 +291,10 @@ compare_local_version_to_git_version() { return 1 fi else - # There is no git directory so check if the web interface was disabled - local setup_vars_web_interface - setup_vars_web_interface=$(< ${PIHOLE_SETUP_VARS_FILE} grep ^INSTALL_WEB_INTERFACE | cut -d '=' -f2) - if [[ "${pihole_component}" == "Web" ]] && [[ "${setup_vars_web_interface}" == "false" ]]; then - log_write "${INFO} ${pihole_component}: Disabled in setupVars.conf via INSTALL_WEB_INTERFACE=false" - else - # Return an error message - log_write "${COL_RED}Directory ${git_dir} doesn't exist${COL_NC}" - # and exit with a non zero code - return 1 - fi + # Return an error message + log_write "${COL_RED}Directory ${git_dir} doesn't exist${COL_NC}" + # and exit with a non zero code + return 1 fi } diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index c41c9232..6185d94a 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -128,20 +128,18 @@ main() { echo -e " ${INFO} Pi-hole Core:\\t${COL_LIGHT_GREEN}up to date${COL_NC}" fi - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - if ! is_repo "${ADMIN_INTERFACE_DIR}" ; then - echo -e "\\n ${COL_LIGHT_RED}Error: Web Admin repo is missing from system!" - echo -e " Please re-run install script from https://pi-hole.net${COL_NC}" - exit 1; - fi + if ! is_repo "${ADMIN_INTERFACE_DIR}" ; then + echo -e "\\n ${COL_LIGHT_RED}Error: Web Admin repo is missing from system!" + echo -e " Please re-run install script from https://pi-hole.net${COL_NC}" + exit 1; + fi - if GitCheckUpdateAvail "${ADMIN_INTERFACE_DIR}" ; then - web_update=true - echo -e " ${INFO} Web Interface:\\t${COL_YELLOW}update available${COL_NC}" - else - web_update=false - echo -e " ${INFO} Web Interface:\\t${COL_LIGHT_GREEN}up to date${COL_NC}" - fi + if GitCheckUpdateAvail "${ADMIN_INTERFACE_DIR}" ; then + web_update=true + echo -e " ${INFO} Web Interface:\\t${COL_YELLOW}update available${COL_NC}" + else + web_update=false + echo -e " ${INFO} Web Interface:\\t${COL_LIGHT_GREEN}up to date${COL_NC}" fi local funcOutput diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 0cc65218..7d7103d2 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -34,10 +34,6 @@ function get_remote_hash(){ git ls-remote "https://github.com/pi-hole/${1}" --tags "${2}" | awk '{print substr($0, 0,8);}' || return 1 } -# Source the setupvars config file -# shellcheck disable=SC1091 -. /etc/pihole/setupVars.conf - # Source the utils file for addOrEditKeyValPair() # shellcheck disable=SC1091 . /opt/pihole/utils.sh @@ -86,24 +82,20 @@ addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_CORE_HASH" "${GITHUB_CORE_HASH}" # get Web versions -if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then +WEB_VERSION="$(get_local_version /var/www/html/admin)" +addOrEditKeyValPair "${VERSION_FILE}" "WEB_VERSION" "${WEB_VERSION}" - WEB_VERSION="$(get_local_version /var/www/html/admin)" - addOrEditKeyValPair "${VERSION_FILE}" "WEB_VERSION" "${WEB_VERSION}" +WEB_BRANCH="$(get_local_branch /var/www/html/admin)" +addOrEditKeyValPair "${VERSION_FILE}" "WEB_BRANCH" "${WEB_BRANCH}" - WEB_BRANCH="$(get_local_branch /var/www/html/admin)" - addOrEditKeyValPair "${VERSION_FILE}" "WEB_BRANCH" "${WEB_BRANCH}" +WEB_HASH="$(get_local_hash /var/www/html/admin)" +addOrEditKeyValPair "${VERSION_FILE}" "WEB_HASH" "${WEB_HASH}" - WEB_HASH="$(get_local_hash /var/www/html/admin)" - addOrEditKeyValPair "${VERSION_FILE}" "WEB_HASH" "${WEB_HASH}" +GITHUB_WEB_VERSION="$(get_remote_version AdminLTE)" +addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_VERSION" "${GITHUB_WEB_VERSION}" - GITHUB_WEB_VERSION="$(get_remote_version AdminLTE)" - addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_VERSION" "${GITHUB_WEB_VERSION}" - - GITHUB_WEB_HASH="$(get_remote_hash AdminLTE "${WEB_BRANCH}")" - addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_HASH" "${GITHUB_WEB_HASH}" - -fi +GITHUB_WEB_HASH="$(get_remote_hash AdminLTE "${WEB_BRANCH}")" +addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_HASH" "${GITHUB_WEB_HASH}" # get FTL versions diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index 946c69fe..af86b045 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -8,10 +8,6 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. -# Source the setupvars config file -# shellcheck disable=SC1091 -. /etc/pihole/setupVars.conf - # Source the versions file poupulated by updatechecker.sh cachedVersions="/etc/pihole/versions" @@ -28,7 +24,7 @@ fi getLocalVersion() { case ${1} in "Pi-hole" ) echo "${CORE_VERSION:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_VERSION:=N/A}";; + "AdminLTE" ) echo "${WEB_VERSION:=N/A}";; "FTL" ) echo "${FTL_VERSION:=N/A}";; esac } @@ -36,7 +32,7 @@ getLocalVersion() { getLocalHash() { case ${1} in "Pi-hole" ) echo "${CORE_HASH:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_HASH:=N/A}";; + "AdminLTE" ) echo "${WEB_HASH:=N/A}";; "FTL" ) echo "${FTL_HASH:=N/A}";; esac } @@ -44,7 +40,7 @@ getLocalHash() { getRemoteHash(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_HASH:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${GITHUB_WEB_HASH:=N/A}";; + "AdminLTE" ) echo "${GITHUB_WEB_HASH:=N/A}";; "FTL" ) echo "${GITHUB_FTL_HASH:=N/A}";; esac } @@ -52,7 +48,7 @@ getRemoteHash(){ getRemoteVersion(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_VERSION:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${GITHUB_WEB_VERSION:=N/A}";; + "AdminLTE" ) echo "${GITHUB_WEB_VERSION:=N/A}";; "FTL" ) echo "${GITHUB_FTL_VERSION:=N/A}";; esac } @@ -60,16 +56,12 @@ getRemoteVersion(){ getLocalBranch(){ case ${1} in "Pi-hole" ) echo "${CORE_BRANCH:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_BRANCH:=N/A}";; + "AdminLTE" ) echo "${WEB_BRANCH:=N/A}";; "FTL" ) echo "${FTL_BRANCH:=N/A}";; esac } versionOutput() { - if [ "$1" = "AdminLTE" ] && [ "${INSTALL_WEB_INTERFACE}" != true ]; then - echo " WebAdmin not installed" - return 1 - fi [ "$2" = "-c" ] || [ "$2" = "--current" ] || [ -z "$2" ] && current=$(getLocalVersion "${1}") && branch=$(getLocalBranch "${1}") [ "$2" = "-l" ] || [ "$2" = "--latest" ] || [ -z "$2" ] && latest=$(getRemoteVersion "${1}") @@ -115,11 +107,7 @@ errorOutput() { defaultOutput() { versionOutput "Pi-hole" "$@" - - if [ "${INSTALL_WEB_INTERFACE}" = true ]; then - versionOutput "AdminLTE" "$@" - fi - + versionOutput "AdminLTE" "$@" versionOutput "FTL" "$@" } diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index bce7aa91..9e5a9963 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -91,7 +91,6 @@ IPV4_ADDRESS=${IPV4_ADDRESS} IPV6_ADDRESS=${IPV6_ADDRESS} # Give settings their default values. These may be changed by prompts later in the script. QUERY_LOGGING=true -INSTALL_WEB_INTERFACE=true WEBPORT=8080 PRIVACY_LEVEL=0 CACHE_SIZE=10000 @@ -1048,44 +1047,6 @@ setPrivacyLevel() { esac } -# Function to ask the user if they want to install the dashboard -setAdminFlag() { - # Similar to the logging function, ask what the user wants - dialog --no-shadow --keep-tite \ - --backtitle "Pihole Installation" \ - --title "Admin Web Interface" \ - --yesno "\\n\\nDo you want to install the Admin Web Interface?" \ - "${r}" "${c}" && result=0 || result=$? - - case ${result} in - "${DIALOG_OK}") - # If they chose yes, - printf " %b Installing Admin Web Interface\\n" "${INFO}" - # Set the flag to install the web interface - INSTALL_WEB_INTERFACE=true - - # Web port TODO: Below whiptail copy pasted from a previous go at this. needs converting to dialog - # Ask for the IPv4 address - WEBPORT=$(whiptail --backtitle "Setting web interface port" --title "Web Port" --inputbox "By default, pihole-FTL listens for http traffic on port 8080. If you wish to change the port, you may do so now. You can also do it later by editing /etc/pihole/pihole-FTL.conf" "${r}" "${c}" "${WEBPORT}" 3>&1 1>&2 2>&3) || \ - # Canceling IPv4 settings window - { echo -e " ${COL_LIGHT_RED}Cancel was selected, exiting installer${COL_NC}"; exit 1; } - printf " %b The Web interface will be accessible on port: %s\\n" "${INFO}" "${WEBPORT}" - - ;; - "${DIALOG_CANCEL}") - # If they chose no, - printf " %b Not installing Admin Web Interface\\n" "${INFO}" - # Set the flag to not install the web interface - INSTALL_WEB_INTERFACE=false - ;; - "${DIALOG_ESC}") - # User pressed - printf " %b Escape pressed, exiting installer at Admin Web Interface choice.%b\\n" "${COL_LIGHT_RED}" "${COL_NC}" - exit 1 - ;; - esac -} - # A function to display a list of example blocklists for users to select chooseBlocklists() { # Back up any existing adlist file, on the off chance that it exists. Useful in case of a reconfigure. @@ -1613,7 +1574,6 @@ finalExports() { addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_1" "${PIHOLE_DNS_1}" addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_2" "${PIHOLE_DNS_2}" addOrEditKeyValPair "${setupVars}" "QUERY_LOGGING" "${QUERY_LOGGING}" - addOrEditKeyValPair "${setupVars}" "INSTALL_WEB_INTERFACE" "${INSTALL_WEB_INTERFACE}" addOrEditKeyValPair "${setupVars}" "CACHE_SIZE" "${CACHE_SIZE}" addOrEditKeyValPair "${setupVars}" "DNS_FQDN_REQUIRED" "${DNS_FQDN_REQUIRED:-true}" addOrEditKeyValPair "${setupVars}" "DNS_BOGUS_PRIV" "${DNS_BOGUS_PRIV:-true}" @@ -1775,11 +1735,10 @@ displayFinalMessage() { # Else, inform the user that there is no set password. pwstring="NOT SET" fi - # If the user wants to install the dashboard, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - # Store a message in a variable and display it - additional="View the web interface at http://pi.hole/admin:${WEBPORT} or http://${IPV4_ADDRESS%/*}:${WEBPORT}/admin\\n\\nYour Admin Webpage login password is ${pwstring}" - fi + + # Store a message in a variable and display it + additional="View the web interface at http://pi.hole/admin:${WEBPORT} or http://${IPV4_ADDRESS%/*}:${WEBPORT}/admin\\n\\nYour Admin Webpage login password is ${pwstring}" + # Final completion message to user dialog --no-shadow --keep-tite \ @@ -1928,14 +1887,11 @@ clone_or_update_repos() { { printf " %b Unable to reset %s, exiting installer%b\\n" "${COL_LIGHT_RED}" "${PI_HOLE_LOCAL_REPO}" "${COL_NC}"; \ exit 1; \ } - # If the Web interface was installed, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - # reset it's repo - resetRepo ${webInterfaceDir} || \ - { printf " %b Unable to reset %s, exiting installer%b\\n" "${COL_LIGHT_RED}" "${webInterfaceDir}" "${COL_NC}"; \ - exit 1; \ - } - fi + # Reset the Web repo + resetRepo ${webInterfaceDir} || \ + { printf " %b Unable to reset %s, exiting installer%b\\n" "${COL_LIGHT_RED}" "${webInterfaceDir}" "${COL_NC}"; \ + exit 1; \ + } # Otherwise, a repair is happening else # so get git files for Core @@ -1943,14 +1899,11 @@ clone_or_update_repos() { { printf " %b Unable to clone %s into %s, unable to continue%b\\n" "${COL_LIGHT_RED}" "${piholeGitUrl}" "${PI_HOLE_LOCAL_REPO}" "${COL_NC}"; \ exit 1; \ } - # If the Web interface was installed, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - # get the Web git files - getGitFiles ${webInterfaceDir} ${webInterfaceGitUrl} || \ - { printf " %b Unable to clone %s into ${webInterfaceDir}, exiting installer%b\\n" "${COL_LIGHT_RED}" "${webInterfaceGitUrl}" "${COL_NC}"; \ - exit 1; \ - } - fi + # get the Web git files + getGitFiles ${webInterfaceDir} ${webInterfaceGitUrl} || \ + { printf " %b Unable to clone %s into ${webInterfaceDir}, exiting installer%b\\n" "${COL_LIGHT_RED}" "${webInterfaceGitUrl}" "${COL_NC}"; \ + exit 1; \ + } fi } @@ -2253,7 +2206,7 @@ main() { # when run via curl piping if [[ "$0" == "bash" ]]; then # Download the install script and run it with admin rights - exec curl -sSL https://raw.githubusercontent.com/pi-hole/pi-hole/master/automated%20install/basic-install.sh | sudo bash "$@" + exec curl -sSL https://install.pi-hole.net | sudo bash "$@" else # when run via calling local bash script exec sudo bash "$0" "$@" @@ -2319,8 +2272,6 @@ main() { setDNS # Give the user a choice of blocklists to include in their install. Or not. chooseBlocklists - # Let the user decide if they want the web interface to be installed automatically - setAdminFlag # Let the user decide if they want query logging enabled... setLogging # Let the user decide the FTL privacy level @@ -2373,17 +2324,13 @@ main() { # Copy the temp log file into final log location for storage copy_to_install_log - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - # Add password to web UI if there is none - pw="" - # If no password is set, - if [[ $(grep 'WEBPASSWORD' -c "${setupVars}") == 0 ]] ; then - # generate a random password - pw=$(tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c 8) - # shellcheck disable=SC1091 - . /opt/pihole/webpage.sh - echo "WEBPASSWORD=$(HashPassword "${pw}")" >> "${setupVars}" - fi + # Add password to web UI if there is none + pw="" + # If no password is set, + if [[ $(pihole-FTL --config webserver.api.pwhash) == "${pw}" ]] ; then + # generate a random password + pw=$(tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c 8) + pihole -a -p "${pw}" fi # Check for and disable systemd-resolved-DNSStubListener before reloading resolved @@ -2441,21 +2388,17 @@ main() { displayFinalMessage "${pw}" fi - # If the Web interface was installed, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - # If there is a password, - if (( ${#pw} > 0 )) ; then - # display the password - printf " %b Web Interface password: %b%s%b\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${pw}" "${COL_NC}" - printf " %b This can be changed using 'pihole -a -p'\\n\\n" "${INFO}" - fi + # If there is a password + if (( ${#pw} > 0 )) ; then + # display the password + printf " %b Web Interface password: %b%s%b\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${pw}" "${COL_NC}" + printf " %b This can be changed using 'pihole -a -p'\\n\\n" "${INFO}" fi if [[ "${useUpdateVars}" == false ]]; then # If the Web interface was installed, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - printf " %b View the web interface at http://pi.hole:${WEBPORT}/admin or http://%s/admin\\n\\n" "${INFO}" "${IPV4_ADDRESS%/*}:${WEBPORT}" - fi + printf " %b View the web interface at http://pi.hole:${WEBPORT}/admin or http://%s/admin\\n\\n" "${INFO}" "${IPV4_ADDRESS%/*}:${WEBPORT}" + # Explain to the user how to use Pi-hole as their DNS server printf " %b You may now configure your devices to use the Pi-hole as their DNS server\\n" "${INFO}" [[ -n "${IPV4_ADDRESS%/*}" ]] && printf " %b Pi-hole DNS (IPv4): %s\\n" "${INFO}" "${IPV4_ADDRESS%/*}" diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 7a1a290d..8cffae20 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -46,10 +46,6 @@ package_manager_detect # Uninstall packages used by the Pi-hole DEPS=("${INSTALLER_DEPS[@]}" "${PIHOLE_DEPS[@]}" "${OS_CHECK_DEPS[@]}") -if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - # Install the Web dependencies - DEPS+=("${PIHOLE_WEB_DEPS[@]}") -fi # Compatibility if [ -x "$(command -v apt-get)" ]; then diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 66d00814..42c94e63 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -120,7 +120,6 @@ def test_installPihole_fresh_install_readableFiles(host): setup_var_file = "cat < /etc/pihole/setupVars.conf\n" for k, v in SETUPVARS.items(): setup_var_file += "{}={}\n".format(k, v) - setup_var_file += "INSTALL_WEB_INTERFACE=true\n" setup_var_file += "EOF\n" host.run(setup_var_file) install = host.run( From 7cbe713873d38eb4b4952ea7e24a59f4c0084ed7 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 18:59:38 +0000 Subject: [PATCH 023/413] REVISIT: Don't rely on existence of setupVars.conf Signed-off-by: Adam Warner --- advanced/Scripts/update.sh | 3 -- advanced/Scripts/utils.sh | 4 +- automated install/basic-install.sh | 65 ++++++------------------------ test/conftest.py | 7 ---- test/test_any_automated_install.py | 47 --------------------- 5 files changed, 15 insertions(+), 111 deletions(-) diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index 6185d94a..b6153293 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -104,9 +104,6 @@ main() { web_update=false FTL_update=false - # shellcheck disable=1090,2154 - source "${setupVars}" - # Install packages used by this installation script (necessary if users have removed e.g. git from their systems) package_manager_detect install_dependent_packages "${INSTALLER_DEPS[@]}" diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 8bab396a..205edd90 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -162,8 +162,8 @@ getFTLConfigValue(){ # Takes two arguments: key and value # Example setFTLConfigValue dns.piholePTR PI.HOLE # -# Note, for complex values such as dnsmasq.upstreams, you should wrap the value in single quotes: -# setFTLConfigValue dnsmasq.upstreams '[ "8.8.8.8" , "8.8.4.4" ]' +# Note, for complex values such as dns.upstreams, you should wrap the value in single quotes: +# setFTLConfigValue dns.upstreams '[ "8.8.8.8" , "8.8.4.4" ]' ####################### setFTLConfigValue(){ pihole-FTL --config "${1}" "${2}" >/dev/null diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9e5a9963..984fe0c0 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -56,8 +56,6 @@ EOM # Location for final installation log storage installLogLoc="/etc/pihole/install.log" -# This is an important file as it contains information specific to the machine it's being installed on -setupVars="/etc/pihole/setupVars.conf" # This is a file used for the colorized output coltable="/opt/pihole/COL_TABLE" @@ -1566,29 +1564,7 @@ create_pihole_user() { fi } -# This function saves any changes to the setup variables into the setupvars.conf file for future runs finalExports() { - # set or update the variables in the file - - addOrEditKeyValPair "${setupVars}" "PIHOLE_INTERFACE" "${PIHOLE_INTERFACE}" - addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_1" "${PIHOLE_DNS_1}" - addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_2" "${PIHOLE_DNS_2}" - addOrEditKeyValPair "${setupVars}" "QUERY_LOGGING" "${QUERY_LOGGING}" - addOrEditKeyValPair "${setupVars}" "CACHE_SIZE" "${CACHE_SIZE}" - addOrEditKeyValPair "${setupVars}" "DNS_FQDN_REQUIRED" "${DNS_FQDN_REQUIRED:-true}" - addOrEditKeyValPair "${setupVars}" "DNS_BOGUS_PRIV" "${DNS_BOGUS_PRIV:-true}" - addOrEditKeyValPair "${setupVars}" "DNSMASQ_LISTENING" "${DNSMASQ_LISTENING:-local}" - - chmod 644 "${setupVars}" - - # Set the privacy level - addOrEditKeyValPair "${FTL_CONFIG_FILE}" "PRIVACYLEVEL" "${PRIVACY_LEVEL}" - - # Set the web port - addOrEditKeyValPair "${FTL_CONFIG_FILE}" "WEBPORT" "${WEBPORT}" - - # Bring in the current settings and the functions to manipulate them - source "${setupVars}" # shellcheck source=advanced/Scripts/webpage.sh source "${PI_HOLE_LOCAL_REPO}/advanced/Scripts/webpage.sh" @@ -1672,7 +1648,6 @@ installPihole() { # install a man page entry for pihole install_manpage - # Update setupvars.conf with any variables that may or may not have been changed during the install finalExports } @@ -1724,11 +1699,12 @@ checkSelinux() { # Installation complete message with instructions for the user displayFinalMessage() { + # TODO: COME BACK TO THIS, WHAT IS GOING ON? # If the number of arguments is > 0, if [[ "${#1}" -gt 0 ]] ; then # set the password to the first argument. pwstring="$1" - elif [[ $(grep 'WEBPASSWORD' -c "${setupVars}") -gt 0 ]]; then + elif [[ $(pihole-FTL --config webserver.api.pwhash) == '""' ]] ; then # Else if the password exists from previous setup, we'll load it later pwstring="unchanged" else @@ -2242,19 +2218,16 @@ main() { printf " %b Checking for / installing Required dependencies for this install script...\\n" "${INFO}" install_dependent_packages "${INSTALLER_DEPS[@]}" - # If the setup variable file exists, - if [[ -f "${setupVars}" ]]; then - # if it's running unattended, - if [[ "${runUnattended}" == true ]]; then - printf " %b Performing unattended setup, no dialogs will be displayed\\n" "${INFO}" - # Use the setup variables - useUpdateVars=true - # also disable debconf-apt-progress dialogs - export DEBIAN_FRONTEND="noninteractive" - else - # If running attended, show the available options (repair/reconfigure) - update_dialogs - fi + # if it's running unattended, + if [[ "${runUnattended}" == true ]]; then + printf " %b Performing unattended setup, no dialogs will be displayed\\n" "${INFO}" + # Use the setup variables + useUpdateVars=true + # also disable debconf-apt-progress dialogs + export DEBIAN_FRONTEND="noninteractive" + else + # If running attended, show the available options (repair/reconfigure) + update_dialogs fi if [[ "${useUpdateVars}" == false ]]; then @@ -2279,18 +2252,6 @@ main() { else # Setup adlist file if not exists installDefaultBlocklists - - # Source ${setupVars} to use predefined user variables in the functions - source "${setupVars}" - - # Get the privacy level if it exists (default is 0) - if [[ -f "${FTL_CONFIG_FILE}" ]]; then - # get the value from $FTL_CONFIG_FILE (and ignoring all commented lines) - PRIVACY_LEVEL=$(sed -e '/^[[:blank:]]*#/d' "${FTL_CONFIG_FILE}" | grep "PRIVACYLEVEL" | awk -F "=" 'NR==1{printf$2}') - - # If no setting was found, default to 0 - PRIVACY_LEVEL="${PRIVACY_LEVEL:-0}" - fi fi # Download or update the scripts by updating the appropriate git repos clone_or_update_repos @@ -2327,7 +2288,7 @@ main() { # Add password to web UI if there is none pw="" # If no password is set, - if [[ $(pihole-FTL --config webserver.api.pwhash) == "${pw}" ]] ; then + if [[ $(pihole-FTL --config webserver.api.pwhash) == '""' ]] ; then # generate a random password pw=$(tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c 8) pihole -a -p "${pw}" diff --git a/test/conftest.py b/test/conftest.py index e395ec27..164e8de5 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -4,13 +4,6 @@ import testinfra.backend.docker import subprocess from textwrap import dedent - -SETUPVARS = { - "PIHOLE_INTERFACE": "eth99", - "PIHOLE_DNS_1": "4.2.2.1", - "PIHOLE_DNS_2": "4.2.2.2", -} - IMAGE = "pytest_pihole:test_container" tick_box = "[\x1b[1;32m\u2713\x1b[0m]" diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 42c94e63..45cb01c5 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -2,7 +2,6 @@ import pytest from textwrap import dedent import re from .conftest import ( - SETUPVARS, tick_box, info_box, cross_box, @@ -32,42 +31,6 @@ def test_supported_package_manager(host): # assert package_manager_detect.rc == 1 -def test_setupVars_are_sourced_to_global_scope(host): - """ - currently update_dialogs sources setupVars with a dot, - then various other functions use the variables. - This confirms the sourced variables are in scope between functions - """ - setup_var_file = "cat < /etc/pihole/setupVars.conf\n" - for k, v in SETUPVARS.items(): - setup_var_file += "{}={}\n".format(k, v) - setup_var_file += "EOF\n" - host.run(setup_var_file) - - script = dedent( - """\ - set -e - printSetupVars() { - # Currently debug test function only - echo "Outputting sourced variables" - echo "PIHOLE_INTERFACE=${PIHOLE_INTERFACE}" - echo "PIHOLE_DNS_1=${PIHOLE_DNS_1}" - echo "PIHOLE_DNS_2=${PIHOLE_DNS_2}" - } - update_dialogs() { - . /etc/pihole/setupVars.conf - } - update_dialogs - printSetupVars - """ - ) - - output = run_script(host, script).stdout - - for k, v in SETUPVARS.items(): - assert "{}={}".format(k, v) in output - - def test_selinux_not_detected(host): """ confirms installer continues when SELinux configuration file does not exist @@ -116,12 +79,6 @@ def test_installPihole_fresh_install_readableFiles(host): host.run("command -v apt-get > /dev/null && apt-get install -qq man") host.run("command -v dnf > /dev/null && dnf install -y man") host.run("command -v yum > /dev/null && yum install -y man") - # create configuration file - setup_var_file = "cat < /etc/pihole/setupVars.conf\n" - for k, v in SETUPVARS.items(): - setup_var_file += "{}={}\n".format(k, v) - setup_var_file += "EOF\n" - host.run(setup_var_file) install = host.run( """ export TERM=xterm @@ -185,10 +142,6 @@ def test_installPihole_fresh_install_readableFiles(host): check_FTLconf = test_cmd.format("w", "/etc/pihole/pihole-FTL.conf", piholeuser) actual_rc = host.run(check_FTLconf).rc assert exit_status_success == actual_rc - # readable setupVars.conf - check_setup = test_cmd.format("r", "/etc/pihole/setupVars.conf", piholeuser) - actual_rc = host.run(check_setup).rc - assert exit_status_success == actual_rc # check readable and executable /etc/init.d/pihole-FTL check_init = test_cmd.format("x", "/etc/init.d/pihole-FTL", piholeuser) actual_rc = host.run(check_init).rc From 6b1c8a7fff92ae08a2b42d2e827249e6e2d345ca Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 21:30:31 +0000 Subject: [PATCH 024/413] @DL6ER changed the config key names... Remove superfluous test Signed-off-by: Adam Warner --- test/test_any_utils.py | 31 ++----------------------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 741b1127..8dc3f1a5 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -172,33 +172,6 @@ def test_getFTLPIDFile_and_getFTLPID_custom(host): assert expected_stdout == output.stdout -def test_setFTLConfigValue_getFTLConfigValue(host): - """ - Confirms setFTLConfigValue works - Requires FTL to be installed, so we do that first (taken from test_FTL_binary_installed_and_responsive_no_errors) - """ - host.run( - """ - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - echo "new/http" > /etc/pihole/ftlbranch - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - - output = host.run( - """ - source /opt/pihole/utils.sh - setFTLConfigValue "dnsmasq.upstreams" '["1.1.1.1"]' - """ - ) - - assert '[ "1.1.1.1" ]' in output.stdout - - def test_getFTLConfigValue_getFTLConfigValue(host): """ Confirms getFTLConfigValue works (also assumes setFTLConfigValue works) @@ -219,8 +192,8 @@ def test_getFTLConfigValue_getFTLConfigValue(host): output = host.run( """ source /opt/pihole/utils.sh - setFTLConfigValue "dnsmasq.upstreams" '["9.9.9.9"]' > /dev/null - getFTLConfigValue "dnsmasq.upstreams" + setFTLConfigValue "dns.upstreams" '["9.9.9.9"]' > /dev/null + getFTLConfigValue "dns.upstreams" """ ) From a91eb48d48f18026a87067e5ba0ce86a4d91a125 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 11 Feb 2023 12:34:12 +0000 Subject: [PATCH 025/413] Remove some code from gravity that gets values from setupVars.conf / pihole-FTL.conf - use getFTLConfigValue from utils.sh instead Signed-off-by: Adam Warner --- gravity.sh | 39 ++++++++++++--------------------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/gravity.sh b/gravity.sh index 515927e8..66a90449 100755 --- a/gravity.sh +++ b/gravity.sh @@ -13,10 +13,17 @@ export LC_ALL=C -coltable="/opt/pihole/COL_TABLE" -source "${coltable}" +PI_HOLE_SCRIPT_DIR="/opt/pihole" +# Source utils.sh for GetFTLConfigValue +utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" +# shellcheck disable=SC1090 +. "${utilsfile}" + +coltable="${PI_HOLE_SCRIPT_DIR}/COL_TABLE" +# shellcheck disable=SC1090 +. "${coltable}" # shellcheck disable=SC1091 -source "/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh" +. "/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh" basename="pihole" PIHOLE_COMMAND="/usr/local/bin/${basename}" @@ -33,20 +40,13 @@ localList="${piholeDir}/local.list" VPNList="/etc/openvpn/ipp.txt" piholeGitDir="/etc/.pihole" -gravityDBfile_default="${piholeDir}/gravity.db" -# GRAVITYDB may be overwritten by source pihole-FTL.conf below -GRAVITYDB="${gravityDBfile_default}" +GRAVITYDB=$(getFTLConfigValue files.gravity) gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql" gravityDBcopy="${piholeGitDir}/advanced/Templates/gravity_copy.sql" domainsExtension="domains" curl_connect_timeout=10 -# Source setupVars from install script -setupVars="${piholeDir}/setupVars.conf" -if [[ -f "${setupVars}" ]];then - source "${setupVars}" -fi # Set up tmp dir variable in case it's not configured : "${GRAVITY_TMPDIR:=/tmp}" @@ -56,12 +56,6 @@ if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then GRAVITY_TMPDIR="/tmp" fi -# Source pihole-FTL from install script -pihole_FTL="${piholeDir}/pihole-FTL.conf" -if [[ -f "${pihole_FTL}" ]]; then - source "${pihole_FTL}" -fi - # Set this only after sourcing pihole-FTL.conf as the gravity database path may # have changed gravityDBfile="${GRAVITYDB}" @@ -69,15 +63,6 @@ gravityTEMPfile="${GRAVITYDB}_temp" gravityDIR="$(dirname -- "${gravityDBfile}")" gravityOLDfile="${gravityDIR}/gravity_old.db" -if [[ -z "${BLOCKINGMODE}" ]] ; then - BLOCKINGMODE="NULL" -fi - -# Determine if superseded pihole.conf exists -if [[ -r "${piholeDir}/pihole.conf" ]]; then - echo -e " ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}" -fi - # Generate new SQLite3 file from schema template generate_gravity_database() { if ! pihole-FTL sqlite3 "${gravityDBfile}" < "${gravityDBschema}"; then @@ -519,7 +504,7 @@ gravity_DownloadBlocklistFromUrl() { str="Status:" echo -ne " ${INFO} ${str} Pending..." blocked=false - case $BLOCKINGMODE in + case $(getFTLConfigValue dns.blocking.mode) in "IP-NODATA-AAAA"|"IP") # Get IP address of this domain ip="$(dig "${domain}" +short)" From c0e352094da78f1d888e34bbc6dc511e3e069bb0 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 11 Feb 2023 12:51:55 +0000 Subject: [PATCH 026/413] REVISIT: Remove some references to setupVars (some files can probably go entirely...) Signed-off-by: Adam Warner --- advanced/Scripts/chronometer.sh | 8 -------- advanced/Scripts/piholeCheckout.sh | 3 --- advanced/Scripts/piholeDebug.sh | 2 +- advanced/Scripts/utils.sh | 1 + advanced/Scripts/webpage.sh | 8 ++++---- 5 files changed, 6 insertions(+), 16 deletions(-) diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh index d69a56d3..99cff230 100755 --- a/advanced/Scripts/chronometer.sh +++ b/advanced/Scripts/chronometer.sh @@ -210,11 +210,6 @@ get_init_stats() { else temp_file="" fi - - # Test existence of setupVars config - if [[ -f "/etc/pihole/setupVars.conf" ]]; then - setupVars="/etc/pihole/setupVars.conf" - fi } get_sys_stats() { @@ -226,9 +221,6 @@ get_sys_stats() { # Update every 12 refreshes (Def: every 60s) count=$((count+1)) if [[ "$count" == "1" ]] || (( "$count" % 12 == 0 )); then - # Do not source setupVars if file does not exist - [[ -n "$setupVars" ]] && source "$setupVars" - mapfile -t ph_ver_raw < <(pihole -v -c 2> /dev/null | sed -n 's/^.* v/v/p') if [[ -n "${ph_ver_raw[0]}" ]]; then ph_core_ver="${ph_ver_raw[0]}" diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index 5a2eebd7..39d39b1c 100755 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -16,15 +16,12 @@ source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh" # webInterfaceDir set in basic-install.sh # piholeGitURL set in basic-install.sh # is_repo() sourced from basic-install.sh -# setupVars set in basic-install.sh # check_download_exists sourced from basic-install.sh # fully_fetch_repo sourced from basic-install.sh # get_available_branches sourced from basic-install.sh # fetch_checkout_pull_branch sourced from basic-install.sh # checkout_pull_branch sourced from basic-install.sh -source "${setupVars}" - warning1() { echo " Please note that changing branches severely alters your Pi-hole subsystems" echo " Features that work on the master branch, may not on a development branch" diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index e7c383b5..ef1d8558 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -77,7 +77,6 @@ PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" -PIHOLE_SETUP_VARS_FILE="${PIHOLE_DIRECTORY}/setupVars.conf" PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole-FTL.conf" PIHOLE_CUSTOM_HOSTS_FILE="${PIHOLE_DIRECTORY}/custom.list" PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" @@ -1465,6 +1464,7 @@ upload_to_tricorder() { # Run through all the functions we made make_temporary_log initialize_debug +# TODO: Address the reliance on setupVars.conf here. Should debug read pihole.toml directly, or rely on pihole-FTL --config? # setupVars.conf needs to be sourced before the networking so the values are # available to the other functions source_setup_variables diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 205edd90..8f33d678 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -25,6 +25,7 @@ # # Example usage: # addOrEditKeyValPair "/etc/pihole/setupVars.conf" "BLOCKING_ENABLED" "true" +# TODO: We miight not actually need this function in v6 ####################### addOrEditKeyValPair() { local file="${1}" diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 53f36a20..8fd0253b 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -12,6 +12,8 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. +# TODO - this entire file might be able to be removed in v6 + readonly dnsmasqconfig="/etc/dnsmasq.d/01-pihole.conf" readonly dhcpconfig="/etc/dnsmasq.d/02-pihole-dhcp.conf" readonly FTLconf="/etc/pihole/pihole-FTL.conf" @@ -175,10 +177,8 @@ SetWebPassword() { fi if [ "${PASSWORD}" == "${CONFIRM}" ] ; then - # We do not wrap this in brackets, otherwise BASH will expand any appropriate syntax - hash=$(HashPassword "$PASSWORD") - # Save hash to file - setFTLConfigValue "webserver.api.pwhash" "${hash}" >/dev/null + # pihole-FTL will automatically hash the password + setFTLConfigValue "webserver.api.password" "${PASSWORD}" >/dev/null echo -e " ${TICK} New password set" else echo -e " ${CROSS} Passwords don't match. Your password has not been changed" From ea748822ef5d6e0962a25005b32ea5357a85614f Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 12 Feb 2023 17:39:37 +0000 Subject: [PATCH 027/413] REVISIT: I don't _think_ this line is nessacery. Why should gravity restart FTL if it is offline? It might be offline on purpose. Signed-off-by: Adam Warner --- gravity.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gravity.sh b/gravity.sh index 66a90449..3b08ff2b 100755 --- a/gravity.sh +++ b/gravity.sh @@ -720,11 +720,11 @@ gravity_Cleanup() { echo -e "${OVER} ${TICK} ${str}" - # Only restart DNS service if offline - if ! pgrep pihole-FTL &> /dev/null; then - "${PIHOLE_COMMAND}" restartdns - dnsWasOffline=true - fi + # # Only restart DNS service if offline + # if ! pgrep pihole-FTL &> /dev/null; then + # "${PIHOLE_COMMAND}" restartdns + # dnsWasOffline=true + # fi # Print Pi-hole status if an error occurred if [[ -n "${error}" ]]; then @@ -894,4 +894,4 @@ gravity_ShowCount gravity_Cleanup echo "" -"${PIHOLE_COMMAND}" status +# "${PIHOLE_COMMAND}" status From 44bfb8ebf095caa2f68d68e89e73fb194c4034c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 24 Feb 2023 21:11:19 +0100 Subject: [PATCH 028/413] Remove the ability to reboot/shutdown via webpage.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 8fd0253b..4c321ec2 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -401,14 +401,6 @@ SetExcludeClients() { addOrEditKeyValPair "${setupVars}" "API_EXCLUDE_CLIENTS" "${args[2]}" } -Poweroff(){ - nohup bash -c "sleep 5; poweroff" &> /dev/null /dev/null Date: Fri, 24 Feb 2023 21:31:35 +0100 Subject: [PATCH 029/413] Remove getFTLAPIPort() function and fix pihole status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/chronometer.sh | 2 +- advanced/Scripts/utils.sh | 23 ----------------- pihole | 7 +++-- test/test_any_utils.py | 46 --------------------------------- 4 files changed, 4 insertions(+), 74 deletions(-) diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh index 99cff230..49de6efd 100755 --- a/advanced/Scripts/chronometer.sh +++ b/advanced/Scripts/chronometer.sh @@ -16,7 +16,7 @@ pihole-FTL() { local ftl_port LINE # shellcheck disable=SC1091 . /opt/pihole/utils.sh - ftl_port=$(getFTLAPIPort) + ftl_port=$(getFTLConfigValue dns.port) if [[ -n "$ftl_port" ]]; then # Open connection to FTL exec 3<>"/dev/tcp/127.0.0.1/$ftl_port" diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 8f33d678..3f9b7031 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -81,29 +81,6 @@ removeKey() { sed -i "/^${key}/d" "${file}" } - -####################### -# returns FTL's current telnet API port based on the setting in /etc/pihole-FTL.conf -######################## -getFTLAPIPort(){ - local FTLCONFFILE="/etc/pihole/pihole-FTL.conf" - local DEFAULT_FTL_PORT=4711 - local ftl_api_port - - if [ -s "$FTLCONFFILE" ]; then - # if FTLPORT is not set in pihole-FTL.conf, use the default port - ftl_api_port="$({ grep '^FTLPORT=' "${FTLCONFFILE}" || echo "${DEFAULT_FTL_PORT}"; } | cut -d'=' -f2-)" - # Exploit prevention: set the port to the default port if there is malicious (non-numeric) - # content set in pihole-FTL.conf - expr "${ftl_api_port}" : "[^[:digit:]]" > /dev/null && ftl_api_port="${DEFAULT_FTL_PORT}" - else - # if there is no pihole-FTL.conf, use the default port - ftl_api_port="${DEFAULT_FTL_PORT}" - fi - - echo "${ftl_api_port}" -} - ####################### # returns path of FTL's PID file ####################### diff --git a/pihole b/pihole index 0c91df35..0be995e4 100755 --- a/pihole +++ b/pihole @@ -320,13 +320,12 @@ analyze_ports() { statusFunc() { # Determine if there is pihole-FTL service is listening - local pid port ftl_api_port ftl_pid_file + local pid port ftl_pid_file ftl_pid_file="$(getFTLPIDFile)" pid="$(getFTLPID ${ftl_pid_file})" - ftl_api_port="$(getFTLAPIPort)" if [[ "$pid" -eq "-1" ]]; then case "${1}" in "web") echo "-1";; @@ -334,8 +333,8 @@ statusFunc() { esac return 0 else - #get the DNS port pihole-FTL is listening on by using FTL's telnet API - port="$(echo ">dns-port >quit" | nc 127.0.0.1 "$ftl_api_port")" + # get the DNS port pihole-FTL is listening on + port="$(getFTLConfigValue dns.port)" if [[ "${port}" == "0" ]]; then case "${1}" in "web") echo "-1";; diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 8dc3f1a5..efab3760 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -82,52 +82,6 @@ def test_key_removal_works(host): assert expected_stdout == output.stdout -def test_getFTLAPIPort_default(host): - """Confirms getFTLAPIPort returns the default API port""" - output = host.run( - """ - source /opt/pihole/utils.sh - getFTLAPIPort - """ - ) - expected_stdout = "4711\n" - assert expected_stdout == output.stdout - - -def test_getFTLAPIPort_custom(host): - """Confirms getFTLAPIPort returns a custom API port""" - host.run( - """ - echo "FTLPORT=1234" > /etc/pihole/pihole-FTL.conf - """ - ) - output = host.run( - """ - source /opt/pihole/utils.sh - getFTLAPIPort - """ - ) - expected_stdout = "1234\n" - assert expected_stdout == output.stdout - - -def test_getFTLAPIPort_malicious(host): - """Confirms getFTLAPIPort returns 4711 if the setting in pihole-FTL.conf contains non-digits""" - host.run( - """ - echo "FTLPORT=*$ssdfsd" > /etc/pihole/pihole-FTL.conf - """ - ) - output = host.run( - """ - source /opt/pihole/utils.sh - getFTLAPIPort - """ - ) - expected_stdout = "4711\n" - assert expected_stdout == output.stdout - - def test_getFTLPIDFile_default(host): """Confirms getFTLPIDFile returns the default PID file path""" output = host.run( From 137e6dc1843c2b9ccfd1e66632f9b476a4d5fd0d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 28 May 2023 08:57:14 +0200 Subject: [PATCH 030/413] Remove all the undocumented now useless stuff from pihole -a Signed-off-by: DL6ER --- advanced/Scripts/webpage.sh | 790 +----------------------------------- 1 file changed, 5 insertions(+), 785 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 4c321ec2..67cbe766 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -43,126 +43,23 @@ fi helpFunc() { echo "Usage: pihole -a [options] Example: pihole -a -p password -Set options for the Admin Console +Set options for the API/Web interface Options: - -p, password Set Admin Console password - -c, celsius Set Celsius as preferred temperature unit - -f, fahrenheit Set Fahrenheit as preferred temperature unit - -k, kelvin Set Kelvin as preferred temperature unit - -h, --help Show this help dialog - -i, interface Specify dnsmasq's interface listening behavior - -l, privacylevel Set privacy level (0 = lowest, 3 = highest) - -t, teleporter Backup configuration as an archive - -t, teleporter myname.tar.gz Backup configuration to archive with name myname.tar.gz as specified" + -p, password Set API/Web interface password + -h, --help Show this help dialog" exit 0 } -add_setting() { - addOrEditKeyValPair "${setupVars}" "${1}" "${2}" -} - -delete_setting() { - removeKey "${setupVars}" "${1}" -} - -change_setting() { - addOrEditKeyValPair "${setupVars}" "${1}" "${2}" -} - -addFTLsetting() { - addOrEditKeyValPair "${FTLconf}" "${1}" "${2}" -} - -deleteFTLsetting() { - removeKey "${FTLconf}" "${1}" -} - -changeFTLsetting() { - pihole-FTL --config "${1}" "${2}" -} - -add_dnsmasq_setting() { - addOrEditKeyValPair "${dnsmasqconfig}" "${1}" "${2}" -} - -delete_dnsmasq_setting() { - removeKey "${dnsmasqconfig}" "${1}" -} - -SetTemperatureUnit() { - addOrEditKeyValPair "${setupVars}" "TEMPERATUREUNIT" "${unit}" - echo -e " ${TICK} Set temperature unit to ${unit}" -} - -HashPassword() { - # Compute password hash twice to avoid rainbow table vulnerability - return=$(echo -n "${1}" | sha256sum | sed 's/\s.*$//') - return=$(echo -n "${return}" | sha256sum | sed 's/\s.*$//') - echo "${return}" -} - -# Check an IP address to see if it is a valid one -valid_ip() { - # Local, named variables - local ip=${1} - local stat=1 - - # Regex matching one IPv4 component, i.e. an integer from 0 to 255. - # See https://tools.ietf.org/html/rfc1340 - local ipv4elem="(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]?|0)"; - # Regex matching an optional port (starting with '#') range of 1-65536 - local portelem="(#(6553[0-5]|655[0-2][0-9]|65[0-4][0-9]{2}|6[0-4][0-9]{3}|[1-5][0-9]{4}|[1-9][0-9]{0,3}|0))?"; - # Build a full IPv4 regex from the above subexpressions - local regex="^${ipv4elem}\\.${ipv4elem}\\.${ipv4elem}\\.${ipv4elem}${portelem}$" - - # Evaluate the regex, and return the result - [[ $ip =~ ${regex} ]] - - stat=$? - return "${stat}" -} - -valid_ip6() { - local ip=${1} - local stat=1 - - # Regex matching one IPv6 element, i.e. a hex value from 0000 to FFFF - local ipv6elem="[0-9a-fA-F]{1,4}" - # Regex matching an IPv6 CIDR, i.e. 1 to 128 - local v6cidr="(\\/([1-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])){0,1}" - # Regex matching an optional port (starting with '#') range of 1-65536 - local portelem="(#(6553[0-5]|655[0-2][0-9]|65[0-4][0-9]{2}|6[0-4][0-9]{3}|[1-5][0-9]{4}|[1-9][0-9]{0,3}|0))?"; - # Build a full IPv6 regex from the above subexpressions - local regex="^(((${ipv6elem}))*((:${ipv6elem}))*::((${ipv6elem}))*((:${ipv6elem}))*|((${ipv6elem}))((:${ipv6elem})){7})${v6cidr}${portelem}$" - - # Evaluate the regex, and return the result - [[ ${ip} =~ ${regex} ]] - - stat=$? - return "${stat}" -} - +# TODO: We can probably remove the reliance on this function too, just tell people to pihole-FTL --config webserver.api.password "password" SetWebPassword() { - if [ "${SUDO_USER}" == "www-data" ]; then - echo "Security measure: user www-data is not allowed to change webUI password!" - echo "Exiting" - exit 1 - fi - - if [ "${SUDO_USER}" == "lighttpd" ]; then - echo "Security measure: user lighttpd is not allowed to change webUI password!" - echo "Exiting" - exit 1 - fi - if (( ${#args[2]} > 0 )) ; then readonly PASSWORD="${args[2]}" readonly CONFIRM="${PASSWORD}" else # Prevents a bug if the user presses Ctrl+C and it continues to hide the text typed. # So we reset the terminal via stty if the user does press Ctrl+C - trap '{ echo -e "\nNo password will be set" ; stty sane ; exit 1; }' INT + trap '{ echo -e "\nNot changed" ; stty sane ; exit 1; }' INT read -s -r -p "Enter New Password (Blank for no password): " PASSWORD echo "" @@ -186,689 +83,12 @@ SetWebPassword() { fi } -ProcessDNSSettings() { - source "${setupVars}" - - removeKey "${dnsmasqconfig}" "server" - - COUNTER=1 - while true ; do - var=PIHOLE_DNS_${COUNTER} - if [ -z "${!var}" ]; then - break; - fi - addKey "${dnsmasqconfig}" "server=${!var}" - (( COUNTER++ )) - done - - # The option LOCAL_DNS_PORT is deprecated - # We apply it once more, and then convert it into the current format - if [ -n "${LOCAL_DNS_PORT}" ]; then - addOrEditKeyValPair "${dnsmasqconfig}" "server" "127.0.0.1#${LOCAL_DNS_PORT}" - addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_${COUNTER}" "127.0.0.1#${LOCAL_DNS_PORT}" - removeKey "${setupVars}" "LOCAL_DNS_PORT" - fi - - removeKey "${dnsmasqconfig}" "domain-needed" - removeKey "${dnsmasqconfig}" "expand-hosts" - - if [[ "${DNS_FQDN_REQUIRED}" == true ]]; then - addKey "${dnsmasqconfig}" "domain-needed" - addKey "${dnsmasqconfig}" "expand-hosts" - fi - - removeKey "${dnsmasqconfig}" "bogus-priv" - - if [[ "${DNS_BOGUS_PRIV}" == true ]]; then - addKey "${dnsmasqconfig}" "bogus-priv" - fi - - removeKey "${dnsmasqconfig}" "dnssec" - removeKey "${dnsmasqconfig}" "trust-anchor" - - if [[ "${DNSSEC}" == true ]]; then - echo "dnssec -trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC683457104237C7F8EC8D -" >> "${dnsmasqconfig}" - fi - - removeKey "${dnsmasqconfig}" "host-record" - - if [ -n "${HOSTRECORD}" ]; then - addOrEditKeyValPair "${dnsmasqconfig}" "host-record" "${HOSTRECORD}" - fi - - # Setup interface listening behavior of dnsmasq - removeKey "${dnsmasqconfig}" "interface" - removeKey "${dnsmasqconfig}" "local-service" - removeKey "${dnsmasqconfig}" "except-interface" - removeKey "${dnsmasqconfig}" "bind-interfaces" - - if [[ "${DNSMASQ_LISTENING}" == "all" ]]; then - # Listen on all interfaces, permit all origins - addOrEditKeyValPair "${dnsmasqconfig}" "except-interface" "nonexisting" - elif [[ "${DNSMASQ_LISTENING}" == "local" ]]; then - # Listen only on all interfaces, but only local subnets - addKey "${dnsmasqconfig}" "local-service" - else - # Options "bind" and "single" - # Listen only on one interface - # Use eth0 as fallback interface if interface is missing in setupVars.conf - if [ -z "${PIHOLE_INTERFACE}" ]; then - PIHOLE_INTERFACE="eth0" - fi - - addOrEditKeyValPair "${dnsmasqconfig}" "interface" "${PIHOLE_INTERFACE}" - - if [[ "${DNSMASQ_LISTENING}" == "bind" ]]; then - # Really bind to interface - addKey "${dnsmasqconfig}" "bind-interfaces" - fi - fi - - if [[ "${CONDITIONAL_FORWARDING}" == true ]]; then - # Convert legacy "conditional forwarding" to rev-server configuration - # Remove any existing REV_SERVER settings - removeKey "${setupVars}" "REV_SERVER" - removeKey "${setupVars}" "REV_SERVER_DOMAIN" - removeKey "${setupVars}" "REV_SERVER_TARGET" - removeKey "${setupVars}" "REV_SERVER_CIDR" - - REV_SERVER=true - addOrEditKeyValPair "${setupVars}" "REV_SERVER" "true" - - REV_SERVER_DOMAIN="${CONDITIONAL_FORWARDING_DOMAIN}" - addOrEditKeyValPair "${setupVars}" "REV_SERVER_DOMAIN" "${REV_SERVER_DOMAIN}" - - REV_SERVER_TARGET="${CONDITIONAL_FORWARDING_IP}" - addOrEditKeyValPair "${setupVars}" "REV_SERVER_TARGET" "${REV_SERVER_TARGET}" - - #Convert CONDITIONAL_FORWARDING_REVERSE if necessary e.g: - # 1.1.168.192.in-addr.arpa to 192.168.1.1/32 - # 1.168.192.in-addr.arpa to 192.168.1.0/24 - # 168.192.in-addr.arpa to 192.168.0.0/16 - # 192.in-addr.arpa to 192.0.0.0/8 - if [[ "${CONDITIONAL_FORWARDING_REVERSE}" == *"in-addr.arpa" ]];then - arrRev=("${CONDITIONAL_FORWARDING_REVERSE//./ }") - case ${#arrRev[@]} in - 6 ) REV_SERVER_CIDR="${arrRev[3]}.${arrRev[2]}.${arrRev[1]}.${arrRev[0]}/32";; - 5 ) REV_SERVER_CIDR="${arrRev[2]}.${arrRev[1]}.${arrRev[0]}.0/24";; - 4 ) REV_SERVER_CIDR="${arrRev[1]}.${arrRev[0]}.0.0/16";; - 3 ) REV_SERVER_CIDR="${arrRev[0]}.0.0.0/8";; - esac - else - # Set REV_SERVER_CIDR to whatever value it was set to - REV_SERVER_CIDR="${CONDITIONAL_FORWARDING_REVERSE}" - fi - - # If REV_SERVER_CIDR is not converted by the above, then use the REV_SERVER_TARGET variable to derive it - if [ -z "${REV_SERVER_CIDR}" ]; then - # Convert existing input to /24 subnet (preserves legacy behavior) - # This sed converts "192.168.1.2" to "192.168.1.0/24" - # shellcheck disable=2001 - REV_SERVER_CIDR="$(sed "s+\\.[0-9]*$+\\.0/24+" <<< "${REV_SERVER_TARGET}")" - fi - addOrEditKeyValPair "${setupVars}" "REV_SERVER_CIDR" "${REV_SERVER_CIDR}" - - # Remove obsolete settings from setupVars.conf - removeKey "${setupVars}" "CONDITIONAL_FORWARDING" - removeKey "${setupVars}" "CONDITIONAL_FORWARDING_REVERSE" - removeKey "${setupVars}" "CONDITIONAL_FORWARDING_DOMAIN" - removeKey "${setupVars}" "CONDITIONAL_FORWARDING_IP" - fi - - removeKey "${dnsmasqconfig}" "rev-server" - - if [[ "${REV_SERVER}" == true ]]; then - addKey "${dnsmasqconfig}" "rev-server=${REV_SERVER_CIDR},${REV_SERVER_TARGET}" - if [ -n "${REV_SERVER_DOMAIN}" ]; then - # Forward local domain names to the CF target, too - addKey "${dnsmasqconfig}" "server=/${REV_SERVER_DOMAIN}/${REV_SERVER_TARGET}" - fi - - if [[ "${DNS_FQDN_REQUIRED}" != true ]]; then - # Forward unqualified names to the CF target only when the "never - # forward non-FQDN" option is unticked - addKey "${dnsmasqconfig}" "server=//${REV_SERVER_TARGET}" - fi - - fi - - # We need to process DHCP settings here as well to account for possible - # changes in the non-FQDN forwarding. This cannot be done in 01-pihole.conf - # as we don't want to delete all local=/.../ lines so it's much safer to - # simply rewrite the entire corresponding config file (which is what the - # DHCP settings subroutine is doing) - ProcessDHCPSettings -} - -SetDNSServers() { - # Save setting to file - removeKey "${setupVars}" "PIHOLE_DNS" - IFS=',' read -r -a array <<< "${args[2]}" - for index in "${!array[@]}" - do - # Replace possible "\#" by "#". This fixes AdminLTE#1427 - local ip - ip="${array[index]//\\#/#}" - - if valid_ip "${ip}" || valid_ip6 "${ip}" ; then - addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_$((index+1))" "${ip}" - else - echo -e " ${CROSS} Invalid IP has been passed" - exit 1 - fi - done - - if [[ "${args[3]}" == "domain-needed" ]]; then - addOrEditKeyValPair "${setupVars}" "DNS_FQDN_REQUIRED" "true" - else - addOrEditKeyValPair "${setupVars}" "DNS_FQDN_REQUIRED" "false" - fi - - if [[ "${args[4]}" == "bogus-priv" ]]; then - addOrEditKeyValPair "${setupVars}" "DNS_BOGUS_PRIV" "true" - else - addOrEditKeyValPair "${setupVars}" "DNS_BOGUS_PRIV" "false" - fi - - if [[ "${args[5]}" == "dnssec" ]]; then - addOrEditKeyValPair "${setupVars}" "DNSSEC" "true" - else - addOrEditKeyValPair "${setupVars}" "DNSSEC" "false" - fi - - if [[ "${args[6]}" == "rev-server" ]]; then - addOrEditKeyValPair "${setupVars}" "REV_SERVER" "true" - addOrEditKeyValPair "${setupVars}" "REV_SERVER_CIDR" "${args[7]}" - addOrEditKeyValPair "${setupVars}" "REV_SERVER_TARGET" "${args[8]}" - addOrEditKeyValPair "${setupVars}" "REV_SERVER_DOMAIN" "${args[9]}" - else - addOrEditKeyValPair "${setupVars}" "REV_SERVER" "false" - fi - - ProcessDNSSettings - - # Restart dnsmasq to load new configuration - RestartDNS -} - -SetExcludeDomains() { - addOrEditKeyValPair "${setupVars}" "API_EXCLUDE_DOMAINS" "${args[2]}" -} - -SetExcludeClients() { - addOrEditKeyValPair "${setupVars}" "API_EXCLUDE_CLIENTS" "${args[2]}" -} - -RestartDNS() { - "${PI_HOLE_BIN_DIR}"/pihole restartdns -} - -SetQueryLogOptions() { - addOrEditKeyValPair "${setupVars}" "API_QUERY_LOG_SHOW" "${args[2]}" -} - -ProcessDHCPSettings() { - source "${setupVars}" - - if [[ "${DHCP_ACTIVE}" == "true" ]]; then - interface="${PIHOLE_INTERFACE}" - - # Use eth0 as fallback interface - if [ -z ${interface} ]; then - interface="eth0" - fi - - if [[ "${PIHOLE_DOMAIN}" == "" ]]; then - PIHOLE_DOMAIN="lan" - addOrEditKeyValPair "${setupVars}" "PIHOLE_DOMAIN" "${PIHOLE_DOMAIN}" - fi - - if [[ "${DHCP_LEASETIME}" == "0" ]]; then - leasetime="infinite" - elif [[ "${DHCP_LEASETIME}" == "" ]]; then - leasetime="24h" - addOrEditKeyValPair "${setupVars}" "DHCP_LEASETIME" "24" - else - leasetime="${DHCP_LEASETIME}h" - fi - - # Write settings to file - echo "############################################################################### -# DHCP SERVER CONFIG FILE AUTOMATICALLY POPULATED BY PI-HOLE WEB INTERFACE. # -# ANY CHANGES MADE TO THIS FILE WILL BE LOST ON CHANGE # -############################################################################### -dhcp-authoritative -dhcp-range=${DHCP_START},${DHCP_END},${leasetime} -dhcp-option=option:router,${DHCP_ROUTER} -dhcp-leasefile=/etc/pihole/dhcp.leases -#quiet-dhcp -" > "${dhcpconfig}" - chmod 644 "${dhcpconfig}" - - if [[ "${PIHOLE_DOMAIN}" != "none" ]]; then - echo "domain=${PIHOLE_DOMAIN}" >> "${dhcpconfig}" - - # When there is a Pi-hole domain set and "Never forward non-FQDNs" is - # ticked, we add `local=/domain/` to tell FTL that this domain is purely - # local and FTL may answer queries from /etc/hosts or DHCP but should - # never forward queries on that domain to any upstream servers - if [[ "${DNS_FQDN_REQUIRED}" == true ]]; then - echo "local=/${PIHOLE_DOMAIN}/" >> "${dhcpconfig}" - fi - fi - - # Sourced from setupVars - # shellcheck disable=SC2154 - if [[ "${DHCP_rapid_commit}" == "true" ]]; then - echo "dhcp-rapid-commit" >> "${dhcpconfig}" - fi - - if [[ "${DHCP_IPv6}" == "true" ]]; then - echo "#quiet-dhcp6 -#enable-ra -dhcp-option=option6:dns-server,[::] -dhcp-range=::,constructor:${interface},ra-names,ra-stateless,64 - -" >> "${dhcpconfig}" - fi - - else - if [[ -f "${dhcpconfig}" ]]; then - rm "${dhcpconfig}" &> /dev/null - fi - fi -} - -EnableDHCP() { - addOrEditKeyValPair "${setupVars}" "DHCP_ACTIVE" "true" - addOrEditKeyValPair "${setupVars}" "DHCP_START" "${args[2]}" - addOrEditKeyValPair "${setupVars}" "DHCP_END" "${args[3]}" - addOrEditKeyValPair "${setupVars}" "DHCP_ROUTER" "${args[4]}" - addOrEditKeyValPair "${setupVars}" "DHCP_LEASETIME" "${args[5]}" - addOrEditKeyValPair "${setupVars}" "PIHOLE_DOMAIN" "${args[6]}" - addOrEditKeyValPair "${setupVars}" "DHCP_IPv6" "${args[7]}" - addOrEditKeyValPair "${setupVars}" "DHCP_rapid_commit" "${args[8]}" - - # Remove possible old setting from file - removeKey "${dnsmasqconfig}" "dhcp-" - removeKey "${dnsmasqconfig}" "quiet-dhcp" - - # If a DHCP client claims that its name is "wpad", ignore that. - # This fixes a security hole. see CERT Vulnerability VU#598349 - # We also ignore "localhost" as Windows behaves strangely if a - # device claims this host name - addKey "${dnsmasqconfig}" "dhcp-name-match=set:hostname-ignore,wpad -dhcp-name-match=set:hostname-ignore,localhost -dhcp-ignore-names=tag:hostname-ignore" - - ProcessDHCPSettings - - RestartDNS -} - -DisableDHCP() { - addOrEditKeyValPair "${setupVars}" "DHCP_ACTIVE" "false" - - # Remove possible old setting from file - removeKey "${dnsmasqconfig}" "dhcp-" - removeKey "${dnsmasqconfig}" "quiet-dhcp" - - ProcessDHCPSettings - - RestartDNS -} - -SetWebUILayout() { - addOrEditKeyValPair "${setupVars}" "WEBUIBOXEDLAYOUT" "${args[2]}" -} - -SetWebUITheme() { - addOrEditKeyValPair "${setupVars}" "WEBTHEME" "${args[2]}" -} - -CheckUrl(){ - local regex check_url - # Check for characters NOT allowed in URLs - regex="[^a-zA-Z0-9:/?&%=~._()-;]" - - # this will remove first @ that is after schema and before domain - # \1 is optional schema, \2 is userinfo - check_url="$( sed -re 's#([^:/]*://)?([^/]+)@#\1\2#' <<< "$1" )" - - if [[ "${check_url}" =~ ${regex} ]]; then - return 1 - else - return 0 - fi -} - -CustomizeAdLists() { - local address - address="${args[3]}" - local comment - comment="${args[4]}" - - if CheckUrl "${address}"; then - if [[ "${args[2]}" == "enable" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'" - elif [[ "${args[2]}" == "disable" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'" - elif [[ "${args[2]}" == "add" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address, comment) VALUES ('${address}', '${comment}')" - elif [[ "${args[2]}" == "del" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'" - else - echo "Not permitted" - return 1 - fi - else - echo "Invalid Url" - return 1 - fi -} - -AddDHCPStaticAddress() { - mac="${args[2]}" - ip="${args[3]}" - host="${args[4]}" - - if [[ "${ip}" == "noip" ]]; then - # Static host name - echo "dhcp-host=${mac},${host}" >> "${dhcpstaticconfig}" - elif [[ "${host}" == "nohost" ]]; then - # Static IP - echo "dhcp-host=${mac},${ip}" >> "${dhcpstaticconfig}" - else - # Full info given - echo "dhcp-host=${mac},${ip},${host}" >> "${dhcpstaticconfig}" - fi -} - -RemoveDHCPStaticAddress() { - mac="${args[2]}" - if [[ "$mac" =~ ^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$ ]]; then - sed -i "/dhcp-host=${mac}.*/d" "${dhcpstaticconfig}" - else - echo " ${CROSS} Invalid Mac Passed!" - exit 1 - fi - -} - -SetListeningMode() { - source "${setupVars}" - - if [[ "$3" == "-h" ]] || [[ "$3" == "--help" ]]; then - echo "Usage: pihole -a -i [interface] -Example: 'pihole -a -i local' -Specify dnsmasq's network interface listening behavior - -Interfaces: - local Only respond to queries from devices that - are at most one hop away (local devices) - single Respond only on interface ${PIHOLE_INTERFACE} - bind Bind only on interface ${PIHOLE_INTERFACE} - all Listen on all interfaces, permit all origins" - exit 0 - fi - - if [[ "${args[2]}" == "all" ]]; then - echo -e " ${INFO} Listening on all interfaces, permitting all origins. Please use a firewall!" - addOrEditKeyValPair "${setupVars}" "DNSMASQ_LISTENING" "all" - elif [[ "${args[2]}" == "local" ]]; then - echo -e " ${INFO} Listening on all interfaces, permitting origins from one hop away (LAN)" - addOrEditKeyValPair "${setupVars}" "DNSMASQ_LISTENING" "local" - elif [[ "${args[2]}" == "bind" ]]; then - echo -e " ${INFO} Binding on interface ${PIHOLE_INTERFACE}" - addOrEditKeyValPair "${setupVars}" "DNSMASQ_LISTENING" "bind" - else - echo -e " ${INFO} Listening only on interface ${PIHOLE_INTERFACE}" - addOrEditKeyValPair "${setupVars}" "DNSMASQ_LISTENING" "single" - fi - - # Don't restart DNS server yet because other settings - # will be applied afterwards if "-web" is set - if [[ "${args[3]}" != "-web" ]]; then - ProcessDNSSettings - # Restart dnsmasq to load new configuration - RestartDNS - fi -} - -Teleporter() { - local filename - filename="${args[2]}" - if [[ -z "${filename}" ]]; then - local datetimestamp - local host - datetimestamp=$(date "+%Y-%m-%d_%H-%M-%S") - host=$(hostname) - host="${host//./_}" - filename="pi-hole-${host:-noname}-teleporter_${datetimestamp}.tar.gz" - fi - php "${webroot}/admin/scripts/pi-hole/php/teleporter.php" > "${filename}" -} - -checkDomain() -{ - local domain validDomain - # Convert to lowercase - domain="${1,,}" - validDomain=$(grep -P "^((-|_)*[a-z0-9]((-|_)*[a-z0-9)*(-|_)*)(\\.(-|_)*([a-z0-9]((-|_)*[a-z0-9])*))*$" <<< "${domain}") # Valid chars check - validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label - echo "${validDomain}" -} - -escapeDots() -{ - # SC suggest bashism ${variable//search/replace} - # shellcheck disable=SC2001 - escaped=$(echo "$1" | sed 's/\./\\./g') - echo "${escaped}" -} - -addAudit() -{ - shift # skip "-a" - shift # skip "audit" - local domains validDomain - domains="" - for domain in "$@" - do - # Check domain to be added. Only continue if it is valid - validDomain="$(checkDomain "${domain}")" - if [[ -n "${validDomain}" ]]; then - # Put comma in between domains when there is - # more than one domains to be added - # SQL INSERT allows adding multiple rows at once using the format - ## INSERT INTO table (domain) VALUES ('abc.de'),('fgh.ij'),('klm.no'),('pqr.st'); - if [[ -n "${domains}" ]]; then - domains="${domains}," - fi - domains="${domains}('${domain}')" - fi - done - # Insert only the domain here. The date_added field will be - # filled with its default value (date_added = current timestamp) - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};" -} - -clearAudit() -{ - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domain_audit;" -} - -SetPrivacyLevel() { - # Set privacy level. Minimum is 0, maximum is 3 - if [ "${args[2]}" -ge 0 ] && [ "${args[2]}" -le 3 ]; then - addOrEditKeyValPair "${FTLconf}" "PRIVACYLEVEL" "${args[2]}" - pihole restartdns reload-lists - fi -} - -AddCustomDNSAddress() { - echo -e " ${TICK} Adding custom DNS entry..." - - ip="${args[2]}" - host="${args[3]}" - reload="${args[4]}" - - validHost="$(checkDomain "${host}")" - if [[ -n "${validHost}" ]]; then - if valid_ip "${ip}" || valid_ip6 "${ip}" ; then - echo "${ip} ${validHost}" >> "${dnscustomfile}" - else - echo -e " ${CROSS} Invalid IP has been passed" - exit 1 - fi - else - echo " ${CROSS} Invalid Domain passed!" - exit 1 - fi - - # Restart dnsmasq to load new custom DNS entries only if $reload not false - if [[ ! $reload == "false" ]]; then - RestartDNS - fi -} - -RemoveCustomDNSAddress() { - echo -e " ${TICK} Removing custom DNS entry..." - - ip="${args[2]}" - host="${args[3]}" - reload="${args[4]}" - - validHost="$(checkDomain "${host}")" - if [[ -n "${validHost}" ]]; then - if valid_ip "${ip}" || valid_ip6 "${ip}" ; then - validHost=$(escapeDots "${validHost}") - sed -i "/^${ip} ${validHost}$/Id" "${dnscustomfile}" - else - echo -e " ${CROSS} Invalid IP has been passed" - exit 1 - fi - else - echo " ${CROSS} Invalid Domain passed!" - exit 1 - fi - - # Restart dnsmasq to load new custom DNS entries only if reload is not false - if [[ ! $reload == "false" ]]; then - RestartDNS - fi -} - -AddCustomCNAMERecord() { - echo -e " ${TICK} Adding custom CNAME record..." - - domain="${args[2]}" - target="${args[3]}" - reload="${args[4]}" - - validDomain="$(checkDomain "${domain}")" - if [[ -n "${validDomain}" ]]; then - validTarget="$(checkDomain "${target}")" - if [[ -n "${validTarget}" ]]; then - if [ "${validDomain}" = "${validTarget}" ]; then - echo " ${CROSS} Domain and target are the same. This would cause a DNS loop." - exit 1 - else - echo "cname=${validDomain},${validTarget}" >> "${dnscustomcnamefile}" - fi - else - echo " ${CROSS} Invalid Target Passed!" - exit 1 - fi - else - echo " ${CROSS} Invalid Domain passed!" - exit 1 - fi - # Restart dnsmasq to load new custom CNAME records only if reload is not false - if [[ ! $reload == "false" ]]; then - RestartDNS - fi -} - -RemoveCustomCNAMERecord() { - echo -e " ${TICK} Removing custom CNAME record..." - - domain="${args[2]}" - target="${args[3]}" - reload="${args[4]}" - - validDomain="$(checkDomain "${domain}")" - if [[ -n "${validDomain}" ]]; then - validTarget="$(checkDomain "${target}")" - if [[ -n "${validTarget}" ]]; then - validDomain=$(escapeDots "${validDomain}") - validTarget=$(escapeDots "${validTarget}") - sed -i "/^cname=${validDomain},${validTarget}$/Id" "${dnscustomcnamefile}" - else - echo " ${CROSS} Invalid Target Passed!" - exit 1 - fi - else - echo " ${CROSS} Invalid Domain passed!" - exit 1 - fi - - # Restart dnsmasq to update removed custom CNAME records only if $reload not false - if [[ ! $reload == "false" ]]; then - RestartDNS - fi -} - -SetRateLimit() { - local rate_limit_count rate_limit_interval reload - rate_limit_count="${args[2]}" - rate_limit_interval="${args[3]}" - reload="${args[4]}" - - # Set rate-limit setting inf valid - if [ "${rate_limit_count}" -ge 0 ] && [ "${rate_limit_interval}" -ge 0 ]; then - addOrEditKeyValPair "${FTLconf}" "RATE_LIMIT" "${rate_limit_count}/${rate_limit_interval}" - fi - - # Restart FTL to update rate-limit settings only if $reload not false - if [[ ! $reload == "false" ]]; then - RestartDNS - fi -} - main() { args=("$@") case "${args[1]}" in "-p" | "password" ) SetWebPassword;; - "-c" | "celsius" ) unit="C"; SetTemperatureUnit;; - "-f" | "fahrenheit" ) unit="F"; SetTemperatureUnit;; - "-k" | "kelvin" ) unit="K"; SetTemperatureUnit;; - "setdns" ) SetDNSServers;; - "setexcludedomains" ) SetExcludeDomains;; - "setexcludeclients" ) SetExcludeClients;; - "restartdns" ) RestartDNS;; - "setquerylog" ) SetQueryLogOptions;; - "enabledhcp" ) EnableDHCP;; - "disabledhcp" ) DisableDHCP;; - "layout" ) SetWebUILayout;; - "theme" ) SetWebUITheme;; "-h" | "--help" ) helpFunc;; - "addstaticdhcp" ) AddDHCPStaticAddress;; - "removestaticdhcp" ) RemoveDHCPStaticAddress;; - "-i" | "interface" ) SetListeningMode "$@";; - "-t" | "teleporter" ) Teleporter;; - "adlist" ) CustomizeAdLists;; - "audit" ) addAudit "$@";; - "clearaudit" ) clearAudit;; - "-l" | "privacylevel" ) SetPrivacyLevel;; - "addcustomdns" ) AddCustomDNSAddress;; - "removecustomdns" ) RemoveCustomDNSAddress;; - "addcustomcname" ) AddCustomCNAMERecord;; - "removecustomcname" ) RemoveCustomCNAMERecord;; - "ratelimit" ) SetRateLimit;; * ) helpFunc;; esac From f7b9d70054758137d60318e6df32ebb0d92987fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 30 May 2023 21:39:10 +0200 Subject: [PATCH 031/413] Remove man page for pihole-FTL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 8 +- manpages/pihole-FTL.8 | 154 ----------------------------- test/test_any_automated_install.py | 5 - 3 files changed, 5 insertions(+), 162 deletions(-) delete mode 100644 manpages/pihole-FTL.8 diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 984fe0c0..28813f1e 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1253,12 +1253,14 @@ install_manpage() { fi # Testing complete, copy the files & update the man db install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/manpages/pihole.8 /usr/local/share/man/man8/pihole.8 - install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.8 /usr/local/share/man/man8/pihole-FTL.8 - # remove previously installed "pihole-FTL.conf.5" man page + # remove previously installed man pages if [[ -f "/usr/local/share/man/man5/pihole-FTL.conf.5" ]]; then rm /usr/local/share/man/man5/pihole-FTL.conf.5 fi + if [[ -f "/usr/local/share/man/man5/pihole-FTL.8" ]]; then + rm /usr/local/share/man/man5/pihole-FTL.8 + fi if mandb -q &>/dev/null; then # Updated successfully @@ -1267,7 +1269,7 @@ install_manpage() { else # Something is wrong with the system's man installation, clean up # our files, (leave everything how we found it). - rm /usr/local/share/man/man8/pihole.8 /usr/local/share/man/man8/pihole-FTL.8 + rm /usr/local/share/man/man8/pihole.8 printf "%b %b man page db not updated, man pages not installed\\n" "${OVER}" "${CROSS}" fi } diff --git a/manpages/pihole-FTL.8 b/manpages/pihole-FTL.8 deleted file mode 100644 index c1b7550f..00000000 --- a/manpages/pihole-FTL.8 +++ /dev/null @@ -1,154 +0,0 @@ -.TH "Pihole-FTL" "8" "pihole-FTL" "Pi-hole" "November 2020" -.SH "NAME" -pihole-FTL - Pi-hole : The Faster-Than-Light (FTL) Engine -.br -.SH "SYNOPSIS" -\fBservice pihole-FTL \fR(\fBstart\fR|\fBstop\fR|\fBrestart\fR) -.br - -\fBpihole-FTL debug\fR -.br -\fBpihole-FTL test\fR -.br -\fBpihole-FTL -v|-vv\fR -.br -\fBpihole-FTL -t\fR -.br -\fBpihole-FTL -b\fR -.br -\fBpihole-FTL -f\fR -.br -\fBpihole-FTL -h\fR -.br -\fBpihole-FTL dnsmasq-test\fR -.br -\fBpihole-FTL regex-test str\fR -.br -\fBpihole-FTL regex-test str rgx\fR -.br -\fBpihole-FTL lua\fR -.br -\fBpihole-FTL luac\fR -.br -\fBpihole-FTL dhcp-discover\fR -.br -\fBpihole-FTL --\fR (\fBoptions\fR) -.br - -.SH "DESCRIPTION" -Pi-hole : The Faster-Than-Light (FTL) Engine is a lightweight, purpose-built daemon used to provide statistics needed for the Pi-hole Web Interface, and its API can be easily integrated into your own projects. Although it is an optional component of the Pi-hole ecosystem, it will be installed by default to provide statistics. As the name implies, FTL does its work \fIvery\fR \fIquickly\fR! -.br - -Usage -.br - -\fBservice pihole-FTL start\fR -.br - Start the pihole-FTL daemon -.br - -\fBservice pihole-FTL stop\fR -.br - Stop the pihole-FTL daemon -.br - -\fBservice pihole-FTL restart\fR -.br - If the pihole-FTP daemon is running, stop and then start, otherwise start. -.br - -Command line arguments -.br - -\fBdebug\fR -.br - Don't go into daemon mode (stay in foreground) + more verbose logging -.br - -\fBtest\fR -.br - Start FTL and process everything, but shut down immediately afterwards -.br - -\fB-v, version\fR -.br - Don't start FTL, show only version -.br - -\fB-vv\fR -.br - Don't start FTL, show verbose version information of embedded applications -.br - -\fB-t, tag\fR -.br - Don't start FTL, show only git tag -.br - -\fB-b, branch\fR -.br - Don't start FTL, show only git branch FTL was compiled from -.br - -\fB-f, no-daemon\fR -.br - Don't go into background (daemon mode) -.br - -\fB-h, help\fR -.br - Don't start FTL, show help -.br - -\fBdnsmasq-test\fR -.br - Test resolver config file syntax -.br - -\fBregex-test str\fR -.br - Test str against all regular expressions in the database -.br - -\fBregex-test str rgx\fR -.br - Test str against regular expression given by rgx -.br - -\fBlua\fR -.br - Start the embedded Lua interpreter -.br - -\fBluac\fR -.br - Execute the embedded Lua compiler -.br - -\fBdhcp-discover\fR -.br - Discover DHCP servers in the local network -.br - -\fB--\fR (options) -.br - Pass options to internal dnsmasq resolver -.br -.SH "EXAMPLE" -Command line arguments can be arbitrarily combined, e.g: -.br - -\fBpihole-FTL debug test\fR -.br - -Start ftl in foreground with more verbose logging, process everything and shutdown immediately -.br -.SH "SEE ALSO" -\fBpihole\fR(8) -.br -\fBFor FTL's config options please see https://docs.pi-hole.net/ftldns/configfile/\fR -.br -.SH "COLOPHON" - -Get sucked into the latest news and community activity by entering Pi-hole's orbit. Information about Pi-hole, and the latest version of the software can be found at https://pi-hole.net -.br diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 45cb01c5..3626ce6f 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -174,11 +174,6 @@ def test_installPihole_fresh_install_readableFiles(host): ) actual_rc = host.run(check_man).rc assert exit_status_success == actual_rc - check_man = test_cmd.format( - "r", "/usr/local/share/man/man8/pihole-FTL.8", piholeuser - ) - actual_rc = host.run(check_man).rc - assert exit_status_success == actual_rc # check not readable sudoers file check_sudo = test_cmd.format("r", "/etc/sudoers.d/pihole", piholeuser) actual_rc = host.run(check_sudo).rc From 792b0d419b20875dab023e311d532318f80ab2ff Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Thu, 1 Jun 2023 19:29:47 +0100 Subject: [PATCH 032/413] Test fixes Signed-off-by: Adam Warner --- automated install/basic-install.sh | 13 ------------- test/test_any_automated_install.py | 27 --------------------------- test/test_any_utils.py | 2 +- 3 files changed, 1 insertion(+), 41 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a6f6b865..6663bbad 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1566,17 +1566,6 @@ create_pihole_user() { fi } -finalExports() { - # shellcheck source=advanced/Scripts/webpage.sh - source "${PI_HOLE_LOCAL_REPO}/advanced/Scripts/webpage.sh" - - # Look for DNS server settings which would have to be reapplied - ProcessDNSSettings - - # Look for DHCP server settings which would have to be reapplied - ProcessDHCPSettings -} - # Install the logrotate script installLogrotate() { local str="Installing latest logrotate script" @@ -1649,8 +1638,6 @@ installPihole() { # install a man page entry for pihole install_manpage - - finalExports } # SELinux diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 07a18100..3626ce6f 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -781,30 +781,3 @@ def test_package_manager_has_web_deps(host): assert "No package" not in output.stdout assert output.rc == 0 - - -def test_webpage_sh_valid_domain(host): - """Confirms checkDomain function in webpage.sh works as expected""" - check1 = host.run( - """ - source /opt/pihole/webpage.sh - checkDomain "pi-hole.net" - """ - ) - check2 = host.run( - """ - source /opt/pihole/webpage.sh - checkDomain "ab.pi-hole.net" - """ - ) - - check3 = host.run( - """ - source /opt/pihole/webpage.sh - checkDomain "abc.pi-hole.net" - """ - ) - - assert "pi-hole.net" in check1.stdout - assert "ab.pi-hole.net" in check2.stdout - assert "abc.pi-hole.net" in check3.stdout diff --git a/test/test_any_utils.py b/test/test_any_utils.py index efab3760..1efd77c3 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -151,4 +151,4 @@ def test_getFTLConfigValue_getFTLConfigValue(host): """ ) - assert '[ "9.9.9.9" ]' in output.stdout + assert '[ 9.9.9.9 ]' in output.stdout From 001024b4da4be7de189c0ae3b7dde2cbe6aa1ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 1 Jun 2023 20:41:54 +0200 Subject: [PATCH 033/413] Fix Tests No2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/test_any_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 1efd77c3..154b5e16 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -151,4 +151,4 @@ def test_getFTLConfigValue_getFTLConfigValue(host): """ ) - assert '[ 9.9.9.9 ]' in output.stdout + assert "[ 9.9.9.9 ]" in output.stdout From a4322c624f5047123c5b8806f7e9222bfa89568f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 1 Jun 2023 22:00:40 +0200 Subject: [PATCH 034/413] Fix gravity in tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/test_any_automated_install.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 3626ce6f..c1ef8af3 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -79,6 +79,11 @@ def test_installPihole_fresh_install_readableFiles(host): host.run("command -v apt-get > /dev/null && apt-get install -qq man") host.run("command -v dnf > /dev/null && dnf install -y man") host.run("command -v yum > /dev/null && yum install -y man") + # Workaround to get FTLv6 installed until it reaches master branch + host.run(""" + echo "new/http" > /etc/pihole/ftlbranch + """ + ) install = host.run( """ export TERM=xterm From 0becc7615a74e50630a153030e4d0fa7aba87e9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 1 Jun 2023 23:51:03 +0200 Subject: [PATCH 035/413] Run prestart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/test_any_automated_install.py | 7 ++++--- test/test_any_utils.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index c1ef8af3..df9c4721 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -64,7 +64,7 @@ def test_installPihole_fresh_install_readableFiles(host): mock_command("dialog", {"*": ("", "0")}, host) # mock git pull mock_command_passthrough("git", {"pull": ("", "0")}, host) - # mock systemctl to not start lighttpd and FTL + # mock systemctl to not start FTL mock_command_2( "systemctl", { @@ -95,6 +95,7 @@ def test_installPihole_fresh_install_readableFiles(host): runUnattended=true useUpdateVars=true main + /opt/pihole/pihole-FTL-prestart.sh """ ) assert 0 == install.rc @@ -507,9 +508,9 @@ def test_FTL_download_aarch64_no_errors(host): assert "error" not in download_binary.stdout.lower() -def test_FTL_binary_installed_and_responsive_no_errors(host): +def test_FTL_development_binary_installed_and_responsive_no_errors(host): """ - confirms FTL binary is copied and functional in installed location + confirms FTL development binary is copied and functional in installed location """ host.run( """ diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 154b5e16..8c1ea521 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -129,7 +129,7 @@ def test_getFTLPIDFile_and_getFTLPID_custom(host): def test_getFTLConfigValue_getFTLConfigValue(host): """ Confirms getFTLConfigValue works (also assumes setFTLConfigValue works) - Requires FTL to be installed, so we do that first (taken from test_FTL_binary_installed_and_responsive_no_errors) + Requires FTL to be installed, so we do that first (taken from test_FTL_development_binary_installed_and_responsive_no_errors) """ host.run( """ From 7da57c6acddf6d77610b934f7902e134ee9ef377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 2 Jun 2023 00:03:22 +0200 Subject: [PATCH 036/413] Don't check and install old FTL config file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 10 ---------- test/test_any_automated_install.py | 12 +++--------- test/test_any_utils.py | 3 ++- 3 files changed, 5 insertions(+), 20 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 6663bbad..37e97e90 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -78,7 +78,6 @@ PI_HOLE_FILES=(chronometer list piholeDebug piholeLogFlush setupLCD update versi PI_HOLE_INSTALL_DIR="/opt/pihole" PI_HOLE_CONFIG_DIR="/etc/pihole" PI_HOLE_BIN_DIR="/usr/local/bin" -FTL_CONFIG_FILE="${PI_HOLE_CONFIG_DIR}/pihole-FTL.conf" if [ -z "$useUpdateVars" ]; then useUpdateVars=false fi @@ -1192,15 +1191,6 @@ installConfigs() { echo "${DNS_SERVERS}" > "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" chmod 644 "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" - # Install template file if it does not exist - if [[ ! -r "${FTL_CONFIG_FILE}" ]]; then - install -d -m 0755 ${PI_HOLE_CONFIG_DIR} - if ! install -T -o pihole -m 664 "${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole-FTL.conf" "${FTL_CONFIG_FILE}" &>/dev/null; then - printf " %b Error: Unable to initialize configuration file %s/pihole-FTL.conf\\n" "${COL_LIGHT_RED}" "${PI_HOLE_CONFIG_DIR}" - return 1 - fi - fi - # Install empty custom.list file if it does not exist if [[ ! -r "${PI_HOLE_CONFIG_DIR}/custom.list" ]]; then if ! install -o root -m 644 /dev/null "${PI_HOLE_CONFIG_DIR}/custom.list" &>/dev/null; then diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index df9c4721..d5f76ba3 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -80,9 +80,10 @@ def test_installPihole_fresh_install_readableFiles(host): host.run("command -v dnf > /dev/null && dnf install -y man") host.run("command -v yum > /dev/null && yum install -y man") # Workaround to get FTLv6 installed until it reaches master branch - host.run(""" - echo "new/http" > /etc/pihole/ftlbranch + host.run( """ + echo "new/http" > /etc/pihole/ftlbranch + """ ) install = host.run( """ @@ -141,13 +142,6 @@ def test_installPihole_fresh_install_readableFiles(host): check_macvendor = test_cmd.format("r", "/etc/pihole/macvendor.db", piholeuser) actual_rc = host.run(check_macvendor).rc assert exit_status_success == actual_rc - # readable and writeable pihole-FTL.conf - check_FTLconf = test_cmd.format("r", "/etc/pihole/pihole-FTL.conf", piholeuser) - actual_rc = host.run(check_FTLconf).rc - assert exit_status_success == actual_rc - check_FTLconf = test_cmd.format("w", "/etc/pihole/pihole-FTL.conf", piholeuser) - actual_rc = host.run(check_FTLconf).rc - assert exit_status_success == actual_rc # check readable and executable /etc/init.d/pihole-FTL check_init = test_cmd.format("x", "/etc/init.d/pihole-FTL", piholeuser) actual_rc = host.run(check_init).rc diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 8c1ea521..0f300457 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -129,7 +129,8 @@ def test_getFTLPIDFile_and_getFTLPID_custom(host): def test_getFTLConfigValue_getFTLConfigValue(host): """ Confirms getFTLConfigValue works (also assumes setFTLConfigValue works) - Requires FTL to be installed, so we do that first (taken from test_FTL_development_binary_installed_and_responsive_no_errors) + Requires FTL to be installed, so we do that first + (taken from test_FTL_development_binary_installed_and_responsive_no_errors) """ host.run( """ From ec82aec55fab2b0e751b3ff181d8aab8b653feab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 2 Jun 2023 00:13:57 +0200 Subject: [PATCH 037/413] centos_common_support.py was removed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/tox.fedora_38.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tox.fedora_38.ini b/test/tox.fedora_38.ini index 0aa7612e..d596092c 100644 --- a/test/tox.fedora_38.ini +++ b/test/tox.fedora_38.ini @@ -5,4 +5,4 @@ envlist = py3 allowlist_externals = docker deps = -rrequirements.txt commands = docker buildx build --load --progress plain -f _fedora_38.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py From 667418c71d0b29c927eb0c16eea4c4dadfd974a8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 31 May 2023 02:22:53 +0200 Subject: [PATCH 038/413] Use new FTL binary names Signed-off-by: DL6ER --- automated install/basic-install.sh | 46 ++++++++++++++---------------- test/test_any_automated_install.py | 24 +++++----------- 2 files changed, 29 insertions(+), 41 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 37e97e90..681b6798 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1951,39 +1951,29 @@ get_binary_name() { if [[ "${lib}" == "/lib/ld-linux-aarch64.so.1" ]]; then printf "%b %b Detected AArch64 (64 Bit ARM) processor\\n" "${OVER}" "${TICK}" # set the binary to be used - l_binary="pihole-FTL-aarch64-linux-gnu" + l_binary="pihole-FTL-arm64" elif [[ "${lib}" == "/lib/ld-linux-armhf.so.3" ]]; then # Hard-float available: Use gnueabihf binaries # If ARMv8 or higher is found (e.g., BCM2837 as found in Raspberry Pi Model 3B) if [[ "${rev}" -gt 7 ]]; then printf "%b %b Detected ARMv8 (or newer) processor\\n" "${OVER}" "${TICK}" # set the binary to be used - l_binary="pihole-FTL-armv8-linux-gnueabihf" + l_binary="pihole-FTL-armv8" elif [[ "${rev}" -eq 7 ]]; then # Otherwise, if ARMv7 is found (e.g., BCM2836 as found in Raspberry Pi Model 2) printf "%b %b Detected ARMv7 processor (with hard-float support)\\n" "${OVER}" "${TICK}" # set the binary to be used - l_binary="pihole-FTL-armv7-linux-gnueabihf" + l_binary="pihole-FTL-armv7" else # Otherwise, use the ARMv6 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) printf "%b %b Detected ARMv6 processor (with hard-float support)\\n" "${OVER}" "${TICK}" # set the binary to be used - l_binary="pihole-FTL-armv6-linux-gnueabihf" + l_binary="pihole-FTL-armv6" fi else - # No hard-float support found: Use gnueabi binaries - # Use the ARMv4-compliant binary only if we detected an ARMv4T core - if [[ "${rev}" -eq 4 ]]; then - printf "%b %b Detected ARMv4 processor\\n" "${OVER}" "${TICK}" - # set the binary to be used - l_binary="pihole-FTL-armv4-linux-gnueabi" - # Otherwise, use the ARMv5 binary. To date (end of 2020), all modern ARM processors - # are backwards-compatible to the ARMv5 - else - printf "%b %b Detected ARMv5 (or newer) processor\\n" "${OVER}" "${TICK}" - # set the binary to be used - l_binary="pihole-FTL-armv5-linux-gnueabi" - fi + # No hard-float support found + printf "%b %b%b ARM processor without hard-float support detected%b\\n" "${OVER}" "${COL_LIGHT_RED}" "${CROSS}" "${COL_NC}" + l_binary="" fi elif [[ "${machine}" == "x86_64" ]]; then # This gives the processor of packages dpkg installs (for example, "i386") @@ -1996,16 +1986,16 @@ get_binary_name() { # in the past (see https://github.com/pi-hole/pi-hole/pull/2004) if [[ "${dpkgarch}" == "i386" ]]; then printf "%b %b Detected 32bit (i686) processor\\n" "${OVER}" "${TICK}" - l_binary="pihole-FTL-linux-x86_32" + l_binary="pihole-FTL-386" else # 64bit printf "%b %b Detected x86_64 processor\\n" "${OVER}" "${TICK}" # set the binary to be used - l_binary="pihole-FTL-linux-x86_64" + l_binary="pihole-FTL-amd64" fi elif [[ "${machine}" == "riscv64" ]]; then printf "%b %b Detected riscv64 processor\\n" "${OVER}" "${TICK}" - l_binary="pihole-FTL-riscv64-linux-gnu" + l_binary="pihole-FTL-riscv64" else # Something else - we try to use 32bit executable and warn the user if [[ ! "${machine}" == "i686" ]]; then @@ -2015,7 +2005,7 @@ get_binary_name() { else printf "%b %b Detected 32bit (i686) processor\\n" "${OVER}" "${TICK}" fi - l_binary="pihole-FTL-linux-x86_32" + l_binary="pihole-FTL-linux-386" fi # Returning a string value via echo @@ -2180,6 +2170,16 @@ main() { fi fi + # Check if there is a usable FTL binary available on this architecture - do + # this early on as FTL is a hard dependency for Pi-hole + local funcOutput + funcOutput=$(get_binary_name) #Store output of get_binary_name here + # Abort early if this processor is not supported (get_binary_name returnS empty string) + if [[ "${funcOutput}" == "" ]]; then + printf " %b Upgrade/install aborted\\n" "${CROSS}" "${DISTRO_NAME}" + exit 1 + fi + # Check if SELinux is Enforcing and exit before doing anything else checkSelinux @@ -2249,9 +2249,7 @@ main() { # Create the pihole user create_pihole_user - # Check if FTL is installed - do this early on as FTL is a hard dependency for Pi-hole - local funcOutput - funcOutput=$(get_binary_name) #Store output of get_binary_name here + # Download and install FTL local binary binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL) local theRest diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index d5f76ba3..4a86d28e 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -268,9 +268,9 @@ def test_FTL_detect_aarch64_no_errors(host): assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv4t_no_errors(host): +def test_FTL_detect_armv4t_no_install(host): """ - confirms only armv4t package is downloaded for FTL engine + confirms armv4t architecture is not supported """ # mock uname to return armv4t platform mock_command("uname", {"-m": ("armv4t", "0")}, host) @@ -288,22 +288,17 @@ def test_FTL_detect_armv4t_no_errors(host): source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" """ ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv4 processor") - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" + expected_stdout = cross_box + (" ARM processor without hard-float support detected") assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv5te_no_errors(host): +def test_FTL_detect_armv5te_no_install(host): """ - confirms only armv5te package is downloaded for FTL engine + confirms armv5te architecture is not supported """ # mock uname to return armv5te platform mock_command("uname", {"-m": ("armv5te", "0")}, host) @@ -321,16 +316,11 @@ def test_FTL_detect_armv5te_no_errors(host): source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" """ ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv5 (or newer) processor") - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" + expected_stdout = cross_box + (" ARM processor without hard-float support detected") assert expected_stdout in detectPlatform.stdout @@ -375,7 +365,7 @@ def test_FTL_detect_armv7l_no_errors(host): """ # mock uname to return armv7l platform mock_command("uname", {"-m": ("armv7l", "0")}, host) - # mock ldd to respond with ld-linux-armhf shared library + # mock ldd to respond with ld-linux-armhf shared lib rary mock_command( "ldd", { From 366345e87e6dc94294ea746e8cea8f781864a9e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 4 Jun 2023 20:20:18 +0200 Subject: [PATCH 039/413] Fix no_installs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 2 +- test/test_any_automated_install.py | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 681b6798..c4ea4764 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2174,7 +2174,7 @@ main() { # this early on as FTL is a hard dependency for Pi-hole local funcOutput funcOutput=$(get_binary_name) #Store output of get_binary_name here - # Abort early if this processor is not supported (get_binary_name returnS empty string) + # Abort early if this processor is not supported (get_binary_name returns empty string) if [[ "${funcOutput}" == "" ]]; then printf " %b Upgrade/install aborted\\n" "${CROSS}" "${DISTRO_NAME}" exit 1 diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 4a86d28e..5c902c2b 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -286,12 +286,9 @@ def test_FTL_detect_armv4t_no_install(host): detectPlatform = host.run( """ source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) + get_binary_name """ ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout expected_stdout = cross_box + (" ARM processor without hard-float support detected") assert expected_stdout in detectPlatform.stdout @@ -314,12 +311,9 @@ def test_FTL_detect_armv5te_no_install(host): detectPlatform = host.run( """ source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) + get_binary_name """ ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout expected_stdout = cross_box + (" ARM processor without hard-float support detected") assert expected_stdout in detectPlatform.stdout From 53fb7ae0ef1b4eefcbf77ed2b4fbaed9a12c55ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 4 Jun 2023 21:06:44 +0200 Subject: [PATCH 040/413] Use shellcheck in smoke-tests. RIP stickler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fca8bb19..d5037f37 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,6 +23,13 @@ jobs: # If FAIL is 1 then we fail. [[ $FAIL == 1 ]] && exit 1 || echo "Scripts are executable!" + - name: Run shellcheck + uses: ludeeus/action-shellcheck@master + with: + check_together: 'yes' + format: tty + severity: error + - name: Spell-Checking uses: codespell-project/actions-codespell@master with: From ba3e290915a41dbe6a8b885e00ba0a796376fb58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 4 Jun 2023 21:21:48 +0200 Subject: [PATCH 041/413] Fix shellcheck errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeLogFlush.sh | 6 +++--- pihole | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 3473fad5..14542e4b 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -30,10 +30,10 @@ if [ -z "$DBFILE" ]; then DBFILE="/etc/pihole/pihole-FTL.db" fi -if [[ "$@" != *"quiet"* ]]; then +if [[ "$*" != *"quiet"* ]]; then echo -ne " ${INFO} Flushing /var/log/pihole/pihole.log ..." fi -if [[ "$@" == *"once"* ]]; then +if [[ "$*" == *"once"* ]]; then # Nightly logrotation if command -v /usr/sbin/logrotate >/dev/null; then # Logrotate once @@ -69,7 +69,7 @@ else sudo pihole restartdns fi -if [[ "$@" != *"quiet"* ]]; then +if [[ "$*" != *"quiet"* ]]; then echo -e "${OVER} ${TICK} Flushed /var/log/pihole/pihole.log" echo -e " ${TICK} Deleted ${deleted} queries from database" fi diff --git a/pihole b/pihole index 0be995e4..b762c7d4 100755 --- a/pihole +++ b/pihole @@ -196,7 +196,7 @@ Time: echo -e " ${INFO} Blocking already disabled, nothing to do" exit 0 fi - if [[ $# > 1 ]]; then + if [[ $# -gt 1 ]]; then local error=false if [[ "${2}" == *"s" ]]; then tt=${2%"s"} From 088b2c29205c8da2145de664cf2b0978e6fcba79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 1 Jun 2023 15:10:05 +0200 Subject: [PATCH 042/413] Do not remove -all|excat when not sourrounded by space in query.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 1d3b0a29..bfa21247 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -50,7 +50,7 @@ fi # Strip valid options, leaving only the domain and invalid options # This allows users to place the options before or after the domain -options=$(sed -E 's/ ?-(all|exact) ?//g' <<< "${options}") +options=$(sed -E 's/ +-(all|exact) ?//g' <<< "${options}") # Handle remaining options # If $options contain non ASCII characters, convert to punycode From 8bf4ab0cd6ae133661e980a7cd067d23f2bc6b22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 7 Jan 2023 21:41:12 +0100 Subject: [PATCH 043/413] Remove option to set static IP address if DHCPCD is installed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 127 ----------------------------- 1 file changed, 127 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index c4ea4764..d49ae012 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -500,9 +500,6 @@ find_IPv4_information() { # disabled as we intentionally want to split on whitespace and have printf populate # the variable with just the first field. printf -v IPv4bare "$(printf ${route#*src })" - # Get the default gateway IPv4 address (the way to reach the Internet) - # shellcheck disable=SC2059,SC2086 - printf -v IPv4gw "$(printf ${route#*via })" if ! valid_ip "${IPv4bare}" ; then IPv4bare="127.0.0.1" @@ -663,135 +660,11 @@ find_IPv6_information() { # A function to collect IPv4 and IPv6 information of the device collect_v4andv6_information() { find_IPv4_information - # Echo the information to the user printf " %b IPv4 address: %s\\n" "${INFO}" "${IPV4_ADDRESS}" - # if `dhcpcd` is used offer to set this as static IP for the device - if [[ -f "/etc/dhcpcd.conf" ]]; then - # configure networking via dhcpcd - getStaticIPv4Settings - fi find_IPv6_information printf " %b IPv6 address: %s\\n" "${INFO}" "${IPV6_ADDRESS}" } -getStaticIPv4Settings() { - # Local, named variables - local ipSettingsCorrect - local DHCPChoice - # Ask if the user wants to use DHCP settings as their static IP - # This is useful for users that are using DHCP reservations; we can use the information gathered - DHCPChoice=$(dialog --no-shadow --keep-tite --output-fd 1 \ - --cancel-label "Exit" --ok-label "Continue" \ - --backtitle "Calibrating network interface" \ - --title "Static IP Address" \ - --menu "Do you want to use your current network settings as a static address?\\n \ - IP address: ${IPV4_ADDRESS}\\n \ - Gateway: ${IPv4gw}\\n" \ - "${r}" "${c}" 3 \ - "Yes" "Set static IP using current values" \ - "No" "Set static IP using custom values" \ - "Skip" "I will set a static IP later, or have already done so") - - result=$? - case ${result} in - "${DIALOG_CANCEL}" | "${DIALOG_ESC}") - printf " %b Cancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}" - exit 1 - ;; - esac - - case ${DHCPChoice} in - "Skip") - return - ;; - "Yes") - # If they choose yes, let the user know that the IP address will not be available via DHCP and may cause a conflict. - dialog --no-shadow --keep-tite \ - --cancel-label "Exit" \ - --backtitle "IP information" \ - --title "FYI: IP Conflict" \ - --msgbox "\\nIt is possible your router could still try to assign this IP to a device, which would cause a conflict, \ -but in most cases the router is smart enough to not do that.\n\n\ -If you are worried, either manually set the address, or modify the DHCP reservation pool so it does not include the IP you want.\n\n\ -It is also possible to use a DHCP reservation, but if you are going to do that, you might as well set a static address."\ - "${r}" "${c}" && result=0 || result=$? - - case ${result} in - "${DIALOG_CANCEL}" | "${DIALOG_ESC}") - printf " %b Cancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}" - exit 1 - ;; - esac - ;; - - "No") - # Otherwise, we need to ask the user to input their desired settings. - # Start by getting the IPv4 address (pre-filling it with info gathered from DHCP) - # Start a loop to let the user enter their information with the chance to go back and edit it if necessary - ipSettingsCorrect=false - until [[ "${ipSettingsCorrect}" = True ]]; do - - # Ask for the IPv4 address - _staticIPv4Temp=$(dialog --no-shadow --keep-tite --output-fd 1 \ - --cancel-label "Exit" \ - --ok-label "Continue" \ - --backtitle "Calibrating network interface" \ - --title "IPv4 Address" \ - --form "\\nEnter your desired IPv4 address" \ - "${r}" "${c}" 0 \ - "IPv4 Address:" 1 1 "${IPV4_ADDRESS}" 1 15 19 0 \ - "IPv4 Gateway:" 2 1 "${IPv4gw}" 2 15 19 0) - - result=$? - case ${result} in - "${DIALOG_CANCEL}" | "${DIALOG_ESC}") - printf " %b Cancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}" - exit 1 - ;; - esac - - IPV4_ADDRESS=${_staticIPv4Temp%$'\n'*} - IPv4gw=${_staticIPv4Temp#*$'\n'} - - # Give the user a chance to review their settings before moving on - dialog --no-shadow --keep-tite \ - --no-label "Edit IP" \ - --backtitle "Calibrating network interface" \ - --title "Static IP Address" \ - --defaultno \ - --yesno "Are these settings correct? - IP address: ${IPV4_ADDRESS} - Gateway: ${IPv4gw}" \ - "${r}" "${c}" && ipSettingsCorrect=True - done - ;; - esac - setDHCPCD -} - -# Configure networking via dhcpcd -setDHCPCD() { - # Regex for matching a non-commented static ip address setting - local regex="^[ \t]*static ip_address[ \t]*=[ \t]*${IPV4_ADDRESS}" - - # Check if static IP is already set in file - if grep -q "${regex}" /etc/dhcpcd.conf; then - printf " %b Static IP already configured\\n" "${INFO}" - # If it's not, - else - # we can append these lines to dhcpcd.conf to enable a static IP - echo "interface ${PIHOLE_INTERFACE} - static ip_address=${IPV4_ADDRESS} - static routers=${IPv4gw} - static domain_name_servers=${PIHOLE_DNS_1} ${PIHOLE_DNS_2}" | tee -a /etc/dhcpcd.conf >/dev/null - # Then use the ip command to immediately set the new address - ip addr replace dev "${PIHOLE_INTERFACE}" "${IPV4_ADDRESS}" - # Also give a warning that the user may need to reboot their system - printf " %b Set IP address to %s\\n" "${TICK}" "${IPV4_ADDRESS%/*}" - printf " %b You may need to restart after the install is complete\\n" "${INFO}" - fi -} - # Check an IP address to see if it is a valid one valid_ip() { # Local, named variables From 9c27e4766dede7a69b65cc412125ac8186d84852 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 8 Jun 2023 14:10:04 +0200 Subject: [PATCH 044/413] Decide which FTL architecture to pick by by installed architectures rather than by detected processor (this may be wrong in an emulated Docker environment) Signed-off-by: DL6ER --- automated install/basic-install.sh | 76 +++++++---- test/test_any_automated_install.py | 211 ++++++++++++++++++++--------- 2 files changed, 197 insertions(+), 90 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index d49ae012..b14546d1 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1806,10 +1806,19 @@ FTLinstall() { } get_binary_name() { - # This gives the machine architecture which may be different from the OS architecture... + # Get the OS architecture (we cannot use uname -m as this may return an incorrect architecture when buildx-compiling with QEMU for arm) local machine machine=$(uname -m) + # Get local GLIBC version (leave at "0.0" if no GLIBC, e.g., on musl) + local l_glibc_version="0.0" + if ldd --version 2>&1 | grep -q "GLIBC"; then + l_glibc_version=$(ldd --version | head -n1 | grep -o '[0-9.]*$') + printf "%b %b Detected GLIBC version %s\\n" "${OVER}" "${TICK}" "${l_glibc_version}" + else + printf "%b %b No GLIBC detected\\n" "${OVER}" "${CROSS}" + fi + local l_binary local str="Detecting processor" @@ -1817,36 +1826,51 @@ get_binary_name() { # If the machine is arm or aarch if [[ "${machine}" == "arm"* || "${machine}" == *"aarch"* ]]; then # ARM + # Get supported processor from other binaries installed on the system + local cpu_arch + cpu_arch=$(readelf -A "$(command -v sh)" | grep Tag_CPU_arch | awk '{ print $2 }') + + # Get the revision from the CPU architecture local rev - rev=$(uname -m | sed "s/[^0-9]//g;") - local lib - lib=$(ldd "$(command -v sh)" | grep -E '^\s*/lib' | awk '{ print $1 }') - if [[ "${lib}" == "/lib/ld-linux-aarch64.so.1" ]]; then - printf "%b %b Detected AArch64 (64 Bit ARM) processor\\n" "${OVER}" "${TICK}" + rev=$(echo "${cpu_arch}" | grep -o '[0-9]*') + if [[ "${machine}" == "aarch64" ]]; then + printf "%b %b Detected AArch64 (64 Bit ARM) architecture\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-arm64" - elif [[ "${lib}" == "/lib/ld-linux-armhf.so.3" ]]; then - # Hard-float available: Use gnueabihf binaries + elif [[ "${cpu_arch}" == "armv6KZ" ]]; then + printf "%b %b Detected ARMv6KZ architecture\\n" "${OVER}" "${TICK}" + # set the binary to be used + l_binary="pihole-FTL-armv6" + else # If ARMv8 or higher is found (e.g., BCM2837 as found in Raspberry Pi Model 3B) if [[ "${rev}" -gt 7 ]]; then - printf "%b %b Detected ARMv8 (or newer) processor\\n" "${OVER}" "${TICK}" - # set the binary to be used - l_binary="pihole-FTL-armv8" - elif [[ "${rev}" -eq 7 ]]; then - # Otherwise, if ARMv7 is found (e.g., BCM2836 as found in Raspberry Pi Model 2) - printf "%b %b Detected ARMv7 processor (with hard-float support)\\n" "${OVER}" "${TICK}" + printf "%b %b Detected ARMv8 (or newer) architecture\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-armv7" - else - # Otherwise, use the ARMv6 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv6 processor (with hard-float support)\\n" "${OVER}" "${TICK}" + elif [[ "${rev}" -gt 6 ]]; then + # Otherwise, if ARMv7 is found (e.g., BCM2836 as found in Raspberry Pi Model 2) + printf "%b %b Detected ARMv7 architecture\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-armv6" + elif [[ "${rev}" -gt 5 ]]; then + # Check if the system is using GLIBC 2.29 or higher + if [[ -n "${l_glibc_version}" && "$(printf '%s\n' "2.29" "${l_glibc_version}" | sort -V | head -n1)" == "2.29" ]]; then + # If so, use the ARMv6 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) + printf "%b %b Detected ARMv6 architecture (running GLIBC 2.29 or higher)\\n" "${OVER}" "${TICK}" + # set the binary to be used + l_binary="pihole-FTL-armv5" + else + # Otherwise, use the ARMv5 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) + printf "%b %b Detected ARMv6 architecture (running GLIBC older than 2.29)\\n" "${OVER}" "${TICK}" + # set the binary to be used + l_binary="pihole-FTL-armv4" + fi + else + # Otherwise, use the ARMv4 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) + printf "%b %b Detected ARMv4 or ARMv5 architecture\\n" "${OVER}" "${TICK}" + # set the binary to be used + l_binary="pihole-FTL-armv4" fi - else - # No hard-float support found - printf "%b %b%b ARM processor without hard-float support detected%b\\n" "${OVER}" "${COL_LIGHT_RED}" "${CROSS}" "${COL_NC}" - l_binary="" fi elif [[ "${machine}" == "x86_64" ]]; then # This gives the processor of packages dpkg installs (for example, "i386") @@ -1858,25 +1882,25 @@ get_binary_name() { # We only check this for Debian-based systems as this has been an issue # in the past (see https://github.com/pi-hole/pi-hole/pull/2004) if [[ "${dpkgarch}" == "i386" ]]; then - printf "%b %b Detected 32bit (i686) processor\\n" "${OVER}" "${TICK}" + printf "%b %b Detected 32bit (i686) architecture\\n" "${OVER}" "${TICK}" l_binary="pihole-FTL-386" else # 64bit - printf "%b %b Detected x86_64 processor\\n" "${OVER}" "${TICK}" + printf "%b %b Detected x86_64 architecture\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-amd64" fi elif [[ "${machine}" == "riscv64" ]]; then - printf "%b %b Detected riscv64 processor\\n" "${OVER}" "${TICK}" + printf "%b %b Detected riscv64 architecture\\n" "${OVER}" "${TICK}" l_binary="pihole-FTL-riscv64" else # Something else - we try to use 32bit executable and warn the user if [[ ! "${machine}" == "i686" ]]; then printf "%b %b %s...\\n" "${OVER}" "${CROSS}" "${str}" - printf " %b %bNot able to detect processor (unknown: %s), trying x86 (32bit) executable%b\\n" "${INFO}" "${COL_LIGHT_RED}" "${machine}" "${COL_NC}" + printf " %b %bNot able to detect architecture (unknown: %s), trying x86 (32bit) executable%b\\n" "${INFO}" "${COL_LIGHT_RED}" "${machine}" "${COL_NC}" printf " %b Contact Pi-hole Support if you experience issues (e.g: FTL not running)\\n" "${INFO}" else - printf "%b %b Detected 32bit (i686) processor\\n" "${OVER}" "${TICK}" + printf "%b %b Detected 32bit (i686) architecture\\n" "${OVER}" "${TICK}" fi l_binary="pihole-FTL-linux-386" fi diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 5c902c2b..4431c565 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -241,12 +241,36 @@ def test_FTL_detect_aarch64_no_errors(host): """ # mock uname to return aarch64 platform mock_command("uname", {"-m": ("aarch64", "0")}, host) - # mock ldd to respond with aarch64 shared library - mock_command( - "ldd", + detectPlatform = host.run( + """ + source /opt/pihole/basic-install.sh + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" + """ + ) + expected_stdout = info_box + " FTL Checks..." + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Detected AArch64 (64 Bit ARM) architecture" + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Downloading and Installing FTL" + assert expected_stdout in detectPlatform.stdout + + +def test_FTL_detect_armv4_no_errors(host): + """ + confirms only armv4 package is downloaded for FTL engine + """ + # mock uname to return armv4 platform + mock_command("uname", {"-m": ("armv4t", "0")}, host) + # mock readelf to respond with armv4 CPU architecture + mock_command_2( + "readelf", { - "/bin/sh": ("/lib/ld-linux-aarch64.so.1", "0"), - "/usr/bin/sh": ("/lib/ld-linux-aarch64.so.1", "0"), + "-A /bin/sh": ("Tag_CPU_arch: armv4t", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv4t", "0"), }, host, ) @@ -262,77 +286,65 @@ def test_FTL_detect_aarch64_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected AArch64 (64 Bit ARM) processor" + expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv4t_no_install(host): +def test_FTL_detect_armv5_no_errors(host): """ - confirms armv4t architecture is not supported - """ - # mock uname to return armv4t platform - mock_command("uname", {"-m": ("armv4t", "0")}, host) - # mock ldd to respond with armv4t shared library - mock_command( - "ldd", - { - "/bin/sh": ("/lib/ld-linux.so.3", "0"), - "/usr/bin/sh": ("/lib/ld-linux.so.3", "0"), - }, - host, - ) - detectPlatform = host.run( - """ - source /opt/pihole/basic-install.sh - get_binary_name - """ - ) - expected_stdout = cross_box + (" ARM processor without hard-float support detected") - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv5te_no_install(host): - """ - confirms armv5te architecture is not supported + confirms only armv5 package is downloaded for FTL engine """ # mock uname to return armv5te platform mock_command("uname", {"-m": ("armv5te", "0")}, host) - # mock ldd to respond with ld-linux shared library - mock_command( - "ldd", + # mock readelf to respond with armv5 CPU architecture + mock_command_2( + "readelf", { - "/bin/sh": ("/lib/ld-linux.so.3", "0"), - "/usr/bin/sh": ("/lib/ld-linux.so.3", "0"), + "-A /bin/sh": ("Tag_CPU_arch: armv5te", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv5te", "0"), }, host, ) detectPlatform = host.run( """ source /opt/pihole/basic-install.sh - get_binary_name + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" """ ) - expected_stdout = cross_box + (" ARM processor without hard-float support detected") + expected_stdout = info_box + " FTL Checks..." + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture" + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv6l_no_errors(host): +def test_FTL_detect_armv6_old_no_errors(host): """ - confirms only armv6l package is downloaded for FTL engine + confirms only armv6 package is downloaded for FTL engine """ # mock uname to return armv6l platform mock_command("uname", {"-m": ("armv6l", "0")}, host) - # mock ldd to respond with ld-linux-armhf shared library - mock_command( - "ldd", + # mock readelf to respond with armv6l CPU architecture + mock_command_2( + "readelf", { - "/bin/sh": ("/lib/ld-linux-armhf.so.3", "0"), - "/usr/bin/sh": ("/lib/ld-linux-armhf.so.3", "0"), + "-A /bin/sh": ("Tag_CPU_arch: armv6l", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv6l", "0"), }, host, ) + # Mock old ldd GLIBC version + mock_command( + "ldd", {"--version": ("ldd (Debian GLIBC 2.13-38+deb7u8) 2.13", "0")}, host + ) + detectPlatform = host.run( """ source /opt/pihole/basic-install.sh @@ -346,25 +358,98 @@ def test_FTL_detect_armv6l_no_errors(host): expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + ( - " Detected ARMv6 processor " "(with hard-float support)" + " Detected ARMv6 architecture (running GLIBC older than 2.29)" ) assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout +def test_FTL_detect_armv6_recent_no_errors(host): + """ + confirms only armv6 package is downloaded for FTL engine + """ + # mock uname to return armv6l platform + mock_command("uname", {"-m": ("armv6l", "0")}, host) + # mock readelf to respond with armv6l CPU architecture + mock_command_2( + "readelf", + { + "-A /bin/sh": ("Tag_CPU_arch: armv6l", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv6l", "0"), + }, + host, + ) + # Mock old ldd GLIBC version + mock_command( + "ldd", {"--version": ("'ldd (Debian GLIBC 2.35-38+deb7u8) 2.35'", "0")}, host + ) + + detectPlatform = host.run( + """ + source /opt/pihole/basic-install.sh + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" + """ + ) + expected_stdout = info_box + " FTL Checks..." + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + ( + " Detected ARMv6 architecture (running GLIBC 2.29 or higher)" + ) + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Downloading and Installing FTL" + assert expected_stdout in detectPlatform.stdout + + +def test_FTL_detect_armv6KZ_no_errors(host): + """ + confirms only armv6KZ package is downloaded for FTL engine + """ + # mock uname to return armv6KZ platform + mock_command("uname", {"-m": ("armv6KZ", "0")}, host) + # mock readelf to respond with armv6l CPU architecture + mock_command_2( + "readelf", + { + "-A /bin/sh": ("Tag_CPU_arch: armv6KZ", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv6KZ", "0"), + }, + host, + ) + detectPlatform = host.run( + """ + source /opt/pihole/basic-install.sh + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" + """ + ) + expected_stdout = info_box + " FTL Checks..." + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Detected ARMv6KZ architecture" + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Downloading and Installing FTL" + assert expected_stdout in detectPlatform.stdout + + def test_FTL_detect_armv7l_no_errors(host): """ confirms only armv7l package is downloaded for FTL engine """ # mock uname to return armv7l platform mock_command("uname", {"-m": ("armv7l", "0")}, host) - # mock ldd to respond with ld-linux-armhf shared lib rary - mock_command( - "ldd", + # mock readelf to respond with armv7l CPU architecture + mock_command_2( + "readelf", { - "/bin/sh": ("/lib/ld-linux-armhf.so.3", "0"), - "/usr/bin/sh": ("/lib/ld-linux-armhf.so.3", "0"), + "-A /bin/sh": ("Tag_CPU_arch: armv7l", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv7l", "0"), }, host, ) @@ -380,9 +465,7 @@ def test_FTL_detect_armv7l_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + ( - " Detected ARMv7 processor " "(with hard-float support)" - ) + expected_stdout = tick_box + (" Detected ARMv7 architecture") assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -394,12 +477,12 @@ def test_FTL_detect_armv8a_no_errors(host): """ # mock uname to return armv8a platform mock_command("uname", {"-m": ("armv8a", "0")}, host) - # mock ldd to respond with ld-linux-armhf shared library - mock_command( - "ldd", + # mock readelf to respond with armv8a CPU architecture + mock_command_2( + "readelf", { - "/bin/sh": ("/lib/ld-linux-armhf.so.3", "0"), - "/usr/bin/sh": ("/lib/ld-linux-armhf.so.3", "0"), + "-A /bin/sh": ("Tag_CPU_arch: armv8a", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv8a", "0"), }, host, ) @@ -415,7 +498,7 @@ def test_FTL_detect_armv8a_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv8 (or newer) processor" + expected_stdout = tick_box + " Detected ARMv8 (or newer) architecture" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -437,7 +520,7 @@ def test_FTL_detect_x86_64_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected x86_64 processor" + expected_stdout = tick_box + " Detected x86_64 architecture" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -457,7 +540,7 @@ def test_FTL_detect_unknown_no_errors(host): FTLdetect "${binary}" "${theRest}" """ ) - expected_stdout = "Not able to detect processor (unknown: mips)" + expected_stdout = "Not able to detect architecture (unknown: mips)" assert expected_stdout in detectPlatform.stdout From 1c4e58efe3487264291761ec48a0dd0c7d61417e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 9 Jun 2023 15:53:02 +0200 Subject: [PATCH 045/413] Use exact architecture matching binaries where appropriate Signed-off-by: DL6ER --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index b14546d1..ed2d0037 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1843,7 +1843,7 @@ get_binary_name() { l_binary="pihole-FTL-armv6" else # If ARMv8 or higher is found (e.g., BCM2837 as found in Raspberry Pi Model 3B) - if [[ "${rev}" -gt 7 ]]; then + if [[ "${cpu_arch}" == "v7" || "${rev}" -gt 7 ]]; then printf "%b %b Detected ARMv8 (or newer) architecture\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-armv7" @@ -1852,7 +1852,7 @@ get_binary_name() { printf "%b %b Detected ARMv7 architecture\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-armv6" - elif [[ "${rev}" -gt 5 ]]; then + elif [[ "${cpu_arch}" == "v5TE" || "${rev}" -gt 5 ]]; then # Check if the system is using GLIBC 2.29 or higher if [[ -n "${l_glibc_version}" && "$(printf '%s\n' "2.29" "${l_glibc_version}" | sort -V | head -n1)" == "2.29" ]]; then # If so, use the ARMv6 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) From 913be6c3494db826ee0914034bb42d479c4ec8c9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 9 Jun 2023 18:50:31 +0200 Subject: [PATCH 046/413] Print detected CPU arch Signed-off-by: DL6ER --- automated install/basic-install.sh | 10 +++---- test/test_any_automated_install.py | 45 ++++++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ed2d0037..5a6be794 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1844,30 +1844,30 @@ get_binary_name() { else # If ARMv8 or higher is found (e.g., BCM2837 as found in Raspberry Pi Model 3B) if [[ "${cpu_arch}" == "v7" || "${rev}" -gt 7 ]]; then - printf "%b %b Detected ARMv8 (or newer) architecture\\n" "${OVER}" "${TICK}" + printf "%b %b Detected ARMv7 (or newer) architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" # set the binary to be used l_binary="pihole-FTL-armv7" elif [[ "${rev}" -gt 6 ]]; then # Otherwise, if ARMv7 is found (e.g., BCM2836 as found in Raspberry Pi Model 2) - printf "%b %b Detected ARMv7 architecture\\n" "${OVER}" "${TICK}" + printf "%b %b Detected ARMv7 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" # set the binary to be used l_binary="pihole-FTL-armv6" elif [[ "${cpu_arch}" == "v5TE" || "${rev}" -gt 5 ]]; then # Check if the system is using GLIBC 2.29 or higher if [[ -n "${l_glibc_version}" && "$(printf '%s\n' "2.29" "${l_glibc_version}" | sort -V | head -n1)" == "2.29" ]]; then # If so, use the ARMv6 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv6 architecture (running GLIBC 2.29 or higher)\\n" "${OVER}" "${TICK}" + printf "%b %b Detected ARMv6 architecture (running GLIBC 2.29 or higher, %s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" # set the binary to be used l_binary="pihole-FTL-armv5" else # Otherwise, use the ARMv5 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv6 architecture (running GLIBC older than 2.29)\\n" "${OVER}" "${TICK}" + printf "%b %b Detected ARMv6 architecture (running GLIBC older than 2.29, %s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" # set the binary to be used l_binary="pihole-FTL-armv4" fi else # Otherwise, use the ARMv4 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv4 or ARMv5 architecture\\n" "${OVER}" "${TICK}" + printf "%b %b Detected ARMv4 or ARMv5 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" # set the binary to be used l_binary="pihole-FTL-armv4" fi diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 4431c565..bc594304 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -286,7 +286,7 @@ def test_FTL_detect_armv4_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture" + expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture (armv4t)" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -319,7 +319,7 @@ def test_FTL_detect_armv5_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture" + expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture (armv5te)" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -358,7 +358,7 @@ def test_FTL_detect_armv6_old_no_errors(host): expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + ( - " Detected ARMv6 architecture (running GLIBC older than 2.29)" + " Detected ARMv6 architecture (running GLIBC older than 2.29, armv6l)" ) assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" @@ -398,7 +398,7 @@ def test_FTL_detect_armv6_recent_no_errors(host): expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + ( - " Detected ARMv6 architecture (running GLIBC 2.29 or higher)" + " Detected ARMv6 architecture (running GLIBC 2.29 or higher, armv6l)" ) assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" @@ -465,7 +465,40 @@ def test_FTL_detect_armv7l_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv7 architecture") + expected_stdout = tick_box + (" Detected ARMv7 architecture (armv7l)") + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Downloading and Installing FTL" + assert expected_stdout in detectPlatform.stdout + + +def test_FTL_detect_armv7_no_errors(host): + """ + confirms only armv7 package is downloaded for FTL engine + """ + # mock uname to return armv7 platform + mock_command("uname", {"-m": ("armv7", "0")}, host) + # mock readelf to respond with armv7 CPU architecture + mock_command_2( + "readelf", + { + "-A /bin/sh": ("Tag_CPU_arch: armv7", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv7", "0"), + }, + host, + ) + detectPlatform = host.run( + """ + source /opt/pihole/basic-install.sh + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" + """ + ) + expected_stdout = info_box + " FTL Checks..." + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + (" Detected ARMv7 architecture (armv7)") assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -498,7 +531,7 @@ def test_FTL_detect_armv8a_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv8 (or newer) architecture" + expected_stdout = tick_box + " Detected ARMv7 (or newer) architecture (armv8a)" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout From e0d7e5df8519883cf08b45500809ef1d04ee4160 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 9 Jun 2023 19:30:28 +0200 Subject: [PATCH 047/413] Install binutils as installer deps (needed to pick correct architecture for FTL) Signed-off-by: DL6ER --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 5a6be794..699fef10 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -304,7 +304,7 @@ package_manager_detect() { # Packages required to perform the os_check (stored as an array) OS_CHECK_DEPS=(grep dnsutils) # Packages required to run this install script (stored as an array) - INSTALLER_DEPS=(git iproute2 dialog ca-certificates) + INSTALLER_DEPS=(git iproute2 dialog ca-certificates binutils) # Packages required to run Pi-hole (stored as an array) PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) @@ -322,7 +322,7 @@ package_manager_detect() { # CentOS package manager returns 100 when there are packages to update so we need to || true to prevent the script from exiting. PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src|.riscv64)' | wc -l || true" OS_CHECK_DEPS=(grep bind-utils) - INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates) + INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates binutils) PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc libcap nmap-ncat jq) # If neither apt-get or yum/dnf package managers were found From 5459daa03daaaf89e18ac3da64082193c9c2b0c4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 15 Jun 2023 09:28:03 +0200 Subject: [PATCH 048/413] Add abp_entries column (default 0) to the adlist table for newly created gravity databases. This updates the gravity database version to 16 Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity-db.sh | 6 ++++++ .../Scripts/database_migration/gravity/15_to_16.sql | 11 +++++++++++ advanced/Templates/gravity.db.sql | 3 ++- 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 advanced/Scripts/database_migration/gravity/15_to_16.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index a7ba60a9..851ddb0a 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -128,4 +128,10 @@ upgrade_gravityDB(){ pihole-FTL sqlite3 "${database}" < "${scriptPath}/14_to_15.sql" version=15 fi + if [[ "$version" == "15" ]]; then + # Add column abp_entries to adlist table + echo -e " ${INFO} Upgrading gravity database from version 15 to 16" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/15_to_16.sql" + version=16 + fi } diff --git a/advanced/Scripts/database_migration/gravity/15_to_16.sql b/advanced/Scripts/database_migration/gravity/15_to_16.sql new file mode 100644 index 00000000..c6159f40 --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/15_to_16.sql @@ -0,0 +1,11 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +ALTER TABLE adlist ADD COLUMN abp_entries INTEGER NOT NULL DEFAULT 0; + +UPDATE info SET value = 16 WHERE property = 'version'; + +COMMIT; diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 3f696d6d..e0f1de10 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -35,7 +35,8 @@ CREATE TABLE adlist date_updated INTEGER, number INTEGER NOT NULL DEFAULT 0, invalid_domains INTEGER NOT NULL DEFAULT 0, - status INTEGER NOT NULL DEFAULT 0 + status INTEGER NOT NULL DEFAULT 0, + abp_entries INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE adlist_by_group From d5013bfd6c3153113fa8a67b6731c9d91a327e67 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 12 Jun 2023 22:33:50 +0100 Subject: [PATCH 049/413] Add code to remove old lighttpd config files left over from v5. Web config is all dealt with by FTL now Signed-off-by: Adam Warner --- automated install/basic-install.sh | 33 ++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 699fef10..9659a140 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1001,6 +1001,38 @@ remove_old_dnsmasq_ftl_configs() { fi } +remove_old_pihole_lighttpd_configs() { + local lighttpdConfig="/etc/lighttpd/lighttpd.conf" + local condfd="/etc/lighttpd/conf.d/pihole-admin.conf" + local confavailable="/etc/lighttpd/conf-available/15-pihole-admin.conf" + local confenabled="/etc/lighttpd/conf-enabled/15-pihole-admin.conf" + + + if [[ -d "/etc/lighttpd/conf.d" ]]; then + if grep -q -F 'include "/etc/lighttpd/conf.d/pihole-admin.conf"' "${lighttpdConfig}"; then + sed -i '/include "/etc/lighttpd/conf.d/pihole-admin.conf"/d' "${lighttpdConfig}" + fi + + if [[ -f "${condfd}" ]]; then + rm "${condfd}" + fi + + + elif [[ -d "/etc/lighttpd/conf-available" ]]; then + if is_command lighty-disable-mod ; then + lighty-disable-mod pihole-admin > /dev/null || true + fi + + if [[ -f "${confavailable}" ]]; then + rm "${confavailable}" + fi + + if [[ -f "${confenabled}" ]]; then + rm "${confenabled}" + fi + fi +} + # Clean an existing installation to prepare for upgrade/reinstall clean_existing() { # Local, named variables @@ -1486,6 +1518,7 @@ installPihole() { fi remove_old_dnsmasq_ftl_configs + remove_old_pihole_lighttpd_configs # Install config files if ! installConfigs; then From c39cb8cfe066d3520354f3cdee293fa6afe47e0e Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 12 Jun 2023 22:43:39 +0100 Subject: [PATCH 050/413] Escape the sed command for removing a line in fed/centos lighttpd.conf Signed-off-by: Adam Warner --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9659a140..43cd9816 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1010,7 +1010,7 @@ remove_old_pihole_lighttpd_configs() { if [[ -d "/etc/lighttpd/conf.d" ]]; then if grep -q -F 'include "/etc/lighttpd/conf.d/pihole-admin.conf"' "${lighttpdConfig}"; then - sed -i '/include "/etc/lighttpd/conf.d/pihole-admin.conf"/d' "${lighttpdConfig}" + sed -i '/include "\/etc\/lighttpd\/conf.d\/pihole-admin.conf"/d' "${lighttpdConfig}" fi if [[ -f "${condfd}" ]]; then From 001f2012a2b40c200e4c7d6384b8cd2f62c523c5 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 13 Jun 2023 19:08:12 +0100 Subject: [PATCH 051/413] Update automated install/basic-install.sh Co-authored-by: RD WebDesign Signed-off-by: Adam Warner --- automated install/basic-install.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 43cd9816..2edad6e3 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1009,9 +1009,7 @@ remove_old_pihole_lighttpd_configs() { if [[ -d "/etc/lighttpd/conf.d" ]]; then - if grep -q -F 'include "/etc/lighttpd/conf.d/pihole-admin.conf"' "${lighttpdConfig}"; then - sed -i '/include "\/etc\/lighttpd\/conf.d\/pihole-admin.conf"/d' "${lighttpdConfig}" - fi + sed -i '/include "\/etc\/lighttpd\/conf.d\/pihole-admin.conf"/d' "${lighttpdConfig}" if [[ -f "${condfd}" ]]; then rm "${condfd}" From d637d2a7a5f76fdea076d01d091e9e4865ac55f4 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 18 Jun 2023 12:38:02 +0100 Subject: [PATCH 052/413] Simplify nested if statements. Co-authored-by: yubiuser Signed-off-by: Adam Warner --- automated install/basic-install.sh | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 2edad6e3..e3d8ff29 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1008,27 +1008,25 @@ remove_old_pihole_lighttpd_configs() { local confenabled="/etc/lighttpd/conf-enabled/15-pihole-admin.conf" - if [[ -d "/etc/lighttpd/conf.d" ]]; then + if [[ -f "${lighttpdConfig}" ]]; then sed -i '/include "\/etc\/lighttpd\/conf.d\/pihole-admin.conf"/d' "${lighttpdConfig}" + fi - if [[ -f "${condfd}" ]]; then - rm "${condfd}" - fi + if [[ -f "${condfd}" ]]; then + rm "${condfd}" + fi + if is_command lighty-disable-mod ; then + lighty-disable-mod pihole-admin > /dev/null || true + fi - elif [[ -d "/etc/lighttpd/conf-available" ]]; then - if is_command lighty-disable-mod ; then - lighty-disable-mod pihole-admin > /dev/null || true - fi + if [[ -f "${confavailable}" ]]; then + rm "${confavailable}" + fi - if [[ -f "${confavailable}" ]]; then - rm "${confavailable}" - fi - - if [[ -f "${confenabled}" ]]; then - rm "${confenabled}" - fi - fi + if [[ -f "${confenabled}" ]]; then + rm "${confenabled}" + fi } # Clean an existing installation to prepare for upgrade/reinstall From a3bb3872bfbf9520bd29b2069a889d7d6d4f1202 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 18 Jun 2023 16:30:59 +0100 Subject: [PATCH 053/413] adlist table now contains 11 columns, not 10. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Christian König Signed-off-by: Adam Warner --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index abf6700b..ed402a34 100755 --- a/gravity.sh +++ b/gravity.sh @@ -178,7 +178,7 @@ database_table_from_file() { echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" elif [[ "${table}" == "adlist" ]]; then # Adlist table format - echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0" >> "${tmpFile}" + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0,0" >> "${tmpFile}" else # White-, black-, and regexlist table format echo "${rowid},${list_type},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\"" >> "${tmpFile}" From ca66c1ea9cd905b06618dbded58ff49b9b4f45a1 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 18 Jun 2023 18:01:54 +0100 Subject: [PATCH 054/413] Switch to new branch name for FTL v6 development Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 2 +- test/test_any_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index bc594304..a8e0e082 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -82,7 +82,7 @@ def test_installPihole_fresh_install_readableFiles(host): # Workaround to get FTLv6 installed until it reaches master branch host.run( """ - echo "new/http" > /etc/pihole/ftlbranch + echo "development-v6" > /etc/pihole/ftlbranch """ ) install = host.run( diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 0f300457..9eee6885 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -137,7 +137,7 @@ def test_getFTLConfigValue_getFTLConfigValue(host): source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) - echo "new/http" > /etc/pihole/ftlbranch + echo "development-v6" > /etc/pihole/ftlbranch binary="pihole-FTL${funcOutput##*pihole-FTL}" theRest="${funcOutput%pihole-FTL*}" FTLdetect "${binary}" "${theRest}" From c557f29db2d8d222fdd9d38c5b53fcfac110aeb8 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 3 Jul 2023 23:01:27 +0100 Subject: [PATCH 055/413] Set new gravity database version to 16, which was missed in 5459daa Signed-off-by: Adam Warner --- advanced/Templates/gravity.db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index e0f1de10..881cfcc3 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -58,7 +58,7 @@ CREATE TABLE info value TEXT NOT NULL ); -INSERT INTO "info" VALUES('version','15'); +INSERT INTO "info" VALUES('version','16'); CREATE TABLE domain_audit ( From 64319fa96efcd1b8172afb85110ae0742d49185c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 5 Jul 2023 22:11:56 +0100 Subject: [PATCH 056/413] Add /var/log/pihole/webserver.log to the logrotate scripts. While it may be empty for most, it can grow quickly if either API or TLS debugging is enabled (it can quickly reach several hundreds of MB per day when debugging TLS) Signed-off-by: DL6ER --- advanced/Templates/logrotate | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/advanced/Templates/logrotate b/advanced/Templates/logrotate index 9a56b552..5f609e0f 100644 --- a/advanced/Templates/logrotate +++ b/advanced/Templates/logrotate @@ -19,3 +19,14 @@ notifempty nomail } + +/var/log/pihole/webserver.log { + # su # + weekly + copytruncate + rotate 3 + compress + delaycompress + notifempty + nomail +} From e295997d51d50f661e05eee06b290ad57a56f502 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Jul 2023 21:36:02 +0000 Subject: [PATCH 057/413] Bump actions/setup-python from 4.6.1 to 4.7.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.6.1 to 4.7.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.6.1...v4.7.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index be64482d..09462eb4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,7 +75,7 @@ jobs: uses: actions/checkout@v3.5.3 - name: Set up Python 3.10 - uses: actions/setup-python@v4.6.1 + uses: actions/setup-python@v4.7.0 with: python-version: "3.10" From a3955a7ebb33cdf1ec3ab537eb2a507705648c55 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Jul 2023 21:36:15 +0000 Subject: [PATCH 058/413] Bump pytest from 7.3.2 to 7.4.0 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.3.2 to 7.4.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.3.2...7.4.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 44e6c94a..4eb9113e 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ docker-compose == 1.29.2 -pytest == 7.3.2 +pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 tox == 4.6.2 From 8dd8f989abce529ed3176eeecc7174c8ed3a9053 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Jul 2023 21:44:31 +0000 Subject: [PATCH 059/413] Bump tox from 4.6.2 to 4.6.4 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.6.2 to 4.6.4. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.6.2...4.6.4) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 4eb9113e..a4f627fa 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ docker-compose == 1.29.2 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 -tox == 4.6.2 +tox == 4.6.4 From e42c692ec7b364d443c947187f7ab5943f6f6173 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 18 Jul 2023 08:04:33 +0100 Subject: [PATCH 060/413] Drop docker-compose from requirements.txt, it's not used. This was the only thing preventing us from using pyyaml 6.0.1 which fixes issues we were seeing with tests not running (https://github.com/yaml/pyyaml/pull/702) Signed-off-by: Adam Warner --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index a4f627fa..5a551fa7 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,4 +1,4 @@ -docker-compose == 1.29.2 +pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 From 6edd6a4d55af1ca29c799c96ee355f0747df1ea7 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 17 Jul 2023 21:44:13 +0100 Subject: [PATCH 061/413] if ${USER} variable is blank, then populate it with whoami Signed-off-by: Adam Warner --- pihole | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pihole b/pihole index b762c7d4..66771b9b 100755 --- a/pihole +++ b/pihole @@ -544,6 +544,13 @@ if [[ ! $EUID -eq 0 && need_root -eq 1 ]];then exit 1 fi fi + +# In the case of alpine running in a container, the USER variable appears to be blank +# which prevents the next trap from working correctly. Set it by running whoami +if [[ -z ${USER} ]]; then + USER=$(whoami) +fi + # Can also be user pihole for other functions if [[ ${USER} != "pihole" && need_root -eq 0 ]];then if [[ -x "$(command -v sudo)" ]]; then From 00cbb8bc8abb119ebc3c3f0d912b94a334e06128 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 5 Jul 2023 22:24:11 +0200 Subject: [PATCH 062/413] Add antigravity support to gravity Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 7 +++++ .../database_migration/gravity/16_to_17.sql | 13 ++++++++++ advanced/Templates/gravity.db.sql | 16 +++++++++++- gravity.sh | 26 +++++++++++++------ 4 files changed, 53 insertions(+), 9 deletions(-) create mode 100644 advanced/Scripts/database_migration/gravity/16_to_17.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 851ddb0a..e36d9b1e 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -134,4 +134,11 @@ upgrade_gravityDB(){ pihole-FTL sqlite3 "${database}" < "${scriptPath}/15_to_16.sql" version=16 fi + if [[ "$version" == "16" ]]; then + # Add antigravity table + # Add column type to adlist table (to support adlist types) + echo -e " ${INFO} Upgrading gravity database from version 16 to 17" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/16_to_17.sql" + version=17 + fi } diff --git a/advanced/Scripts/database_migration/gravity/16_to_17.sql b/advanced/Scripts/database_migration/gravity/16_to_17.sql new file mode 100644 index 00000000..c7b9049b --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/16_to_17.sql @@ -0,0 +1,13 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +ALTER TABLE adlist ADD COLUMN type INTEGER NOT NULL DEFAULT 0; + +UPDATE adlist SET type = 0; + +UPDATE info SET value = 17 WHERE property = 'version'; + +COMMIT; diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 881cfcc3..17712cf7 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -36,7 +36,8 @@ CREATE TABLE adlist number INTEGER NOT NULL DEFAULT 0, invalid_domains INTEGER NOT NULL DEFAULT 0, status INTEGER NOT NULL DEFAULT 0, - abp_entries INTEGER NOT NULL DEFAULT 0 + abp_entries INTEGER NOT NULL DEFAULT 0, + type INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE adlist_by_group @@ -52,6 +53,12 @@ CREATE TABLE gravity adlist_id INTEGER NOT NULL REFERENCES adlist (id) ); +CREATE TABLE antigravity +( + domain TEXT NOT NULL, + adlist_id INTEGER NOT NULL REFERENCES adlist (id) +); + CREATE TABLE info ( property TEXT PRIMARY KEY, @@ -144,6 +151,13 @@ CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id LEFT JOIN "group" ON "group".id = adlist_by_group.group_id WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1); +CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id + FROM antigravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id + LEFT JOIN adlist ON adlist.id = antigravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; + CREATE VIEW vw_adlist AS SELECT DISTINCT address, id FROM adlist WHERE enabled = 1 diff --git a/gravity.sh b/gravity.sh index ed402a34..d784a2e7 100755 --- a/gravity.sh +++ b/gravity.sh @@ -361,6 +361,7 @@ gravity_DownloadBlocklists() { # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true) mapfile -t sources <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)" + mapfile -t sourceTypes <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2> /dev/null)" # Parse source domains from $sources mapfile -t sourceDomains <<< "$( @@ -382,7 +383,7 @@ gravity_DownloadBlocklists() { unset sources fi - local url domain agent str target compression + local url domain agent str target compression adlist_type echo "" # Prepare new gravity database @@ -394,7 +395,7 @@ gravity_DownloadBlocklists() { if [[ "${status}" -ne 0 ]]; then echo -e "\\n ${CROSS} Unable to create new database ${gravityTEMPfile}\\n ${output}" - gravity_Cleanup "error" + #gravity_Cleanup "error" else echo -e "${OVER} ${TICK} ${str}" fi @@ -433,6 +434,15 @@ gravity_DownloadBlocklists() { url="${sources[$i]}" domain="${sourceDomains[$i]}" id="${sourceIDs[$i]}" + if [[ "${sourceTypes[$i]}" -eq "0" ]]; then + # Gravity list + str="blocklist" + adlist_type="gravity" + else + # AntiGravity list + str="allowlist" + adlist_type="antigravity" + fi # Save the file as list.#.domain saveLocation="${piholeDir}/list.${id}.${domain}.${domainsExtension}" @@ -441,7 +451,7 @@ gravity_DownloadBlocklists() { # Default user-agent (for Cloudflare's Browser Integrity Check: https://support.cloudflare.com/hc/en-us/articles/200170086-What-does-the-Browser-Integrity-Check-do-) agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36" - echo -e " ${INFO} Target: ${url}" + echo -e " ${INFO} Target: ${url} (${str})" local regex check_url # Check for characters NOT allowed in URLs regex="[^a-zA-Z0-9:/?&%=~._()-;]" @@ -453,7 +463,7 @@ gravity_DownloadBlocklists() { if [[ "${check_url}" =~ ${regex} ]]; then echo -e " ${CROSS} Invalid Target" else - gravity_DownloadBlocklistFromUrl "${url}" "${agent}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" + gravity_DownloadBlocklistFromUrl "${url}" "${agent}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" "${adlist_type}" fi echo "" done @@ -485,7 +495,7 @@ compareLists() { # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { - local url="${1}" agent="${2}" adlistID="${3}" saveLocation="${4}" target="${5}" compression="${6}" + local url="${1}" agent="${2}" adlistID="${3}" saveLocation="${4}" target="${5}" compression="${6}" gravity_type="${7}" local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext # Create temp file to store content on disk instead of RAM @@ -579,7 +589,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ "${success}" == true ]]; then if [[ "${httpCode}" == "304" ]]; then # Add domains to database table file - pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" database_adlist_status "${adlistID}" "2" done="true" # Check if $listCurlBuffer is a non-zero length file @@ -589,7 +599,7 @@ gravity_DownloadBlocklistFromUrl() { # Remove curl buffer file after its use rm "${listCurlBuffer}" # Add domains to database table file - pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" # Compare lists, are they identical? compareLists "${adlistID}" "${saveLocation}" done="true" @@ -605,7 +615,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ -r "${saveLocation}" ]]; then echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" # Add domains to database table file - pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" database_adlist_status "${adlistID}" "3" else echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" From 35512c4dc9071502b10f251d514d297a79fe7ea5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 6 Jul 2023 19:19:55 +0200 Subject: [PATCH 063/413] Fix adlist.list migration step failing during tests Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index d784a2e7..89b4ab84 100755 --- a/gravity.sh +++ b/gravity.sh @@ -178,7 +178,7 @@ database_table_from_file() { echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" elif [[ "${table}" == "adlist" ]]; then # Adlist table format - echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0,0" >> "${tmpFile}" + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0,0,0" >> "${tmpFile}" else # White-, black-, and regexlist table format echo "${rowid},${list_type},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\"" >> "${tmpFile}" From 375d4d9bc13b668682ded29efd2387d62d5623c5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 6 Jul 2023 21:33:59 +0200 Subject: [PATCH 064/413] Add type as new field of view vw_adlist Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 17712cf7..46f26ba7 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -65,7 +65,7 @@ CREATE TABLE info value TEXT NOT NULL ); -INSERT INTO "info" VALUES('version','16'); +INSERT INTO "info" VALUES('version','17'); CREATE TABLE domain_audit ( @@ -158,7 +158,7 @@ CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_i LEFT JOIN "group" ON "group".id = adlist_by_group.group_id WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; -CREATE VIEW vw_adlist AS SELECT DISTINCT address, id +CREATE VIEW vw_adlist AS SELECT DISTINCT address, id, type FROM adlist WHERE enabled = 1 ORDER BY id; From 2a03671fb9f15cfe7e6ee378350d7f32d2e04000 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 6 Jul 2023 22:52:28 +0200 Subject: [PATCH 065/413] Reinstall gravity cleanup on error Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 89b4ab84..acf55ae3 100755 --- a/gravity.sh +++ b/gravity.sh @@ -395,7 +395,7 @@ gravity_DownloadBlocklists() { if [[ "${status}" -ne 0 ]]; then echo -e "\\n ${CROSS} Unable to create new database ${gravityTEMPfile}\\n ${output}" - #gravity_Cleanup "error" + gravity_Cleanup "error" else echo -e "${OVER} ${TICK} ${str}" fi From 5ae0405446103a29c525cba9cdeb12ad390372b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 3 Aug 2023 20:46:01 +0200 Subject: [PATCH 066/413] Ensure pihole-FTL can write custom.list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 4 ++-- automated install/basic-install.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index ff4abf3a..f6e28fec 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -17,9 +17,9 @@ mkdir -pm 0755 /run/pihole /var/log/pihole [ -f /var/log/pihole/pihole.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/pihole.log [ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit all necessary files -chown pihole:pihole /run/pihole /etc/pihole /var/log/pihole /var/log/pihole/FTL.log /var/log/pihole/pihole.log /etc/pihole/dhcp.leases +chown pihole:pihole /run/pihole /etc/pihole /var/log/pihole /var/log/pihole/FTL.log /var/log/pihole/pihole.log /etc/pihole/dhcp.leases /etc/pihole/custom.list # Ensure that permissions are set so that pihole-FTL can edit the files. We ignore errors as the file may not (yet) exist -chmod -f 0644 /etc/pihole/macvendor.db /etc/pihole/dhcp.leases /var/log/pihole/FTL.log +chmod -f 0644 /etc/pihole/macvendor.db /etc/pihole/dhcp.leases /var/log/pihole/FTL.log /etc/pihole/custom.list chmod -f 0640 /var/log/pihole/pihole.log # Chown database files to the user FTL runs as. We ignore errors as the files may not (yet) exist chown -f pihole:pihole /etc/pihole/pihole-FTL.db /etc/pihole/gravity.db /etc/pihole/macvendor.db diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index e3d8ff29..050883cd 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1094,7 +1094,7 @@ installConfigs() { # Install empty custom.list file if it does not exist if [[ ! -r "${PI_HOLE_CONFIG_DIR}/custom.list" ]]; then - if ! install -o root -m 644 /dev/null "${PI_HOLE_CONFIG_DIR}/custom.list" &>/dev/null; then + if ! install -o pihole -g pihole -m 644 /dev/null "${PI_HOLE_CONFIG_DIR}/custom.list" &>/dev/null; then printf " %b Error: Unable to initialize configuration file %s/custom.list\\n" "${COL_LIGHT_RED}" "${PI_HOLE_CONFIG_DIR}" return 1 fi From a3ea2cd8c311e4c81d7b0654c0893b4bf5cc1e5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 4 Aug 2023 19:32:19 +0200 Subject: [PATCH 067/413] User pihole should be allowed to edit all its files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index f6e28fec..e6a5aeb2 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -17,14 +17,7 @@ mkdir -pm 0755 /run/pihole /var/log/pihole [ -f /var/log/pihole/pihole.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/pihole.log [ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit all necessary files -chown pihole:pihole /run/pihole /etc/pihole /var/log/pihole /var/log/pihole/FTL.log /var/log/pihole/pihole.log /etc/pihole/dhcp.leases /etc/pihole/custom.list -# Ensure that permissions are set so that pihole-FTL can edit the files. We ignore errors as the file may not (yet) exist -chmod -f 0644 /etc/pihole/macvendor.db /etc/pihole/dhcp.leases /var/log/pihole/FTL.log /etc/pihole/custom.list -chmod -f 0640 /var/log/pihole/pihole.log -# Chown database files to the user FTL runs as. We ignore errors as the files may not (yet) exist -chown -f pihole:pihole /etc/pihole/pihole-FTL.db /etc/pihole/gravity.db /etc/pihole/macvendor.db -# Chmod database file permissions so that the pihole group (web interface) can edit the file. We ignore errors as the files may not (yet) exist -chmod -f 0664 /etc/pihole/pihole-FTL.db +chown -R pihole:pihole /run/pihole /etc/pihole /var/log/pihole # Backward compatibility for user-scripts that still expect log files in /var/log instead of /var/log/pihole # Should be removed with Pi-hole v6.0 From 8ef8a275830341140009bdbf229ae8cebad7aede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 4 Aug 2023 19:41:19 +0200 Subject: [PATCH 068/413] Remove webpage.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 100 ------------------------------------ pihole | 41 ++++++++++++--- 2 files changed, 33 insertions(+), 108 deletions(-) delete mode 100755 advanced/Scripts/webpage.sh diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh deleted file mode 100755 index 67cbe766..00000000 --- a/advanced/Scripts/webpage.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC1090 -# shellcheck disable=SC2154 - - -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Web interface settings -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. - -# TODO - this entire file might be able to be removed in v6 - -readonly dnsmasqconfig="/etc/dnsmasq.d/01-pihole.conf" -readonly dhcpconfig="/etc/dnsmasq.d/02-pihole-dhcp.conf" -readonly FTLconf="/etc/pihole/pihole-FTL.conf" -# 03 -> wildcards -readonly dhcpstaticconfig="/etc/dnsmasq.d/04-pihole-static-dhcp.conf" -readonly dnscustomfile="/etc/pihole/custom.list" -readonly dnscustomcnamefile="/etc/dnsmasq.d/05-pihole-custom-cname.conf" - -readonly gravityDBfile="/etc/pihole/gravity.db" - - -readonly setupVars="/etc/pihole/setupVars.conf" -readonly PI_HOLE_BIN_DIR="/usr/local/bin" - -# Root of the web server -readonly webroot="/var/www/html" - -# Source utils script -utilsfile="/opt/pihole/utils.sh" -source "${utilsfile}" - -coltable="/opt/pihole/COL_TABLE" -if [[ -f ${coltable} ]]; then - source ${coltable} -fi - -helpFunc() { - echo "Usage: pihole -a [options] -Example: pihole -a -p password -Set options for the API/Web interface - -Options: - -p, password Set API/Web interface password - -h, --help Show this help dialog" - exit 0 -} - -# TODO: We can probably remove the reliance on this function too, just tell people to pihole-FTL --config webserver.api.password "password" -SetWebPassword() { - if (( ${#args[2]} > 0 )) ; then - readonly PASSWORD="${args[2]}" - readonly CONFIRM="${PASSWORD}" - else - # Prevents a bug if the user presses Ctrl+C and it continues to hide the text typed. - # So we reset the terminal via stty if the user does press Ctrl+C - trap '{ echo -e "\nNot changed" ; stty sane ; exit 1; }' INT - read -s -r -p "Enter New Password (Blank for no password): " PASSWORD - echo "" - - if [ "${PASSWORD}" == "" ]; then - setFTLConfigValue "webserver.api.pwhash" "" >/dev/null - echo -e " ${TICK} Password Removed" - exit 0 - fi - - read -s -r -p "Confirm Password: " CONFIRM - echo "" - fi - - if [ "${PASSWORD}" == "${CONFIRM}" ] ; then - # pihole-FTL will automatically hash the password - setFTLConfigValue "webserver.api.password" "${PASSWORD}" >/dev/null - echo -e " ${TICK} New password set" - else - echo -e " ${CROSS} Passwords don't match. Your password has not been changed" - exit 1 - fi -} - -main() { - args=("$@") - - case "${args[1]}" in - "-p" | "password" ) SetWebPassword;; - "-h" | "--help" ) helpFunc;; - * ) helpFunc;; - esac - - shift - - if [[ $# = 0 ]]; then - helpFunc - fi -} diff --git a/pihole b/pihole index 66771b9b..54b20f7d 100755 --- a/pihole +++ b/pihole @@ -30,10 +30,36 @@ if [ -f "${versionsfile}" ]; then source "${versionsfile}" fi -webpageFunc() { - source "${PI_HOLE_SCRIPT_DIR}/webpage.sh" - main "$@" - exit 0 +# TODO: We can probably remove the reliance on this function too, just tell people to pihole-FTL --config webserver.api.password "password" +SetWebPassword() { + if [ -n "$2" ] ; then + readonly PASSWORD="$2" + readonly CONFIRM="${PASSWORD}" + else + # Prevents a bug if the user presses Ctrl+C and it continues to hide the text typed. + # So we reset the terminal via stty if the user does press Ctrl+C + trap '{ echo -e "\nNot changed" ; stty sane ; exit 1; }' INT + read -s -r -p "Enter New Password (Blank for no password): " PASSWORD + echo "" + + if [ "${PASSWORD}" == "" ]; then + setFTLConfigValue "webserver.api.pwhash" "" >/dev/null + echo -e " ${TICK} Password Removed" + exit 0 + fi + + read -s -r -p "Confirm Password: " CONFIRM + echo "" + fi + + if [ "${PASSWORD}" == "${CONFIRM}" ] ; then + # pihole-FTL will automatically hash the password + setFTLConfigValue "webserver.api.password" "${PASSWORD}" >/dev/null + echo -e " ${TICK} New password set" + else + echo -e " ${CROSS} Passwords don't match. Your password has not been changed" + exit 1 + fi } listFunc() { @@ -466,8 +492,7 @@ Debugging Options: Options: - -a, admin Web interface options - Add '-h' for more info on Web Interface usage + setpassword set the password for the web interface -c, chronometer Calculates stats and displays to an LCD Add '-h' for more info on chronometer usage -g, updateGravity Update the list of ad-serving domains @@ -526,7 +551,7 @@ case "${1}" in "restartdns" ) ;; "-g" | "updateGravity" ) need_root=0;; "reloaddns" ) need_root=0;; - "-a" | "admin" ) ;; + "setpassword" ) ;; "checkout" ) ;; "updatechecker" ) ;; "arpflush" ) ;; @@ -581,7 +606,7 @@ case "${1}" in "disable" ) piholeEnable 0 "$2";; "restartdns" ) restartDNS "$2";; "reloaddns" ) restartDNS "reload";; - "-a" | "admin" ) webpageFunc "$@";; + "setpassword" ) SetWebPassword "$@";; "checkout" ) piholeCheckoutFunc "$@";; "updatechecker" ) shift; updateCheckFunc "$@";; "arpflush" ) arpFunc "$@";; From c360743d41620972cfa0b9b10190d010d12bba5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 4 Aug 2023 21:12:40 +0200 Subject: [PATCH 069/413] Re-add file/folder permissions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index e6a5aeb2..ef8e1b88 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -18,6 +18,8 @@ mkdir -pm 0755 /run/pihole /var/log/pihole [ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit all necessary files chown -R pihole:pihole /run/pihole /etc/pihole /var/log/pihole +chmod -R 0640 /var/log/pihole +chmod -R 0660 /etc/pihole /run/pihole # Backward compatibility for user-scripts that still expect log files in /var/log instead of /var/log/pihole # Should be removed with Pi-hole v6.0 From 587a2a1c04fa743480d808f74b87da3d8539bc9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 4 Aug 2023 23:52:53 +0200 Subject: [PATCH 070/413] # allow all users to enter der directories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index ef8e1b88..62183db9 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -20,6 +20,8 @@ mkdir -pm 0755 /run/pihole /var/log/pihole chown -R pihole:pihole /run/pihole /etc/pihole /var/log/pihole chmod -R 0640 /var/log/pihole chmod -R 0660 /etc/pihole /run/pihole +# allow all users to enter der directories +chmod 0755 /etc/pihole /run/pihole /var/log/pihole # Backward compatibility for user-scripts that still expect log files in /var/log instead of /var/log/pihole # Should be removed with Pi-hole v6.0 From 7bb0ca59d157f21a50e1e4abcf18734ae2c62347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 5 Aug 2023 14:54:09 +0200 Subject: [PATCH 071/413] Remove traces of /run/pihole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index 62183db9..a0353f34 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -11,17 +11,17 @@ FTL_PID_FILE="$(getFTLPIDFile)" # Touch files to ensure they exist (create if non-existing, preserve if existing) # shellcheck disable=SC2174 -mkdir -pm 0755 /run/pihole /var/log/pihole +mkdir -pm 0755 /var/log/pihole [ -f "${FTL_PID_FILE}" ] || install -D -m 644 -o pihole -g pihole /dev/null "${FTL_PID_FILE}" [ -f /var/log/pihole/FTL.log ] || install -m 644 -o pihole -g pihole /dev/null /var/log/pihole/FTL.log [ -f /var/log/pihole/pihole.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/pihole.log [ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit all necessary files -chown -R pihole:pihole /run/pihole /etc/pihole /var/log/pihole +chown -R pihole:pihole /etc/pihole /var/log/pihole chmod -R 0640 /var/log/pihole chmod -R 0660 /etc/pihole /run/pihole # allow all users to enter der directories -chmod 0755 /etc/pihole /run/pihole /var/log/pihole +chmod 0755 /etc/pihole /var/log/pihole # Backward compatibility for user-scripts that still expect log files in /var/log instead of /var/log/pihole # Should be removed with Pi-hole v6.0 From 3c693c1da54b15869d7d612bdf20153234911073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 6 Aug 2023 12:04:55 +0200 Subject: [PATCH 072/413] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: DL6ER Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 4 ++-- automated install/basic-install.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index a0353f34..17900f0b 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -19,8 +19,8 @@ mkdir -pm 0755 /var/log/pihole # Ensure that permissions are set so that pihole-FTL can edit all necessary files chown -R pihole:pihole /etc/pihole /var/log/pihole chmod -R 0640 /var/log/pihole -chmod -R 0660 /etc/pihole /run/pihole -# allow all users to enter der directories +chmod -R 0660 /etc/pihole +# allow all users to enter the directories chmod 0755 /etc/pihole /var/log/pihole # Backward compatibility for user-scripts that still expect log files in /var/log instead of /var/log/pihole diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 050883cd..bf26631a 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1094,7 +1094,7 @@ installConfigs() { # Install empty custom.list file if it does not exist if [[ ! -r "${PI_HOLE_CONFIG_DIR}/custom.list" ]]; then - if ! install -o pihole -g pihole -m 644 /dev/null "${PI_HOLE_CONFIG_DIR}/custom.list" &>/dev/null; then + if ! install -o pihole -g pihole -m 660 /dev/null "${PI_HOLE_CONFIG_DIR}/custom.list" &>/dev/null; then printf " %b Error: Unable to initialize configuration file %s/custom.list\\n" "${COL_LIGHT_RED}" "${PI_HOLE_CONFIG_DIR}" return 1 fi From 2227a2c5693633d89fb99fe3a2079d26c9bfe225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 9 Aug 2023 21:25:50 +0200 Subject: [PATCH 073/413] Add API functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/api.sh | 140 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 advanced/Scripts/api.sh diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh new file mode 100644 index 00000000..afd88671 --- /dev/null +++ b/advanced/Scripts/api.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env sh +# shellcheck disable=SC3043 #https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions + +# Pi-hole: A black hole for Internet advertisements +# (c) 2017 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# Script to hold api functions for use in other scripts +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + + +# The basic usage steps are +# 1) Test Availability of the API +# 2) Try to authenticate (read password if needed) +# 3) Get the data from the API endpoint +# 4) Delete the session + + +TestAPIAvailability() { + + # as we are running locally, we can get the port value from FTL directly + PORT="$(pihole-FTL --config webserver.port)" + PORT="${PORT%%,*}" + + availabilityResonse=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:${PORT}/api/auth") + + # test if http status code was 200 (OK) or 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 401 ]; then + echo "API not available at: http://localhost:${PORT}/api" + echo "Exiting." + exit 1 + fi +} + +Authenthication() { + # Try to authenticate + LoginAPI + + while [ "${validSession}" = false ] || [ -z "${validSession}" ] ; do + echo "Authentication failed. Please enter your Pi-hole password" + + # secretly read the password + secretRead; printf '\n' + + # Try to authenticate again + LoginAPI + done + + # Loop exited, authentication was successful + echo "Authentication successful." + +} + +LoginAPI() { + sessionResponse="$(curl --silent -X POST "http://localhost:${PORT}/api/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" + + if [ -z "${sessionResponse}" ]; then + echo "No response from FTL server. Please check connectivity" + exit 1 + fi + # obtain validity and session ID from session response + validSession=$(echo "${sessionResponse}"| jq .session.valid 2>/dev/null) + SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null) +} + +DeleteSession() { + # if a valid Session exists (no password required or successful authenthication) and + # SID is not null (successful authenthication only), delete the session + if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then + # Try to delete the session. Omit the output, but get the http status code + deleteResponse=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE "http://localhost:${PORT}/api/auth" -H "Accept: application/json" -H "sid: ${SID}") + + case "${deleteResponse}" in + "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; + "401") printf "%b" "Logout attempt without a valid session. Unauthorized!\n";; + "410") printf "%b" "Session successfully deleted.\n";; + esac; + fi + +} + +GetFTLData() { + local data + # get the data from querying the API as well as the http status code + data=$(curl -s -X GET "http://localhost:${PORT}/api$1" -H "Accept: application/json" -H "sid: ${SID}" ) + echo "${data}" +} + +secretRead() { + + # POSIX compliant function to read user-input and + # mask every character entered by (*) + # + # This is challenging, because in POSIX, `read` does not support + # `-s` option (suppressing the input) or + # `-n` option (reading n chars) + + + # This workaround changes the terminal characteristics to not echo input and later resets this option + # credits https://stackoverflow.com/a/4316765 + # showing asterisk instead of password + # https://stackoverflow.com/a/24600839 + # https://unix.stackexchange.com/a/464963 + + + # Save current terminal settings (needed for later restore after password prompt) + stty_orig=$(stty -g) + + stty -echo # do not echo user input + stty -icanon min 1 time 0 # disable canonical mode https://man7.org/linux/man-pages/man3/termios.3.html + + unset password + unset key + unset charcount + charcount=0 + while key=$(dd ibs=1 count=1 2>/dev/null); do #read one byte of input + if [ "${key}" = "$(printf '\0' | tr -d '\0')" ] ; then + # Enter - accept password + break + fi + if [ "${key}" = "$(printf '\177')" ] ; then + # Backspace + if [ $charcount -gt 0 ] ; then + charcount=$((charcount-1)) + printf '\b \b' + password="${password%?}" + fi + else + # any other character + charcount=$((charcount+1)) + printf '*' + password="$password$key" + fi + done + + # restore original terminal settings + stty "${stty_orig}" +} From bd55b2e5660d31dd5f99e2343ab5c31e871c0075 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 12 Aug 2023 10:25:00 +0000 Subject: [PATCH 074/413] Bump tox from 4.6.4 to 4.7.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.6.4 to 4.7.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.6.4...4.7.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 5a551fa7..c5d002e5 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 -tox == 4.6.4 +tox == 4.7.0 From 8e0a40e71706b70b880a288e0bc93780d4ee54db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 13 Aug 2023 22:00:44 +0200 Subject: [PATCH 075/413] Use api.sh within query.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/COL_TABLE | 2 +- advanced/Scripts/query.sh | 325 ++++++++++++------------------------- 2 files changed, 104 insertions(+), 223 deletions(-) mode change 100755 => 100644 advanced/Scripts/query.sh diff --git a/advanced/Scripts/COL_TABLE b/advanced/Scripts/COL_TABLE index 2d2b074b..4344c7b8 100644 --- a/advanced/Scripts/COL_TABLE +++ b/advanced/Scripts/COL_TABLE @@ -1,5 +1,5 @@ # Determine if terminal is capable of showing colors -if ([[ -t 1 ]] && [[ $(tput colors) -ge 8 ]]) || [[ "${WEBCALL}" ]]; then +if ([ -t 1 ] && [ $(tput colors) -ge 8 ]) || [ "${WEBCALL}" ]; then # Bold and underline may not show up on all clients # If something MUST be emphasized, use both COL_BOLD='' diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh old mode 100755 new mode 100644 index bfa21247..8407d3f6 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -1,259 +1,140 @@ -#!/usr/bin/env bash +#!/usr/bin/env sh # shellcheck disable=SC1090 +# Ignore warning about `local` being undefinded in POSIX +# shellcheck disable=SC3043 +# https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions + # Pi-hole: A black hole for Internet advertisements -# (c) 2018 Pi-hole, LLC (https://pi-hole.net) +# (c) 2023 Pi-hole, LLC (https://pi-hole.net) # Network-wide ad blocking via your own hardware. # -# Query Domain Lists +# Search Adlists # # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. # Globals -piholeDir="/etc/pihole" -GRAVITYDB="${piholeDir}/gravity.db" -options="$*" -all="" -exact="" -matchType="match" -# Source pihole-FTL from install script -pihole_FTL="${piholeDir}/pihole-FTL.conf" -if [[ -f "${pihole_FTL}" ]]; then - source "${pihole_FTL}" -fi - -# Set this only after sourcing pihole-FTL.conf as the gravity database path may -# have changed -gravityDBfile="${GRAVITYDB}" +PI_HOLE_INSTALL_DIR="/opt/pihole" +max_results="20" +partial="true" +domain="" +# Source color table colfile="/opt/pihole/COL_TABLE" -source "${colfile}" +. "${colfile}" -if [[ "${options}" == "-h" ]] || [[ "${options}" == "--help" ]]; then +# Source api functions +. "${PI_HOLE_INSTALL_DIR}/api.sh" + +Help(){ echo "Usage: pihole -q [option] -Example: 'pihole -q -exact domain.com' +Example: 'pihole -q --exact domain.com' Query the adlists for a specified domain Options: - -exact Search the adlists for exact domain matches - -all Return all query matches within the adlists + --exact Search the adlists for exact domain matches + --all Return all query matches within the adlists -h, --help Show this help dialog" exit 0 -fi - -# Handle valid options -[[ "${options}" == *"-all"* ]] && all=true -if [[ "${options}" == *"-exact"* ]]; then - exact="exact"; matchType="exact ${matchType}" -fi - -# Strip valid options, leaving only the domain and invalid options -# This allows users to place the options before or after the domain -options=$(sed -E 's/ +-(all|exact) ?//g' <<< "${options}") - -# Handle remaining options -# If $options contain non ASCII characters, convert to punycode -case "${options}" in - "" ) str="No domain specified";; - *" "* ) str="Unknown query option specified";; - *[![:ascii:]]* ) rawDomainQuery=$(idn2 "${options}");; - * ) rawDomainQuery="${options}";; -esac - -# convert the domain to lowercase -domainQuery=$(echo "${rawDomainQuery}" | tr '[:upper:]' '[:lower:]') - -if [[ -n "${str:-}" ]]; then - echo -e "${str}${COL_NC}\\nTry 'pihole -q --help' for more information." - exit 1 -fi - -# Scan a domain again a list of RegEX -scanRegExList(){ - local domain="${1}" list="${2}" - - for entry in ${list}; do - if [[ "${domain}" =~ ${entry} ]]; then - printf "%b\n" "${entry}"; - fi - done - } -scanDatabaseTable() { - local domain table list_type querystr result extra abpquerystr abpfound abpentry searchstr - domain="$(printf "%q" "${1}")" - table="${2}" - list_type="${3:-}" - # As underscores are legitimate parts of domains, we escape them when using the LIKE operator. - # Underscores are SQLite wildcards matching exactly one character. We obviously want to suppress this - # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched - # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores. - if [[ "${table}" == "gravity" ]]; then +GenerateOutput(){ + local data gravity_data lists_data num_gravity num_lists search_type_str + local gravity_data_csv lists_data_csv line current_domain + data="${1}" - # Are there ABP entries on gravity? - # Return 1 if abp_domain=1 or Zero if abp_domain=0 or not set - abpquerystr="SELECT EXISTS (SELECT 1 FROM info WHERE property='abp_domains' and value='1')" - abpfound="$(pihole-FTL sqlite3 "${gravityDBfile}" "${abpquerystr}")" 2> /dev/null + # construct a new json for the list results where each object contains the domain and the related type + lists_data=$(echo "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]') - # Create search string for ABP entries only if needed - if [ "${abpfound}" -eq 1 ]; then - abpentry="${domain}" + # construct a new json for the gravity results where each object contains the adlist URL and the related domains + gravity_data=$(echo "${data}" | jq '.search.gravity | group_by(.address) | map({ address: (.[0].address), domains: [.[] | .domain] })') - searchstr="'||${abpentry}^'" + # number of objects in each json + num_gravity=$(echo "${gravity_data}" | jq length ) + num_lists=$(echo "${lists_data}" | jq length ) - # While a dot is found ... - while [ "${abpentry}" != "${abpentry/./}" ] - do - # ... remove text before the dot (including the dot) and append the result to $searchstr - abpentry=$(echo "${abpentry}" | cut -f 2- -d '.') - searchstr="$searchstr, '||${abpentry}^'" + if [ "${partial}" = true ]; then + search_type_str="partially" + else + search_type_str="exactly" + fi + + # Results from allow/deny list + printf "%s\n\n" "Found ${num_lists} domains ${search_type_str} matching '${COL_BLUE}${domain}${COL_NC}'." + if [ "${num_lists}" -gt 0 ]; then + # Convert the data to a csv, each line is a "domain,type" string + # not using jq's @csv here as it quotes each value individually + lists_data_csv=$(echo "${lists_data}" | jq --raw-output '.[] | [.domain, .type] | join(",")' ) + + # Generate output for each csv line, separating line in a domain and type substring at the ',' + echo "${lists_data_csv}" | while read -r line; do + printf "%s\n\n" " - ${COL_GREEN}${line%,*}${COL_NC} (type: exact ${line#*,} domain)" + done + fi + + # Results from gravity + printf "%s\n\n" "Found ${num_gravity} adlists ${search_type_str} matching '${COL_BLUE}${domain}${COL_NC}'." + if [ "${num_gravity}" -gt 0 ]; then + # Convert the data to a csv, each line is a "URL,domain,domain,...." string + # not using jq's @csv here as it quotes each value individually + gravity_data_csv=$(echo "${gravity_data}" | jq --raw-output '.[] | [.address, .domains[]] | join(",")' ) + + # Generate line-by-line output for each csv line + echo "${gravity_data_csv}" | while read -r line; do + + # print adlist URL + printf "%s\n\n" " - ${COL_BLUE}${line%%,*}${COL_NC}" + + # cut off URL, leaving "domain,domain,...." + line=${line#*,} + # print each domain and remove it from the string until nothing is left + while [ ${#line} -gt 0 ]; do + current_domain=${line%%,*} + printf ' - %s\n' "${COL_GREEN}${current_domain}${COL_NC}" + # we need to remove the current_domain and the comma in two steps because + # the last domain won't have a trailing comma and the while loop wouldn't exit + line=${line#"${current_domain}"} + line=${line#,} done - - # The final search string will look like: - # "domain IN ('||sub2.sub1.domain.com^', '||sub1.domain.com^', '||domain.com^', '||com^') OR" - searchstr="domain IN (${searchstr}) OR " - fi - - case "${exact}" in - "exact" ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain = '${domain}'";; - * ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE ${searchstr} domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; - esac - else - case "${exact}" in - "exact" ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND domain = '${domain}'";; - * ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; - esac - fi - - # Send prepared query to gravity database - result="$(pihole-FTL sqlite3 -separator ',' "${gravityDBfile}" "${querystr}")" 2> /dev/null - if [[ -z "${result}" ]]; then - # Return early when there are no matches in this table - return - fi - - if [[ "${table}" == "gravity" ]]; then - echo "${result}" - return - fi - - # Mark domain as having been white-/blacklist matched (global variable) - wbMatch=true - - # Print table name - echo " ${matchType^} found in ${COL_BOLD}exact ${table}${COL_NC}" - - # Loop over results and print them - mapfile -t results <<< "${result}" - for result in "${results[@]}"; do - domain="${result/,*}" - if [[ "${result#*,}" == "0" ]]; then - extra=" (disabled)" - else - extra="" - fi - echo " ${domain}${extra}" - done -} - -scanRegexDatabaseTable() { - local domain list list_type - domain="${1}" - list="${2}" - list_type="${3:-}" - - # Query all regex from the corresponding database tables - mapfile -t regexList < <(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain FROM domainlist WHERE type = ${list_type}" 2> /dev/null) - - # If we have regexps to process - if [[ "${#regexList[@]}" -ne 0 ]]; then - # Split regexps over a new line - str_regexList=$(printf '%s\n' "${regexList[@]}") - # Check domain against regexps - mapfile -t regexMatches < <(scanRegExList "${domain}" "${str_regexList}") - # If there were regex matches - if [[ "${#regexMatches[@]}" -ne 0 ]]; then - # Split matching regexps over a new line - str_regexMatches=$(printf '%s\n' "${regexMatches[@]}") - # Form a "matched" message - str_message="${matchType^} found in ${COL_BOLD}regex ${list}${COL_NC}" - # Form a "results" message - str_result="${COL_BOLD}${str_regexMatches}${COL_NC}" - # If we are displaying more than just the source of the block - # Set the wildcard match flag - wcMatch=true - # Echo the "matched" message, indented by one space - echo " ${str_message}" - # Echo the "results" message, each line indented by three spaces - # shellcheck disable=SC2001 - echo "${str_result}" | sed 's/^/ /' - fi + printf "\n\n" + done fi } -# Scan Whitelist and Blacklist -scanDatabaseTable "${domainQuery}" "whitelist" "0" -scanDatabaseTable "${domainQuery}" "blacklist" "1" +Main(){ + local data -# Scan Regex table -scanRegexDatabaseTable "${domainQuery}" "whitelist" "2" -scanRegexDatabaseTable "${domainQuery}" "blacklist" "3" - -# Query block lists -mapfile -t results <<< "$(scanDatabaseTable "${domainQuery}" "gravity")" - -# Handle notices -if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then - echo -e " ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} within the adlists" - exit 0 -elif [[ -z "${results[*]}" ]]; then - # Result found in WL/BL/Wildcards - exit 0 -elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 100 ]]; then - echo -e " ${INFO} Over 100 ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} - This can be overridden using the -all option" - exit 0 -fi - -# Print "Exact matches for" title -if [[ -n "${exact}" ]]; then - plural=""; [[ "${#results[*]}" -gt 1 ]] && plural="es" - echo " ${matchType^}${plural} for ${COL_BOLD}${domainQuery}${COL_NC} found in:" -fi - -for result in "${results[@]}"; do - match="${result/,*/}" - extra="${result#*,}" - adlistAddress="${extra/,*/}" - extra="${extra#*,}" - if [[ "${extra}" == "0" ]]; then - extra=" (disabled)" + if [ -z "${domain}" ]; then + echo "No domain specified"; exit 1 else - extra="" + # convert domain to punycode + domain=$(idn2 "${domain}") + + # convert the domain to lowercase + domain=$(echo "${domain}" | tr '[:upper:]' '[:lower:]') fi - if [[ -n "${exact}" ]]; then - echo " - ${adlistAddress}${extra}" - else - if [[ ! "${adlistAddress}" == "${adlistAddress_prev:-}" ]]; then - count="" - echo " ${matchType^} found in ${COL_BOLD}${adlistAddress}${COL_NC}:" - adlistAddress_prev="${adlistAddress}" - fi - : $((count++)) + # Test if the authentication endpoint is available + TestAPIAvailability + # Authenticate with the FTL server + Authenthication - # Print matching domain if $max_count has not been reached - [[ -z "${all}" ]] && max_count="50" - if [[ -z "${all}" ]] && [[ "${count}" -ge "${max_count}" ]]; then - [[ "${count}" -gt "${max_count}" ]] && continue - echo " ${COL_GRAY}Over ${count} results found, skipping rest of file${COL_NC}" - else - echo " ${match}${extra}" - fi - fi + data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}") + GenerateOutput "${data}" + DeleteSession +} + +# Process all options (if present) +while [ "$#" -gt 0 ]; do + case "$1" in + "-h" | "--help" ) Help;; + "--exact" ) partial="false";; + "--all" ) max_results=10000;; # hard-coded FTL limit + * ) domain=$1;; + esac + shift done -exit 0 +Main "${domain}" From ee4eb8db201a375afe3fdf9f7af1d52ce2a630cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 13 Aug 2023 22:26:44 +0200 Subject: [PATCH 076/413] chmod +x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/api.sh | 0 advanced/Scripts/query.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 advanced/Scripts/api.sh mode change 100644 => 100755 advanced/Scripts/query.sh diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh old mode 100644 new mode 100755 diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh old mode 100644 new mode 100755 From 43882693a5d72efdd2fb91b593a6b50dc9041b92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 19 Aug 2023 10:19:26 +0000 Subject: [PATCH 077/413] Bump tox from 4.7.0 to 4.9.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.7.0 to 4.9.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.7.0...4.9.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index c5d002e5..0210aa4e 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 -tox == 4.7.0 +tox == 4.9.0 From 1afc96c05550753a84ad0defe1a9bcf8c95c5970 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Aug 2023 10:46:28 +0000 Subject: [PATCH 078/413] Bump pytest-testinfra from 8.1.0 to 9.0.0 in /test Bumps [pytest-testinfra](https://github.com/pytest-dev/pytest-testinfra) from 8.1.0 to 9.0.0. - [Release notes](https://github.com/pytest-dev/pytest-testinfra/releases) - [Changelog](https://github.com/pytest-dev/pytest-testinfra/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-testinfra/compare/8.1.0...9.0.0) --- updated-dependencies: - dependency-name: pytest-testinfra dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 0210aa4e..b1b36de1 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 -pytest-testinfra == 8.1.0 +pytest-testinfra == 9.0.0 tox == 4.9.0 From e65b171aea04d8f40dceaf5fa4706a2f0b793f14 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Aug 2023 11:24:09 +0000 Subject: [PATCH 079/413] Bump tox from 4.9.0 to 4.10.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.9.0 to 4.10.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.9.0...4.10.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index b1b36de1..90255f7f 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 -tox == 4.9.0 +tox == 4.10.0 From 53e8127781645d5af39644d95c6ffd894bb8d8cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Aug 2023 11:33:09 +0000 Subject: [PATCH 080/413] Bump actions/checkout from 3.5.3 to 3.6.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.5.3 to 3.6.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.5.3...v3.6.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 7fd0e4e4..1194ba0f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 43d5ca96..13b05df2 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 5435b7db..60f38cf6 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 09462eb4..6c76e9fd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Check scripts in repository are executable run: | @@ -72,7 +72,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.0 From eca84e0986c7373ebfe057447fa244f3ecf3dba2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 21 Aug 2023 21:28:17 +0200 Subject: [PATCH 081/413] Remove user agent when downloading adlists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/gravity.sh b/gravity.sh index ed402a34..248aa2da 100755 --- a/gravity.sh +++ b/gravity.sh @@ -382,7 +382,7 @@ gravity_DownloadBlocklists() { unset sources fi - local url domain agent str target compression + local url domain str target compression echo "" # Prepare new gravity database @@ -438,9 +438,6 @@ gravity_DownloadBlocklists() { saveLocation="${piholeDir}/list.${id}.${domain}.${domainsExtension}" activeDomains[$i]="${saveLocation}" - # Default user-agent (for Cloudflare's Browser Integrity Check: https://support.cloudflare.com/hc/en-us/articles/200170086-What-does-the-Browser-Integrity-Check-do-) - agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36" - echo -e " ${INFO} Target: ${url}" local regex check_url # Check for characters NOT allowed in URLs @@ -453,7 +450,7 @@ gravity_DownloadBlocklists() { if [[ "${check_url}" =~ ${regex} ]]; then echo -e " ${CROSS} Invalid Target" else - gravity_DownloadBlocklistFromUrl "${url}" "${agent}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" + gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" fi echo "" done @@ -485,7 +482,7 @@ compareLists() { # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { - local url="${1}" agent="${2}" adlistID="${3}" saveLocation="${4}" target="${5}" compression="${6}" + local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext # Create temp file to store content on disk instead of RAM @@ -545,7 +542,7 @@ gravity_DownloadBlocklistFromUrl() { fi # shellcheck disable=SC2086 - httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${listCurlBuffer}" 2> /dev/null) + httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2> /dev/null) case $url in # Did we "download" a local file? From c33963b25000272d023ff00f1c910bab3c891429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 4 Jun 2023 22:51:10 +0200 Subject: [PATCH 082/413] Check for valid OS via IPv4 and IPv6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 17 ++++++++++++++-- automated install/basic-install.sh | 31 ++++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index c3bc81b0..622ebd7a 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -339,7 +339,7 @@ os_check() { detected_os=$(grep "\bID\b" /etc/os-release | cut -d '=' -f2 | tr -d '"') detected_version=$(grep VERSION_ID /etc/os-release | cut -d '=' -f2 | tr -d '"') - cmdResult="$(dig +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" + cmdResult="$(dig -4 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" #Get the return code of the previous command (last line) digReturnCode="${cmdResult##*$'\n'}" @@ -349,7 +349,20 @@ os_check() { if [ "${digReturnCode}" -ne 0 ]; then log_write "${INFO} Distro: ${detected_os^}" log_write "${INFO} Version: ${detected_version}" - log_write "${CROSS} dig return code: ${COL_RED}${digReturnCode}${COL_NC}" + log_write "${CROSS} dig IPv4 return code: ${COL_RED}${digReturnCode}${COL_NC}" + log_write "${CROSS} dig response: ${response}" + log_write "${INFO} Retrying via IPv6" + + cmdResult="$(dig -6 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" + #Get the return code of the previous command (last line) + digReturnCode="${cmdResult##*$'\n'}" + + # Extract dig response + response="${cmdResult%%$'\n'*}" + fi + # If also no success via IPv6 + if [ "${digReturnCode}" -ne 0 ]; then + log_write "${CROSS} dig IPv6 return code: ${COL_RED}${digReturnCode}${COL_NC}" log_write "${CROSS} dig response: ${response}" log_write "${CROSS} Error: ${COL_RED}dig command failed - Unable to check OS${COL_NC}" else diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index e3d8ff29..990b8789 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -176,7 +176,8 @@ os_check() { detected_os=$(grep '^ID=' /etc/os-release | cut -d '=' -f2 | tr -d '"') detected_version=$(grep VERSION_ID /etc/os-release | cut -d '=' -f2 | tr -d '"') - cmdResult="$(dig +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" + # Test via IPv4 + cmdResult="$(dig -4 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" # Gets the return code of the previous command (last line) digReturnCode="${cmdResult##*$'\n'}" @@ -188,8 +189,34 @@ os_check() { # If the value of ${response} is a single 0, then this is the return code, not an actual response. if [ "${response}" == 0 ]; then valid_response=false + else + valid_response=true fi + fi + # Try again via IPv6 + if [ "$valid_response" = false ]; then + unset valid_response + + cmdResult="$(dig -6 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" + # Gets the return code of the previous command (last line) + digReturnCode="${cmdResult##*$'\n'}" + + if [ ! "${digReturnCode}" == "0" ]; then + valid_response=false + else + # Dig returned 0 (success), so get the actual response, and loop through it to determine if the detected variables above are valid + response="${cmdResult%%$'\n'*}" + # If the value of ${response} is a single 0, then this is the return code, not an actual response. + if [ "${response}" == 0 ]; then + valid_response=false + else + valid_response=true + fi + fi + fi + + if [ "$valid_response" = true ]; then IFS=" " read -r -a supportedOS < <(echo "${response}" | tr -d '"') for distro_and_versions in "${supportedOS[@]}" do @@ -212,7 +239,7 @@ os_check() { done fi - if [ "$valid_os" = true ] && [ "$valid_version" = true ] && [ ! "$valid_response" = false ]; then + if [ "$valid_os" = true ] && [ "$valid_version" = true ] && [ "$valid_response" = true ]; then display_warning=false fi From ff2c2290c7390ef73a34b1c1255af12efe476b61 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Sep 2023 10:27:10 +0000 Subject: [PATCH 083/413] Bump tox from 4.10.0 to 4.11.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.10.0 to 4.11.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.10.0...4.11.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 90255f7f..7f4d7e09 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 -tox == 4.10.0 +tox == 4.11.1 From 52268f01550acf609ddac0d7726cfb3fe03e4cc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 7 Sep 2023 22:27:49 +0200 Subject: [PATCH 084/413] Ignore ABP style entries in debug log dig test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index c3bc81b0..6ecb49b4 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -809,11 +809,15 @@ dig_at() { local record_type="A" fi - # Find a random blocked url that has not been whitelisted. + # Find a random blocked url that has not been whitelisted and is not ABP style. # This helps emulate queries to different domains that a user might query # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains local random_url - random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity ORDER BY RANDOM() LIMIT 1") + random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") + # Falback if no non-ABP style domains were found + if [ -z "${random_url}" ]; then + random_url="flurry.com" + fi # Next we need to check if Pi-hole can resolve a domain when the query is sent to it's IP address # This better emulates how clients will interact with Pi-hole as opposed to above where Pi-hole is From a229a623bbb282c7aa72e4485f4820f273c7fd5c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Sep 2023 10:05:32 +0000 Subject: [PATCH 085/413] Bump tox from 4.11.1 to 4.11.3 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.11.1 to 4.11.3. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.11.1...4.11.3) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 7f4d7e09..139be0fc 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 -tox == 4.11.1 +tox == 4.11.3 From 16ea50ad552de3cb70ffc6637614b1f3ef4878e0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Sep 2023 10:17:15 +0000 Subject: [PATCH 086/413] Bump actions/checkout from 3.6.0 to 4.0.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.6.0 to 4.0.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.6.0...v4.0.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1194ba0f..570fabdd 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 13b05df2..8d09e5e0 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 60f38cf6..184319e0 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6c76e9fd..bf027210 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Check scripts in repository are executable run: | @@ -72,7 +72,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.0 From dc73ace7c46c72d5692945026ae0fea0e45f5756 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Sep 2023 15:07:06 +0000 Subject: [PATCH 087/413] Bump pytest from 7.4.0 to 7.4.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.0 to 7.4.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.0...7.4.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 139be0fc..27417754 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.0 +pytest == 7.4.2 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 tox == 4.11.3 From df92b8ac14988ddeece92b733dec7e8b9fb84d3a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 12 Sep 2023 20:43:34 +0200 Subject: [PATCH 088/413] Add missing creation of view vw_antigravity as well as schema change to vw_adlist Signed-off-by: DL6ER --- .../database_migration/gravity/15_to_16.sql | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/advanced/Scripts/database_migration/gravity/15_to_16.sql b/advanced/Scripts/database_migration/gravity/15_to_16.sql index c6159f40..7c3b8197 100644 --- a/advanced/Scripts/database_migration/gravity/15_to_16.sql +++ b/advanced/Scripts/database_migration/gravity/15_to_16.sql @@ -6,6 +6,20 @@ BEGIN TRANSACTION; ALTER TABLE adlist ADD COLUMN abp_entries INTEGER NOT NULL DEFAULT 0; +CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id + FROM antigravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id + LEFT JOIN adlist ON adlist.id = antigravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; + +DROP VIEW vw_adlist; + +CREATE VIEW vw_adlist AS SELECT DISTINCT address, id, type + FROM adlist + WHERE enabled = 1 + ORDER BY id; + UPDATE info SET value = 16 WHERE property = 'version'; COMMIT; From ea23c8364dd75190b1a8765e8a302877063b9b99 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 16 Sep 2023 10:04:29 +0200 Subject: [PATCH 089/413] Move antigravity-related changed to gravits database migration step 16->17 Signed-off-by: DL6ER --- .../database_migration/gravity/15_to_16.sql | 14 -------------- .../database_migration/gravity/16_to_17.sql | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity/15_to_16.sql b/advanced/Scripts/database_migration/gravity/15_to_16.sql index 7c3b8197..c6159f40 100644 --- a/advanced/Scripts/database_migration/gravity/15_to_16.sql +++ b/advanced/Scripts/database_migration/gravity/15_to_16.sql @@ -6,20 +6,6 @@ BEGIN TRANSACTION; ALTER TABLE adlist ADD COLUMN abp_entries INTEGER NOT NULL DEFAULT 0; -CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id - FROM antigravity - LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id - LEFT JOIN adlist ON adlist.id = antigravity.adlist_id - LEFT JOIN "group" ON "group".id = adlist_by_group.group_id - WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; - -DROP VIEW vw_adlist; - -CREATE VIEW vw_adlist AS SELECT DISTINCT address, id, type - FROM adlist - WHERE enabled = 1 - ORDER BY id; - UPDATE info SET value = 16 WHERE property = 'version'; COMMIT; diff --git a/advanced/Scripts/database_migration/gravity/16_to_17.sql b/advanced/Scripts/database_migration/gravity/16_to_17.sql index c7b9049b..23532e3a 100644 --- a/advanced/Scripts/database_migration/gravity/16_to_17.sql +++ b/advanced/Scripts/database_migration/gravity/16_to_17.sql @@ -8,6 +8,20 @@ ALTER TABLE adlist ADD COLUMN type INTEGER NOT NULL DEFAULT 0; UPDATE adlist SET type = 0; +CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id + FROM antigravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id + LEFT JOIN adlist ON adlist.id = antigravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; + +DROP VIEW vw_adlist; + +CREATE VIEW vw_adlist AS SELECT DISTINCT address, id, type + FROM adlist + WHERE enabled = 1 + ORDER BY id; + UPDATE info SET value = 17 WHERE property = 'version'; COMMIT; From ec9d84692f55c3069ba895631437a050c7199086 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 23 Sep 2023 10:45:09 +0000 Subject: [PATCH 090/413] Bump actions/checkout from 4.0.0 to 4.1.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.0.0 to 4.1.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.0.0...v4.1.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 570fabdd..4685aa2c 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 8d09e5e0..0ff0a24a 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 184319e0..a1025629 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bf027210..054c09ac 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Check scripts in repository are executable run: | @@ -72,7 +72,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.0 From aba41b45b0e7ff67ed88cb6f706bdddb3bbadeb9 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 4 Oct 2023 16:20:38 +0100 Subject: [PATCH 091/413] Some verbiage change to outputs Signed-off-by: Adam Warner --- gravity.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index eced477d..9133d33d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -675,10 +675,10 @@ gravity_ShowCount() { # Here we use the table "gravity" instead of the view "vw_gravity" for speed. # It's safe to replace it here, because right after a gravity run both will show the exactly same number of domains. gravity_Table_Count "gravity" "gravity domains" "" - gravity_Table_Count "vw_blacklist" "exact blacklisted domains" - gravity_Table_Count "vw_regex_blacklist" "regex blacklist filters" - gravity_Table_Count "vw_whitelist" "exact whitelisted domains" - gravity_Table_Count "vw_regex_whitelist" "regex whitelist filters" + gravity_Table_Count "vw_blacklist" "exact denied domains" + gravity_Table_Count "vw_regex_blacklist" "regex denied filters" + gravity_Table_Count "vw_whitelist" "exact allowed domains" + gravity_Table_Count "vw_regex_whitelist" "regex allowed filters" } # Create "localhost" entries into hosts format From 885b626a68099bd44eb952307857438513a45cc9 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 4 Oct 2023 16:22:01 +0100 Subject: [PATCH 092/413] Some unrelated spelling mistakes that spellcheck is grumbling about Signed-off-by: Adam Warner --- advanced/Scripts/piholeDebug.sh | 6 +++--- automated install/basic-install.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 6ecb49b4..0e3bbf3d 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -397,7 +397,7 @@ os_check() { } diagnose_operating_system() { - # error message in a variable so we can easily modify it later (or re-use it) + # error message in a variable so we can easily modify it later (or reuse it) local error_msg="Distribution unknown -- most likely you are on an unsupported platform and may run into issues." # Display the current test that is running echo_current_diagnostic "Operating system" @@ -814,7 +814,7 @@ dig_at() { # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains local random_url random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") - # Falback if no non-ABP style domains were found + # Fallback if no non-ABP style domains were found if [ -z "${random_url}" ]; then random_url="flurry.com" fi @@ -1451,7 +1451,7 @@ upload_to_tricorder() { # If no token was generated else # Show an error and some help instructions - # Skip this if being called from web interface and autmatic mode was not chosen (users opt-out to upload) + # Skip this if being called from web interface and automatic mode was not chosen (users opt-out to upload) if [[ "${WEBCALL}" ]] && [[ ! "${AUTOMATED}" ]]; then : else diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index bf26631a..4e4bdfc7 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2203,7 +2203,7 @@ main() { # Check for and disable systemd-resolved-DNSStubListener before reloading resolved # DNSStubListener needs to remain in place for installer to download needed files, # so this change needs to be made after installation is complete, - # but before starting or resarting the ftl service + # but before starting or restarting the ftl service disable_resolved_stublistener printf " %b Restarting services...\\n" "${INFO}" From 70547755d661add4df58a39b18db01543cac2141 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Oct 2023 10:39:32 +0000 Subject: [PATCH 093/413] Bump actions/setup-python from 4.7.0 to 4.7.1 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.7.0 to 4.7.1. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.7.0...v4.7.1) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 054c09ac..9f32302e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,7 +75,7 @@ jobs: uses: actions/checkout@v4.1.0 - name: Set up Python 3.10 - uses: actions/setup-python@v4.7.0 + uses: actions/setup-python@v4.7.1 with: python-version: "3.10" From 044e856e6bd84a128b46930fb8a1a3489cf16ad1 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 8 Oct 2023 13:23:44 +0100 Subject: [PATCH 094/413] Disable checkout function for (official) docker containers Signed-off-by: Adam Warner --- pihole | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/pihole b/pihole index 54b20f7d..5281c715 100755 --- a/pihole +++ b/pihole @@ -413,26 +413,30 @@ tailFunc() { } piholeCheckoutFunc() { - if [[ "$2" == "-h" ]] || [[ "$2" == "--help" ]]; then - echo "Usage: pihole checkout [repo] [branch] -Example: 'pihole checkout master' or 'pihole checkout core dev' -Switch Pi-hole subsystems to a different GitHub branch + if [ -n "${DOCKER_VERSION}" ]; then + unsupportedFunc + else + if [[ "$2" == "-h" ]] || [[ "$2" == "--help" ]]; then + echo "Usage: pihole checkout [repo] [branch] + Example: 'pihole checkout master' or 'pihole checkout core dev' + Switch Pi-hole subsystems to a different GitHub branch -Repositories: - core [branch] Change the branch of Pi-hole's core subsystem - web [branch] Change the branch of Web Interface subsystem - ftl [branch] Change the branch of Pi-hole's FTL subsystem + Repositories: + core [branch] Change the branch of Pi-hole's core subsystem + web [branch] Change the branch of Web Interface subsystem + ftl [branch] Change the branch of Pi-hole's FTL subsystem -Branches: - master Update subsystems to the latest stable release - dev Update subsystems to the latest development release - branchname Update subsystems to the specified branchname" - exit 0 + Branches: + master Update subsystems to the latest stable release + dev Update subsystems to the latest development release + branchname Update subsystems to the specified branchname" + exit 0 + fi + + source "${PI_HOLE_SCRIPT_DIR}"/piholeCheckout.sh + shift + checkout "$@" fi - - source "${PI_HOLE_SCRIPT_DIR}"/piholeCheckout.sh - shift - checkout "$@" } tricorderFunc() { From 7886dc017266a63041bbbf2f3c751452a6661d9e Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 8 Oct 2023 14:09:47 +0100 Subject: [PATCH 095/413] adminlte->web Signed-off-by: Adam Warner --- README.md | 2 +- advanced/Scripts/chronometer.sh | 2 +- advanced/Scripts/update.sh | 2 +- advanced/Scripts/updatecheck.sh | 4 ++-- advanced/Scripts/version.sh | 16 ++++++++-------- automated install/basic-install.sh | 4 ++-- manpages/pihole.8 | 4 ++-- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index adfd3450..eb50030b 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ You can read our [Core Feature Breakdown](https://docs.pi-hole.net/core/pihole-c ### The Web Interface Dashboard -This [optional dashboard](https://github.com/pi-hole/AdminLTE) allows you to view stats, change settings, and configure your Pi-hole. It's the power of the Command Line Interface, with none of the learning curve! +This [optional dashboard](https://github.com/pi-hole/web) allows you to view stats, change settings, and configure your Pi-hole. It's the power of the Command Line Interface, with none of the learning curve! Some notable features include: diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh index 49de6efd..fc728e17 100755 --- a/advanced/Scripts/chronometer.sh +++ b/advanced/Scripts/chronometer.sh @@ -225,7 +225,7 @@ get_sys_stats() { if [[ -n "${ph_ver_raw[0]}" ]]; then ph_core_ver="${ph_ver_raw[0]}" if [[ ${#ph_ver_raw[@]} -eq 2 ]]; then - # AdminLTE not installed + # web not installed ph_lte_ver="(not installed)" ph_ftl_ver="${ph_ver_raw[1]}" else diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index b6153293..9dae66df 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -11,7 +11,7 @@ # Please see LICENSE file for your rights under this license. # Variables -readonly ADMIN_INTERFACE_GIT_URL="https://github.com/pi-hole/AdminLTE.git" +readonly ADMIN_INTERFACE_GIT_URL="https://github.com/pi-hole/web.git" readonly ADMIN_INTERFACE_DIR="/var/www/html/admin" readonly PI_HOLE_GIT_URL="https://github.com/pi-hole/pi-hole.git" readonly PI_HOLE_FILES_DIR="/etc/.pihole" diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 7d7103d2..8bb1888b 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -91,10 +91,10 @@ addOrEditKeyValPair "${VERSION_FILE}" "WEB_BRANCH" "${WEB_BRANCH}" WEB_HASH="$(get_local_hash /var/www/html/admin)" addOrEditKeyValPair "${VERSION_FILE}" "WEB_HASH" "${WEB_HASH}" -GITHUB_WEB_VERSION="$(get_remote_version AdminLTE)" +GITHUB_WEB_VERSION="$(get_remote_version web)" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_VERSION" "${GITHUB_WEB_VERSION}" -GITHUB_WEB_HASH="$(get_remote_hash AdminLTE "${WEB_BRANCH}")" +GITHUB_WEB_HASH="$(get_remote_hash web "${WEB_BRANCH}")" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_HASH" "${GITHUB_WEB_HASH}" # get FTL versions diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index af86b045..e3b4a6ae 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -24,7 +24,7 @@ fi getLocalVersion() { case ${1} in "Pi-hole" ) echo "${CORE_VERSION:=N/A}";; - "AdminLTE" ) echo "${WEB_VERSION:=N/A}";; + "web" ) echo "${WEB_VERSION:=N/A}";; "FTL" ) echo "${FTL_VERSION:=N/A}";; esac } @@ -32,7 +32,7 @@ getLocalVersion() { getLocalHash() { case ${1} in "Pi-hole" ) echo "${CORE_HASH:=N/A}";; - "AdminLTE" ) echo "${WEB_HASH:=N/A}";; + "web" ) echo "${WEB_HASH:=N/A}";; "FTL" ) echo "${FTL_HASH:=N/A}";; esac } @@ -40,7 +40,7 @@ getLocalHash() { getRemoteHash(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_HASH:=N/A}";; - "AdminLTE" ) echo "${GITHUB_WEB_HASH:=N/A}";; + "web" ) echo "${GITHUB_WEB_HASH:=N/A}";; "FTL" ) echo "${GITHUB_FTL_HASH:=N/A}";; esac } @@ -48,7 +48,7 @@ getRemoteHash(){ getRemoteVersion(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_VERSION:=N/A}";; - "AdminLTE" ) echo "${GITHUB_WEB_VERSION:=N/A}";; + "web" ) echo "${GITHUB_WEB_VERSION:=N/A}";; "FTL" ) echo "${GITHUB_FTL_VERSION:=N/A}";; esac } @@ -56,7 +56,7 @@ getRemoteVersion(){ getLocalBranch(){ case ${1} in "Pi-hole" ) echo "${CORE_BRANCH:=N/A}";; - "AdminLTE" ) echo "${WEB_BRANCH:=N/A}";; + "web" ) echo "${WEB_BRANCH:=N/A}";; "FTL" ) echo "${FTL_BRANCH:=N/A}";; esac } @@ -107,7 +107,7 @@ errorOutput() { defaultOutput() { versionOutput "Pi-hole" "$@" - versionOutput "AdminLTE" "$@" + versionOutput "web" "$@" versionOutput "FTL" "$@" } @@ -118,7 +118,7 @@ Show Pi-hole, Admin Console & FTL versions Repositories: -p, --pihole Only retrieve info regarding Pi-hole repository - -a, --admin Only retrieve info regarding AdminLTE repository + -a, --admin Only retrieve info regarding web repository -f, --ftl Only retrieve info regarding FTL repository Options: @@ -131,7 +131,7 @@ Options: case "${1}" in "-p" | "--pihole" ) shift; versionOutput "Pi-hole" "$@";; - "-a" | "--admin" ) shift; versionOutput "AdminLTE" "$@";; + "-a" | "--admin" ) shift; versionOutput "web" "$@";; "-f" | "--ftl" ) shift; versionOutput "FTL" "$@";; "-h" | "--help" ) helpFunc;; * ) defaultOutput "$@";; diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4e4bdfc7..ef2c8d52 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -64,11 +64,11 @@ webroot="/var/www/html" # We clone (or update) two git repositories during the install. This helps to make sure that we always have the latest versions of the relevant files. -# AdminLTE is used to set up the Web admin interface. +# web is used to set up the Web admin interface. # Pi-hole contains various setup scripts and files which are critical to the installation. # Search for "PI_HOLE_LOCAL_REPO" in this file to see all such scripts. # Two notable scripts are gravity.sh (used to generate the HOSTS file) and advanced/Scripts/webpage.sh (used to install the Web admin interface) -webInterfaceGitUrl="https://github.com/pi-hole/AdminLTE.git" +webInterfaceGitUrl="https://github.com/pi-hole/web.git" webInterfaceDir="${webroot}/admin" piholeGitUrl="https://github.com/pi-hole/pi-hole.git" PI_HOLE_LOCAL_REPO="/etc/.pihole" diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 1cf8ab35..fec1fa5e 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -212,7 +212,7 @@ Available commands and options: .br -p, --pihole Only retrieve info regarding Pi-hole repository .br - -a, --admin Only retrieve info regarding AdminLTE + -a, --admin Only retrieve info regarding web repository .br -f, --ftl Only retrieve info regarding FTL repository @@ -339,7 +339,7 @@ Displaying version information \fBpihole -v -a -c\fR .br - Display the current version of AdminLTE + Display the current version of web .br Temporarily disabling Pi-hole From 6001fe34ec3d3ad652a7013f467d1948efb3e096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 8 Oct 2023 23:03:33 +0200 Subject: [PATCH 096/413] Set owner of gravity output files to pihole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/gravity.sh b/gravity.sh index 9133d33d..4f87c1d7 100755 --- a/gravity.sh +++ b/gravity.sh @@ -488,6 +488,10 @@ compareLists() { # We assume here it was changed upstream database_adlist_status "${adlistID}" "1" fi + + # set owner of the file to pihole + chown pihole:pihole "${target}.sha1" + } # Download specified URL and perform checks on HTTP status and file content @@ -621,6 +625,9 @@ gravity_DownloadBlocklistFromUrl() { database_adlist_status "${adlistID}" "4" fi fi + + # set owner of the file to pihole + chown pihole:pihole "${saveLocation}" } # Parse source files into domains format From 679aab10d0400d55470a5d53215158490736068a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 9 Oct 2023 21:52:43 +0200 Subject: [PATCH 097/413] Run gravity as user pihole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- gravity.sh | 7 ------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ef2c8d52..ae605fcf 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1385,8 +1385,8 @@ installCron() { # Gravity is a very important script as it aggregates all of the domains into a single HOSTS formatted list, # which is what Pi-hole needs to begin blocking ads runGravity() { - # Run gravity in the current shell - { /opt/pihole/gravity.sh --force; } + # Run gravity in the current shell as user pihole + { exec sudo -u pihole bash /opt/pihole/gravity.sh --force; } } # Check if the pihole user exists and create if it does not diff --git a/gravity.sh b/gravity.sh index 4f87c1d7..9133d33d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -488,10 +488,6 @@ compareLists() { # We assume here it was changed upstream database_adlist_status "${adlistID}" "1" fi - - # set owner of the file to pihole - chown pihole:pihole "${target}.sha1" - } # Download specified URL and perform checks on HTTP status and file content @@ -625,9 +621,6 @@ gravity_DownloadBlocklistFromUrl() { database_adlist_status "${adlistID}" "4" fi fi - - # set owner of the file to pihole - chown pihole:pihole "${saveLocation}" } # Parse source files into domains format From 22863845a0018708aaf8b215cd00666e87fceebf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 9 Oct 2023 22:05:01 +0200 Subject: [PATCH 098/413] Set owner/group of /etc/pihole to pihole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ae605fcf..4a938dc7 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2141,8 +2141,8 @@ main() { if [[ "${useUpdateVars}" == false ]]; then # Display welcome dialogs welcomeDialogs - # Create directory for Pi-hole storage - install -d -m 755 /etc/pihole/ + # Create directory for Pi-hole storage (/etc/pihole/) + install -o pihole -g pihole -d -m 660 "${PI_HOLE_CONFIG_DIR}" # Determine available interfaces get_available_interfaces # Find interfaces and let the user choose one From 2061f3a70e7f0e64709e330552ba8e85671080ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 9 Oct 2023 22:35:02 +0200 Subject: [PATCH 099/413] Set owner/permissions at the right place MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4a938dc7..efe1d4f3 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1085,12 +1085,15 @@ installScripts() { installConfigs() { printf "\\n %b Installing configs from %s...\\n" "${INFO}" "${PI_HOLE_LOCAL_REPO}" + # Ensure that permissions are correctly set + chown -R pihole:pihole /etc/pihole # Install list of DNS servers # Format: Name;Primary IPv4;Secondary IPv4;Primary IPv6;Secondary IPv6 # Some values may be empty (for example: DNS servers without IPv6 support) echo "${DNS_SERVERS}" > "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" chmod 644 "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" + chown pihole:pihole "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" # Install empty custom.list file if it does not exist if [[ ! -r "${PI_HOLE_CONFIG_DIR}/custom.list" ]]; then @@ -1386,7 +1389,7 @@ installCron() { # which is what Pi-hole needs to begin blocking ads runGravity() { # Run gravity in the current shell as user pihole - { exec sudo -u pihole bash /opt/pihole/gravity.sh --force; } + { sudo -u pihole bash /opt/pihole/gravity.sh --force; } } # Check if the pihole user exists and create if it does not @@ -1480,7 +1483,7 @@ installLogrotate() { return 2 fi # Copy the file over from the local repo - install -D -m 644 -T "${PI_HOLE_LOCAL_REPO}"/advanced/Templates/logrotate ${target} + install -o pihole -g pihole -D -m 644 -T "${PI_HOLE_LOCAL_REPO}"/advanced/Templates/logrotate ${target} # Different operating systems have different user / group # settings for logrotate that makes it impossible to create # a static logrotate file that will work with e.g. @@ -2049,6 +2052,7 @@ copy_to_install_log() { # Since we use color codes such as '\e[1;33m', they should be removed sed 's/\[[0-9;]\{1,5\}m//g' < /proc/$$/fd/3 > "${installLogLoc}" chmod 644 "${installLogLoc}" + chown pihole:pihole "${installLogLoc}" } main() { @@ -2142,7 +2146,7 @@ main() { # Display welcome dialogs welcomeDialogs # Create directory for Pi-hole storage (/etc/pihole/) - install -o pihole -g pihole -d -m 660 "${PI_HOLE_CONFIG_DIR}" + install -d -m 755 "${PI_HOLE_CONFIG_DIR}" # Determine available interfaces get_available_interfaces # Find interfaces and let the user choose one From 46ff25734485c9f960da0c4312a6c5ebc5e0f8a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 12 Oct 2023 13:44:51 +0200 Subject: [PATCH 100/413] Remove Chronometer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/chronometer.sh | 569 ----------------------------- advanced/bash-completion/pihole | 8 +- automated install/basic-install.sh | 2 +- manpages/pihole.8 | 14 - pihole | 6 +- 5 files changed, 5 insertions(+), 594 deletions(-) delete mode 100755 advanced/Scripts/chronometer.sh diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh deleted file mode 100755 index fc728e17..00000000 --- a/advanced/Scripts/chronometer.sh +++ /dev/null @@ -1,569 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC1090,SC1091 -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Calculates stats and displays to an LCD -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. -LC_ALL=C -LC_NUMERIC=C - -# Retrieve stats from FTL engine -pihole-FTL() { - local ftl_port LINE - # shellcheck disable=SC1091 - . /opt/pihole/utils.sh - ftl_port=$(getFTLConfigValue dns.port) - if [[ -n "$ftl_port" ]]; then - # Open connection to FTL - exec 3<>"/dev/tcp/127.0.0.1/$ftl_port" - - # Test if connection is open - if { "true" >&3; } 2> /dev/null; then - # Send command to FTL and ask to quit when finished - echo -e ">$1 >quit" >&3 - - # Read input until we received an empty string and the connection is - # closed - read -r -t 1 LINE <&3 - until [[ -z "${LINE}" ]] && [[ ! -t 3 ]]; do - echo "$LINE" >&1 - read -r -t 1 LINE <&3 - done - - # Close connection - exec 3>&- - exec 3<&- - fi - else - echo "0" - fi -} - -# Print spaces to align right-side additional text -printFunc() { - local text_last - - title="$1" - title_len="${#title}" - - text_main="$2" - text_main_nocol="$text_main" - if [[ "${text_main:0:1}" == "" ]]; then - text_main_nocol=$(sed 's/\[[0-9;]\{1,5\}m//g' <<< "$text_main") - fi - text_main_len="${#text_main_nocol}" - - text_addn="$3" - if [[ "$text_addn" == "last" ]]; then - text_addn="" - text_last="true" - fi - - # If there is additional text, define max length of text_main - if [[ -n "$text_addn" ]]; then - case "$scr_cols" in - [0-9]|1[0-9]|2[0-9]|3[0-9]|4[0-4]) text_main_max_len="9";; - 4[5-9]) text_main_max_len="14";; - *) text_main_max_len="19";; - esac - fi - - [[ -z "$text_addn" ]] && text_main_max_len="$(( scr_cols - title_len ))" - - # Remove excess characters from main text - if [[ "$text_main_len" -gt "$text_main_max_len" ]]; then - # Trim text without colors - text_main_trim="${text_main_nocol:0:$text_main_max_len}" - # Replace with trimmed text - text_main="${text_main/$text_main_nocol/$text_main_trim}" - fi - - # Determine amount of spaces for each line - if [[ -n "$text_last" ]]; then - # Move cursor to end of screen - spc_num=$(( scr_cols - ( title_len + text_main_len ) )) - else - spc_num=$(( text_main_max_len - text_main_len )) - fi - - [[ "$spc_num" -le 0 ]] && spc_num="0" - spc=$(printf "%${spc_num}s") - #spc="${spc// /.}" # Debug: Visualize spaces - - printf "%s%s$spc" "$title" "$text_main" - - if [[ -n "$text_addn" ]]; then - printf "%s(%s)%s\\n" "$COL_NC$COL_DARK_GRAY" "$text_addn" "$COL_NC" - else - # Do not print trailing newline on final line - [[ -z "$text_last" ]] && printf "%s\\n" "$COL_NC" - fi -} - -# Perform on first Chrono run (not for JSON formatted string) -get_init_stats() { - calcFunc(){ awk "BEGIN {print $*}" 2> /dev/null; } - - # Convert bytes to human-readable format - hrBytes() { - awk '{ - num=$1; - if(num==0) { - print "0 B" - } else { - xxx=(num<0?-num:num) - sss=(num<0?-1:1) - split("B KB MB GB TB PB",type) - for(i=5;yyy < 1;i--) { - yyy=xxx / (2^(10*i)) - } - printf "%.0f " type[i+2], yyy*sss - } - }' <<< "$1"; - } - - # Convert seconds to human-readable format - hrSecs() { - day=$(( $1/60/60/24 )); hrs=$(( $1/3600%24 )) - mins=$(( ($1%3600)/60 )); secs=$(( $1%60 )) - [[ "$day" -ge "2" ]] && plu="s" - [[ "$day" -ge "1" ]] && days="$day day${plu}, " || days="" - printf "%s%02d:%02d:%02d\\n" "$days" "$hrs" "$mins" "$secs" - } - - # Set Color Codes - coltable="/opt/pihole/COL_TABLE" - if [[ -f "${coltable}" ]]; then - source ${coltable} - else - COL_NC="" - COL_DARK_GRAY="" - COL_LIGHT_GREEN="" - COL_LIGHT_BLUE="" - COL_LIGHT_RED="" - COL_YELLOW="" - COL_LIGHT_RED="" - COL_URG_RED="" - fi - - # Get RPi throttle state (RPi 3B only) & model number, or OS distro info - if command -v vcgencmd &> /dev/null; then - local sys_throttle_raw - local sys_rev_raw - - sys_throttle_raw=$(vgt=$(sudo vcgencmd get_throttled); echo "${vgt##*x}") - - # Active Throttle Notice: https://bit.ly/2gnunOo - if [[ "$sys_throttle_raw" != "0" ]]; then - case "$sys_throttle_raw" in - *0001) thr_type="${COL_YELLOW}Under Voltage";; - *0002) thr_type="${COL_LIGHT_BLUE}Arm Freq Cap";; - *0003) thr_type="${COL_YELLOW}UV${COL_DARK_GRAY},${COL_NC} ${COL_LIGHT_BLUE}AFC";; - *0004) thr_type="${COL_LIGHT_RED}Throttled";; - *0005) thr_type="${COL_YELLOW}UV${COL_DARK_GRAY},${COL_NC} ${COL_LIGHT_RED}TT";; - *0006) thr_type="${COL_LIGHT_BLUE}AFC${COL_DARK_GRAY},${COL_NC} ${COL_LIGHT_RED}TT";; - *0007) thr_type="${COL_YELLOW}UV${COL_DARK_GRAY},${COL_NC} ${COL_LIGHT_BLUE}AFC${COL_DARK_GRAY},${COL_NC} ${COL_LIGHT_RED}TT";; - esac - [[ -n "$thr_type" ]] && sys_throttle="$thr_type${COL_DARK_GRAY}" - fi - - sys_rev_raw=$(awk '/Revision/ {print $3}' < /proc/cpuinfo) - case "$sys_rev_raw" in - 000[2-6]) sys_model=" 1, Model B";; # 256MB - 000[7-9]) sys_model=" 1, Model A";; # 256MB - 000d|000e|000f) sys_model=" 1, Model B";; # 512MB - 0010|0013) sys_model=" 1, Model B+";; # 512MB - 0012|0015) sys_model=" 1, Model A+";; # 256MB - a0104[0-1]|a21041|a22042) sys_model=" 2, Model B";; # 1GB - 900021) sys_model=" 1, Model A+";; # 512MB - 900032) sys_model=" 1, Model B+";; # 512MB - 90009[2-3]|920093) sys_model=" Zero";; # 512MB - 9000c1) sys_model=" Zero W";; # 512MB - a02082|a[2-3]2082) sys_model=" 3, Model B";; # 1GB - a020d3) sys_model=" 3, Model B+";; # 1GB - *) sys_model="";; - esac - sys_type="Raspberry Pi$sys_model" - else - source "/etc/os-release" - CODENAME=$(sed 's/[()]//g' <<< "${VERSION/* /}") - sys_type="${NAME/ */} ${CODENAME^} $VERSION_ID" - fi - - # Get core count - sys_cores=$(grep -c "^processor" /proc/cpuinfo) - - # Test existence of clock speed file for ARM CPU - if [[ -f "/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq" ]]; then - scaling_freq_file="/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq" - fi - - # Test existence of temperature file - if [[ -f "/sys/class/thermal/thermal_zone0/temp" ]]; then - temp_file="/sys/class/thermal/thermal_zone0/temp" - elif [[ -f "/sys/class/hwmon/hwmon0/temp1_input" ]]; then - temp_file="/sys/class/hwmon/hwmon0/temp1_input" - else - temp_file="" - fi -} - -get_sys_stats() { - local ph_ver_raw - local cpu_raw - local ram_raw - local disk_raw - - # Update every 12 refreshes (Def: every 60s) - count=$((count+1)) - if [[ "$count" == "1" ]] || (( "$count" % 12 == 0 )); then - mapfile -t ph_ver_raw < <(pihole -v -c 2> /dev/null | sed -n 's/^.* v/v/p') - if [[ -n "${ph_ver_raw[0]}" ]]; then - ph_core_ver="${ph_ver_raw[0]}" - if [[ ${#ph_ver_raw[@]} -eq 2 ]]; then - # web not installed - ph_lte_ver="(not installed)" - ph_ftl_ver="${ph_ver_raw[1]}" - else - ph_lte_ver="${ph_ver_raw[1]}" - ph_ftl_ver="${ph_ver_raw[2]}" - fi - else - ph_core_ver="-1" - fi - - sys_name=$(hostname) - - [[ -n "$TEMPERATUREUNIT" ]] && temp_unit="${TEMPERATUREUNIT^^}" || temp_unit="C" - - # Get storage stats for partition mounted on / - read -r -a disk_raw <<< "$(df -B1 / 2> /dev/null | awk 'END{ print $3,$2,$5 }')" - disk_used="${disk_raw[0]}" - disk_total="${disk_raw[1]}" - disk_perc="${disk_raw[2]}" - - net_gateway=$(ip route | grep default | cut -d ' ' -f 3 | head -n 1) - - # Get DHCP stats, if feature is enabled - if [[ "$DHCP_ACTIVE" == "true" ]]; then - ph_dhcp_max=$(( ${DHCP_END##*.} - ${DHCP_START##*.} + 1 )) - fi - - # Get DNS server count - dns_count="0" - [[ -n "${PIHOLE_DNS_1}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_2}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_3}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_4}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_5}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_6}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_7}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_8}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_9}" ]] && dns_count="$dns_count+" - fi - - # Get screen size - read -r -a scr_size <<< "$(stty size 2>/dev/null || echo 24 80)" - scr_lines="${scr_size[0]}" - scr_cols="${scr_size[1]}" - - # Determine Chronometer size behavior - if [[ "$scr_cols" -ge 58 ]]; then - chrono_width="large" - elif [[ "$scr_cols" -gt 40 ]]; then - chrono_width="medium" - else - chrono_width="small" - fi - - # Determine max length of divider string - scr_line_len=$(( scr_cols - 2 )) - [[ "$scr_line_len" -ge 58 ]] && scr_line_len="58" - scr_line_str=$(printf "%${scr_line_len}s") - scr_line_str="${scr_line_str// /—}" - - sys_uptime=$(hrSecs "$(cut -d. -f1 /proc/uptime)") - sys_loadavg=$(cut -d " " -f1,2,3 /proc/loadavg) - - # Get CPU usage, only counting processes over 1% as active - # shellcheck disable=SC2009 - cpu_raw=$(ps -eo pcpu,rss --no-headers | grep -E -v " 0") - cpu_tasks=$(wc -l <<< "$cpu_raw") - cpu_taskact=$(sed -r "/(^ 0.)/d" <<< "$cpu_raw" | wc -l) - cpu_perc=$(awk '{sum+=$1} END {printf "%.0f\n", sum/'"$sys_cores"'}' <<< "$cpu_raw") - - # Get CPU clock speed - if [[ -n "$scaling_freq_file" ]]; then - cpu_mhz=$(( $(< /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq) / 1000 )) - else - cpu_mhz=$(lscpu | awk -F ":" '/MHz/ {print $2;exit}') - cpu_mhz=$(printf "%.0f" "${cpu_mhz//[[:space:]]/}") - fi - - # Determine whether to display CPU clock speed as MHz or GHz - if [[ -n "$cpu_mhz" ]]; then - [[ "$cpu_mhz" -le "999" ]] && cpu_freq="$cpu_mhz MHz" || cpu_freq="$(printf "%.1f" $(calcFunc "$cpu_mhz"/1000)) GHz" - [[ "${cpu_freq}" == *".0"* ]] && cpu_freq="${cpu_freq/.0/}" - fi - - # Determine color for temperature - if [[ -n "$temp_file" ]]; then - if [[ "$temp_unit" == "C" ]]; then - cpu_temp=$(printf "%.0fc\\n" "$(calcFunc "$(< $temp_file) / 1000")") - - case "${cpu_temp::-1}" in - -*|[0-9]|[1-3][0-9]) cpu_col="$COL_LIGHT_BLUE";; - 4[0-9]) cpu_col="";; - 5[0-9]) cpu_col="$COL_YELLOW";; - 6[0-9]) cpu_col="$COL_LIGHT_RED";; - *) cpu_col="$COL_URG_RED";; - esac - - # $COL_NC$COL_DARK_GRAY is needed for $COL_URG_RED - cpu_temp_str=" @ $cpu_col$cpu_temp$COL_NC$COL_DARK_GRAY" - - elif [[ "$temp_unit" == "F" ]]; then - cpu_temp=$(printf "%.0ff\\n" "$(calcFunc "($(< $temp_file) / 1000) * 9 / 5 + 32")") - - case "${cpu_temp::-1}" in - -*|[0-9]|[0-9][0-9]) cpu_col="$COL_LIGHT_BLUE";; - 1[0-1][0-9]) cpu_col="";; - 1[2-3][0-9]) cpu_col="$COL_YELLOW";; - 1[4-5][0-9]) cpu_col="$COL_LIGHT_RED";; - *) cpu_col="$COL_URG_RED";; - esac - - cpu_temp_str=" @ $cpu_col$cpu_temp$COL_NC$COL_DARK_GRAY" - - else - cpu_temp_str=$(printf " @ %.0fk\\n" "$(calcFunc "($(< $temp_file) / 1000) + 273.15")") - fi - else - cpu_temp_str="" - fi - - read -r -a ram_raw <<< "$(awk '/MemTotal:/{total=$2} /MemFree:/{free=$2} /Buffers:/{buffers=$2} /^Cached:/{cached=$2} END {printf "%.0f %.0f %.0f", (total-free-buffers-cached)*100/total, (total-free-buffers-cached)*1024, total*1024}' /proc/meminfo)" - ram_perc="${ram_raw[0]}" - ram_used="${ram_raw[1]}" - ram_total="${ram_raw[2]}" - - if [[ "$(pihole status web 2> /dev/null)" -ge "1" ]]; then - ph_status="${COL_LIGHT_GREEN}Active" - else - ph_status="${COL_LIGHT_RED}Offline" - fi - - if [[ "$DHCP_ACTIVE" == "true" ]]; then - local ph_dhcp_range - - ph_dhcp_range=$(seq -s "|" -f "${DHCP_START%.*}.%g" "${DHCP_START##*.}" "${DHCP_END##*.}") - - # Count dynamic leases from available range, and not static leases - ph_dhcp_num=$(grep -cE "$ph_dhcp_range" "/etc/pihole/dhcp.leases") - ph_dhcp_percent=$(( ph_dhcp_num * 100 / ph_dhcp_max )) - fi -} - -get_ftl_stats() { - local stats_raw - - mapfile -t stats_raw < <(pihole-FTL "stats") - domains_being_blocked_raw="${stats_raw[0]#* }" - dns_queries_today_raw="${stats_raw[1]#* }" - ads_blocked_today_raw="${stats_raw[2]#* }" - ads_percentage_today_raw="${stats_raw[3]#* }" - queries_forwarded_raw="${stats_raw[5]#* }" - queries_cached_raw="${stats_raw[6]#* }" - - # Only retrieve these stats when not called from jsonFunc - if [[ -z "$1" ]]; then - local top_ad_raw - local top_domain_raw - local top_client_raw - - domains_being_blocked=$(printf "%.0f\\n" "${domains_being_blocked_raw}" 2> /dev/null) - dns_queries_today=$(printf "%.0f\\n" "${dns_queries_today_raw}") - ads_blocked_today=$(printf "%.0f\\n" "${ads_blocked_today_raw}") - ads_percentage_today=$(printf "%'.0f\\n" "${ads_percentage_today_raw}") - queries_cached_percentage=$(printf "%.0f\\n" "$(calcFunc "$queries_cached_raw * 100 / ( $queries_forwarded_raw + $queries_cached_raw )")") - recent_blocked=$(pihole-FTL recentBlocked) - read -r -a top_ad_raw <<< "$(pihole-FTL "top-ads (1)")" - read -r -a top_domain_raw <<< "$(pihole-FTL "top-domains (1)")" - read -r -a top_client_raw <<< "$(pihole-FTL "top-clients (1)")" - - top_ad="${top_ad_raw[2]}" - top_domain="${top_domain_raw[2]}" - if [[ "${top_client_raw[3]}" ]]; then - top_client="${top_client_raw[3]}" - else - top_client="${top_client_raw[2]}" - fi - fi -} - -get_strings() { - # Expand or contract strings depending on screen size - if [[ "$chrono_width" == "large" ]]; then - phc_str=" ${COL_DARK_GRAY}Core" - lte_str=" ${COL_DARK_GRAY}Web" - ftl_str=" ${COL_DARK_GRAY}FTL" - api_str="${COL_LIGHT_RED}API Offline" - - host_info="$sys_type" - sys_info="$sys_throttle" - sys_info2="Active: $cpu_taskact of $cpu_tasks tasks" - used_str="Used: " - leased_str="Leased: " - domains_being_blocked=$(printf "%'.0f" "$domains_being_blocked") - ads_blocked_today=$(printf "%'.0f" "$ads_blocked_today") - dns_queries_today=$(printf "%'.0f" "$dns_queries_today") - ph_info="Blocking: $domains_being_blocked sites" - total_str="Total: " - else - phc_str=" ${COL_DARK_GRAY}Core" - lte_str=" ${COL_DARK_GRAY}Web" - ftl_str=" ${COL_DARK_GRAY}FTL" - api_str="${COL_LIGHT_RED}API Down" - ph_info="$domains_being_blocked blocked" - fi - - [[ "$sys_cores" -ne 1 ]] && sys_cores_txt="${sys_cores}x " - cpu_info="$sys_cores_txt$cpu_freq$cpu_temp_str" - ram_info="$used_str$(hrBytes "$ram_used") of $(hrBytes "$ram_total")" - disk_info="$used_str$(hrBytes "$disk_used") of $(hrBytes "$disk_total")" - - lan_info="Gateway: $net_gateway" - dhcp_info="$leased_str$ph_dhcp_num of $ph_dhcp_max" - - ads_info="$total_str$ads_blocked_today of $dns_queries_today" - dns_info="$dns_count DNS servers" - - [[ "$recent_blocked" == "0" ]] && recent_blocked="${COL_LIGHT_RED}FTL offline${COL_NC}" -} - -chronoFunc() { - local extra_arg="$1" - local extra_value="$2" - - get_init_stats - - for (( ; ; )); do - get_sys_stats - get_ftl_stats - get_strings - - # Strip excess development version numbers - if [[ "$ph_core_ver" != "-1" ]]; then - phc_ver_str="$phc_str: ${ph_core_ver%-*}${COL_NC}" - lte_ver_str="$lte_str: ${ph_lte_ver%-*}${COL_NC}" - ftl_ver_str="$ftl_str: ${ph_ftl_ver%-*}${COL_NC}" - else - phc_ver_str="$phc_str: $api_str${COL_NC}" - fi - - # Get refresh number - if [[ "${extra_arg}" = "refresh" ]]; then - num="${extra_value}" - num_str="Refresh set for every $num seconds" - else - num_str="" - fi - - clear - - # Remove exit message heading on third refresh - if [[ "$count" -le 2 ]] && [[ "${extra_arg}" != "exit" ]]; then - echo -e " ${COL_LIGHT_GREEN}Pi-hole Chronometer${COL_NC} - $num_str - ${COL_LIGHT_RED}Press Ctrl-C to exit${COL_NC} - ${COL_DARK_GRAY}$scr_line_str${COL_NC}" - else - echo -e "|¯¯¯(¯)_|¯|_ ___|¯|___$phc_ver_str\\n| ¯_/¯|_| ' \\/ _ \\ / -_)$lte_ver_str\\n|_| |_| |_||_\\___/_\\___|$ftl_ver_str\\n ${COL_DARK_GRAY}$scr_line_str${COL_NC}" - fi - - printFunc " Hostname: " "$sys_name" "$host_info" - printFunc " Uptime: " "$sys_uptime" "$sys_info" - printFunc " Task Load: " "$sys_loadavg" "$sys_info2" - printFunc " CPU usage: " "$cpu_perc%" "$cpu_info" - printFunc " RAM usage: " "$ram_perc%" "$ram_info" - printFunc " HDD usage: " "$disk_perc" "$disk_info" - - if [[ "$DHCP_ACTIVE" == "true" ]]; then - printFunc "DHCP usage: " "$ph_dhcp_percent%" "$dhcp_info" - fi - - printFunc " Pi-hole: " "$ph_status" "$ph_info" - printFunc " Blocked: " "$ads_percentage_today%" "$ads_info" - printFunc "Local Qrys: " "$queries_cached_percentage%" "$dns_info" - - printFunc "Last Block: " "$recent_blocked" - printFunc " Top Block: " "$top_ad" - - # Provide more stats on screens with more lines - if [[ "$scr_lines" -eq 17 ]]; then - if [[ "$DHCP_ACTIVE" == "true" ]]; then - printFunc "Top Domain: " "$top_domain" "last" - else - print_client="true" - fi - else - print_client="true" - fi - - if [[ -n "$print_client" ]]; then - printFunc "Top Domain: " "$top_domain" - printFunc "Top Client: " "$top_client" "last" - fi - - # Handle exit/refresh options - if [[ "${extra_arg}" == "exit" ]]; then - exit 0 - else - if [[ "${extra_arg}" == "refresh" ]]; then - sleep "$num" - else - sleep 5 - fi - fi - - done -} - -jsonFunc() { - get_ftl_stats "json" - echo "{\"domains_being_blocked\":${domains_being_blocked_raw},\"dns_queries_today\":${dns_queries_today_raw},\"ads_blocked_today\":${ads_blocked_today_raw},\"ads_percentage_today\":${ads_percentage_today_raw}}" -} - -helpFunc() { - if [[ "$1" == "?" ]]; then - echo "Unknown option. Please view 'pihole -c --help' for more information" - else - echo "Usage: pihole -c [options] -Example: 'pihole -c -j' -Calculates stats and displays to an LCD - -Options: - -j, --json Output stats as JSON formatted string - -r, --refresh Set update frequency (in seconds) - -e, --exit Output stats and exit without refreshing - -h, --help Display this help text" - fi - - exit 0 -} - -if [[ $# = 0 ]]; then - chronoFunc -fi - -case "$1" in - "-j" | "--json" ) jsonFunc;; - "-h" | "--help" ) helpFunc;; - "-r" | "--refresh" ) chronoFunc refresh "$2";; - "-e" | "--exit" ) chronoFunc exit;; - * ) helpFunc "?";; -esac diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 29a3270d..305a3f5b 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -1,5 +1,5 @@ _pihole() { - local cur prev opts opts_admin opts_checkout opts_chronometer opts_debug opts_interface opts_logging opts_privacy opts_query opts_update opts_version + local cur prev opts opts_admin opts_checkout opts_debug opts_interface opts_logging opts_privacy opts_query opts_update opts_version COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" @@ -7,7 +7,7 @@ _pihole() { case "${prev}" in "pihole") - opts="admin blacklist checkout chronometer debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" + opts="admin blacklist checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) ;; "whitelist"|"blacklist"|"wildcard"|"regex") @@ -22,10 +22,6 @@ _pihole() { opts_checkout="core ftl web master dev" COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) ;; - "chronometer") - opts_chronometer="\--exit \--json \--refresh" - COMPREPLY=( $(compgen -W "${opts_chronometer}" -- ${cur}) ) - ;; "debug") opts_debug="-a" COMPREPLY=( $(compgen -W "${opts_debug}" -- ${cur}) ) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ef2c8d52..cc187499 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -73,7 +73,7 @@ webInterfaceDir="${webroot}/admin" piholeGitUrl="https://github.com/pi-hole/pi-hole.git" PI_HOLE_LOCAL_REPO="/etc/.pihole" # List of pihole scripts, stored in an array -PI_HOLE_FILES=(chronometer list piholeDebug piholeLogFlush setupLCD update version gravity uninstall webpage) +PI_HOLE_FILES=(list piholeDebug piholeLogFlush setupLCD update version gravity uninstall webpage) # This directory is where the Pi-hole scripts will be installed PI_HOLE_INSTALL_DIR="/opt/pihole" PI_HOLE_CONFIG_DIR="/etc/pihole" diff --git a/manpages/pihole.8 b/manpages/pihole.8 index fec1fa5e..55bbe6cb 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -141,20 +141,6 @@ Available commands and options: (0 = lowest, 3 = highest) .br -\fB-c, chronometer\fR [options] -.br - Calculates stats and displays to an LCD -.br - - (Chronometer Options): -.br - -j, --json Output stats as JSON formatted string -.br - -r, --refresh Set update frequency (in seconds) -.br - -e, --exit Output stats and exit without refreshing -.br - \fB-g, updateGravity\fR .br Update the list of ad-serving domains diff --git a/pihole b/pihole index 5281c715..817bfd01 100755 --- a/pihole +++ b/pihole @@ -125,8 +125,7 @@ queryFunc() { } chronometerFunc() { - shift - "${PI_HOLE_SCRIPT_DIR}"/chronometer.sh "$@" + echo "Chronometer is gone, use PADD (https://github.com/pi-hole/PADD)" exit 0 } @@ -497,8 +496,7 @@ Debugging Options: Options: setpassword set the password for the web interface - -c, chronometer Calculates stats and displays to an LCD - Add '-h' for more info on chronometer usage + -c, chronometer Chronometer is gone, use PADD -g, updateGravity Update the list of ad-serving domains -h, --help, help Show this help dialog -l, logging Specify whether the Pi-hole log should be used From 188b2b858acb907c2a856bbe032db833db9df75e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 12 Oct 2023 13:48:01 +0200 Subject: [PATCH 101/413] PADD is fine, really! MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .codespellignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.codespellignore b/.codespellignore index 501a3d67..0755931d 100644 --- a/.codespellignore +++ b/.codespellignore @@ -1,3 +1,4 @@ doubleclick wan nwe +padd From 40c75289b5c88bea75bf3803729a1ecf49b4b2d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 13 Oct 2023 19:59:29 +0200 Subject: [PATCH 102/413] Allow pihole to access subdirs in /etc/pihole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index 17900f0b..abeaabc4 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -9,20 +9,27 @@ utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" # Get file paths FTL_PID_FILE="$(getFTLPIDFile)" -# Touch files to ensure they exist (create if non-existing, preserve if existing) -# shellcheck disable=SC2174 -mkdir -pm 0755 /var/log/pihole -[ -f "${FTL_PID_FILE}" ] || install -D -m 644 -o pihole -g pihole /dev/null "${FTL_PID_FILE}" -[ -f /var/log/pihole/FTL.log ] || install -m 644 -o pihole -g pihole /dev/null /var/log/pihole/FTL.log -[ -f /var/log/pihole/pihole.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/pihole.log -[ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit all necessary files +# shellcheck disable=SC2174 +mkdir -pm 0640 /var/log/pihole chown -R pihole:pihole /etc/pihole /var/log/pihole chmod -R 0640 /var/log/pihole chmod -R 0660 /etc/pihole + # allow all users to enter the directories chmod 0755 /etc/pihole /var/log/pihole +# allow pihole to access subdirs in /etc/pihole (sets execution bit on dirs) +# credits https://stackoverflow.com/a/11512211 +find /etc/pihole -type d -exec chmod 0755 {} \; + +# Touch files to ensure they exist (create if non-existing, preserve if existing) +[ -f "${FTL_PID_FILE}" ] || install -D -m 644 -o pihole -g pihole /dev/null "${FTL_PID_FILE}" +[ -f /var/log/pihole/FTL.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/FTL.log +[ -f /var/log/pihole/pihole.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/pihole.log +[ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases + + # Backward compatibility for user-scripts that still expect log files in /var/log instead of /var/log/pihole # Should be removed with Pi-hole v6.0 if [ ! -f /var/log/pihole.log ]; then From f7ba059b94215160731d4e30fbc239c375c104ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 14 Oct 2023 14:01:20 +0200 Subject: [PATCH 103/413] Query directly and authenticate only if required MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/api.sh | 21 ++++++++++++++++++--- advanced/Scripts/query.sh | 15 +++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index afd88671..449f146f 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -82,10 +82,25 @@ DeleteSession() { } GetFTLData() { - local data + local data response status # get the data from querying the API as well as the http status code - data=$(curl -s -X GET "http://localhost:${PORT}/api$1" -H "Accept: application/json" -H "sid: ${SID}" ) - echo "${data}" + response=$(curl -s -w "%{http_code}" -X GET "http://localhost:${PORT}/api$1" -H "Accept: application/json" -H "sid: ${SID}" ) + + # status are the last 3 characters + status=$(printf %s "${response#"${response%???}"}") + # data is everything from response without the last 3 characters + data=$(printf %s "${response%???}") + + if [ "${status}" = 200 ]; then + # response OK + echo "${data}" + elif [ "${status}" = 000 ]; then + # connection lost + echo "000" + elif [ "${status}" = 401 ]; then + # unauthorized + echo "401" + fi } secretRead() { diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 8407d3f6..a26d249c 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -118,10 +118,21 @@ Main(){ # Test if the authentication endpoint is available TestAPIAvailability - # Authenticate with the FTL server - Authenthication + + # Users can configure FTL in a way, that for accessing a) all endpoints (webserver.api.localAPIauth) + # or b) for the /search endpoint (webserver.api.searchAPIauth) no authentication is required. + # Therefore, we try to query directly without authentication but do authenticat if 401 is returned data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}") + + if [ "${data}" = 401 ]; then + # Unauthenticated, so authenticate with the FTL server required + Authenthication + + # send query again + data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}") + fi + GenerateOutput "${data}" DeleteSession } From ec83d6b7931dc3493efb535ce8c66586bec53d7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 14 Oct 2023 14:06:47 +0200 Subject: [PATCH 104/413] No help for gone functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- pihole | 1 - 1 file changed, 1 deletion(-) diff --git a/pihole b/pihole index 817bfd01..7be13a6f 100755 --- a/pihole +++ b/pihole @@ -496,7 +496,6 @@ Debugging Options: Options: setpassword set the password for the web interface - -c, chronometer Chronometer is gone, use PADD -g, updateGravity Update the list of ad-serving domains -h, --help, help Show this help dialog -l, logging Specify whether the Pi-hole log should be used From 2005d04625c48948c1010563dac3a060f114f311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 14 Oct 2023 21:50:57 +0200 Subject: [PATCH 105/413] Exact search is the new default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/query.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index a26d249c..27a33f39 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -17,7 +17,7 @@ # Globals PI_HOLE_INSTALL_DIR="/opt/pihole" max_results="20" -partial="true" +partial="false" domain="" # Source color table @@ -29,13 +29,13 @@ colfile="/opt/pihole/COL_TABLE" Help(){ echo "Usage: pihole -q [option] -Example: 'pihole -q --exact domain.com' +Example: 'pihole -q --partial domain.com' Query the adlists for a specified domain Options: - --exact Search the adlists for exact domain matches + --partial Search the adlists for partially matching domains --all Return all query matches within the adlists - -h, --help Show this help dialog" + -h, --help Show this help dialog" exit 0 } @@ -141,7 +141,7 @@ Main(){ while [ "$#" -gt 0 ]; do case "$1" in "-h" | "--help" ) Help;; - "--exact" ) partial="false";; + "--partial" ) partial="true";; "--all" ) max_results=10000;; # hard-coded FTL limit * ) domain=$1;; esac From 123ba1f154845ad9b549f3f81638c3ef340dbda6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 15 Oct 2023 22:15:11 +0200 Subject: [PATCH 106/413] Remove temp dir created when downloading FTL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ac516415..4dd31976 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1777,7 +1777,8 @@ FTLinstall() { # Move into the temp ftl directory pushd "$(mktemp -d)" > /dev/null || { printf "Unable to make temporary directory for FTL binary download\\n"; return 1; } - + local tempdir + tempdir="$(pwd)" local ftlBranch local url @@ -1819,12 +1820,19 @@ FTLinstall() { # Installed the FTL service printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" + + # Remove temp dir + remove_dir "${tempdir}" + return 0 else # Otherwise, the hash download failed, so print and exit. popd > /dev/null || { printf "Unable to return to original directory after FTL binary download.\\n"; return 1; } printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" printf " %b Error: Download of %s/%s failed (checksum error)%b\\n" "${COL_LIGHT_RED}" "${url}" "${binary}" "${COL_NC}" + + # Remove temp dir + remove_dir "${tempdir}" return 1 fi else @@ -1833,10 +1841,19 @@ FTLinstall() { printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" # The URL could not be found printf " %b Error: URL %s/%s not found%b\\n" "${COL_LIGHT_RED}" "${url}" "${binary}" "${COL_NC}" + + # Remove temp dir + remove_dir "${tempdir}" return 1 fi } +remove_dir() { + # Delete dir + rm -r "${1}" > /dev/null 2>&1 || \ + echo -e " ${CROSS} Unable to remove ${1}" +} + get_binary_name() { # Get the OS architecture (we cannot use uname -m as this may return an incorrect architecture when buildx-compiling with QEMU for arm) local machine From 27522fbc18dc53cd0828778d5b7a8a2dfd72ff6d Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Mon, 16 Oct 2023 17:19:44 -0300 Subject: [PATCH 107/413] Add a final message to gravity The terminal version doesn't really need a final message, but this will be read by the web interface to show a success message Signed-off-by: RD WebDesign --- gravity.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gravity.sh b/gravity.sh index 9133d33d..e61ade42 100755 --- a/gravity.sh +++ b/gravity.sh @@ -901,4 +901,6 @@ gravity_ShowCount gravity_Cleanup echo "" +echo " ${TICK} Done." + # "${PIHOLE_COMMAND}" status From 2c7fa4a7b36aa4ae440ea5500438f72e56bb0f91 Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Wed, 18 Oct 2023 02:07:06 -0300 Subject: [PATCH 108/413] Avoid printing getFTLConfigValue return in statusFunc() Signed-off-by: RD WebDesign --- pihole | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pihole b/pihole index 7be13a6f..9cc1069f 100755 --- a/pihole +++ b/pihole @@ -345,7 +345,7 @@ analyze_ports() { statusFunc() { # Determine if there is pihole-FTL service is listening - local pid port ftl_pid_file + local pid port ftl_pid_file block_status ftl_pid_file="$(getFTLPIDFile)" @@ -375,7 +375,8 @@ statusFunc() { fi # Determine if Pi-hole's blocking is enabled - if getFTLConfigValue dns.blocking.active; then + block_status=$(getFTLConfigValue dns.blocking.active) + if [ ${block_status} == "true" ]; then case "${1}" in "web") echo "$port";; *) echo -e " ${TICK} Pi-hole blocking is enabled";; From d77dbf736cbeb1f5bc7e885e7ccb0b8cfc9c1bdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 18 Oct 2023 22:09:38 +0200 Subject: [PATCH 109/413] Logrotate config file needs to be owned by root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 4 ++++ automated install/basic-install.sh | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index abeaabc4..c6817828 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -16,6 +16,10 @@ chown -R pihole:pihole /etc/pihole /var/log/pihole chmod -R 0640 /var/log/pihole chmod -R 0660 /etc/pihole +# Logrotate config file need to be owned by root and must not be writable by group and others +chown root:root /etc/pihole/logrotate +chmod 0644 /etc/pihole/logrotate + # allow all users to enter the directories chmod 0755 /etc/pihole /var/log/pihole diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ac516415..4d656283 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1483,7 +1483,8 @@ installLogrotate() { return 2 fi # Copy the file over from the local repo - install -o pihole -g pihole -D -m 644 -T "${PI_HOLE_LOCAL_REPO}"/advanced/Templates/logrotate ${target} + # Logrotate config file must be owned by root and not writable by group or other + install -o root -g root -D -m 644 -T "${PI_HOLE_LOCAL_REPO}"/advanced/Templates/logrotate ${target} # Different operating systems have different user / group # settings for logrotate that makes it impossible to create # a static logrotate file that will work with e.g. From c571d8d37d9b92336024fdf8e5fe98cfa2aee517 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 19 Oct 2023 22:04:11 +0200 Subject: [PATCH 110/413] Drop support for ancient ARMv4 and ARMv5, the Debian binaries turn out to cause unresolvable SIGFPE on ARMv6 and lower. Fortunately, we can still use the Alpine binaries on all Raspberry Pi models (even the oldest ones) Signed-off-by: DL6ER --- automated install/basic-install.sh | 26 ++--- test/test_any_automated_install.py | 154 +---------------------------- 2 files changed, 10 insertions(+), 170 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4d656283..b6806141 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1870,9 +1870,9 @@ get_binary_name() { printf "%b %b Detected AArch64 (64 Bit ARM) architecture\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-arm64" - elif [[ "${cpu_arch}" == "armv6KZ" ]]; then - printf "%b %b Detected ARMv6KZ architecture\\n" "${OVER}" "${TICK}" - # set the binary to be used + elif [[ "${cpu_arch}" == "armv6"* ]]; then + printf "%b %b Detected ARMv6 architecture\\n" "${OVER}" "${TICK}" + # set the binary to be used (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) l_binary="pihole-FTL-armv6" else # If ARMv8 or higher is found (e.g., BCM2837 as found in Raspberry Pi Model 3B) @@ -1885,24 +1885,10 @@ get_binary_name() { printf "%b %b Detected ARMv7 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" # set the binary to be used l_binary="pihole-FTL-armv6" - elif [[ "${cpu_arch}" == "v5TE" || "${rev}" -gt 5 ]]; then - # Check if the system is using GLIBC 2.29 or higher - if [[ -n "${l_glibc_version}" && "$(printf '%s\n' "2.29" "${l_glibc_version}" | sort -V | head -n1)" == "2.29" ]]; then - # If so, use the ARMv6 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv6 architecture (running GLIBC 2.29 or higher, %s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" - # set the binary to be used - l_binary="pihole-FTL-armv5" - else - # Otherwise, use the ARMv5 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv6 architecture (running GLIBC older than 2.29, %s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" - # set the binary to be used - l_binary="pihole-FTL-armv4" - fi else - # Otherwise, use the ARMv4 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv4 or ARMv5 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" - # set the binary to be used - l_binary="pihole-FTL-armv4" + # Otherwise, Pi-hole does not support this architecture + printf "%b %b This processor architecture is not supported by Pi-hole (%s)\\n" "${OVER}" "${CROSS}" "${cpu_arch}" + l_binary="" fi fi elif [[ "${machine}" == "x86_64" ]]; then diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index a8e0e082..1980f0b6 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -259,158 +259,12 @@ def test_FTL_detect_aarch64_no_errors(host): assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv4_no_errors(host): - """ - confirms only armv4 package is downloaded for FTL engine - """ - # mock uname to return armv4 platform - mock_command("uname", {"-m": ("armv4t", "0")}, host) - # mock readelf to respond with armv4 CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv4t", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv4t", "0"), - }, - host, - ) - detectPlatform = host.run( - """ - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture (armv4t)" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv5_no_errors(host): - """ - confirms only armv5 package is downloaded for FTL engine - """ - # mock uname to return armv5te platform - mock_command("uname", {"-m": ("armv5te", "0")}, host) - # mock readelf to respond with armv5 CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv5te", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv5te", "0"), - }, - host, - ) - detectPlatform = host.run( - """ - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture (armv5te)" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv6_old_no_errors(host): +def test_FTL_detect_armv6_no_errors(host): """ confirms only armv6 package is downloaded for FTL engine """ - # mock uname to return armv6l platform - mock_command("uname", {"-m": ("armv6l", "0")}, host) - # mock readelf to respond with armv6l CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv6l", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv6l", "0"), - }, - host, - ) - # Mock old ldd GLIBC version - mock_command( - "ldd", {"--version": ("ldd (Debian GLIBC 2.13-38+deb7u8) 2.13", "0")}, host - ) - - detectPlatform = host.run( - """ - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + ( - " Detected ARMv6 architecture (running GLIBC older than 2.29, armv6l)" - ) - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv6_recent_no_errors(host): - """ - confirms only armv6 package is downloaded for FTL engine - """ - # mock uname to return armv6l platform - mock_command("uname", {"-m": ("armv6l", "0")}, host) - # mock readelf to respond with armv6l CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv6l", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv6l", "0"), - }, - host, - ) - # Mock old ldd GLIBC version - mock_command( - "ldd", {"--version": ("'ldd (Debian GLIBC 2.35-38+deb7u8) 2.35'", "0")}, host - ) - - detectPlatform = host.run( - """ - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + ( - " Detected ARMv6 architecture (running GLIBC 2.29 or higher, armv6l)" - ) - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv6KZ_no_errors(host): - """ - confirms only armv6KZ package is downloaded for FTL engine - """ - # mock uname to return armv6KZ platform - mock_command("uname", {"-m": ("armv6KZ", "0")}, host) + # mock uname to return armv6 platform + mock_command("uname", {"-m": ("armv6", "0")}, host) # mock readelf to respond with armv6l CPU architecture mock_command_2( "readelf", @@ -432,7 +286,7 @@ def test_FTL_detect_armv6KZ_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv6KZ architecture" + expected_stdout = tick_box + " Detected ARMv6 architecture" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout From 79ebbacc4af790164ee927443bb53b55d7a87a93 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 21 Oct 2023 10:27:01 +0000 Subject: [PATCH 111/413] Bump actions/checkout from 4.1.0 to 4.1.1 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.0 to 4.1.1. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.0...v4.1.1) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4685aa2c..6544db61 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 0ff0a24a..d9de09d2 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index a1025629..e52d4ae9 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9f32302e..8166d253 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 - name: Check scripts in repository are executable run: | @@ -72,7 +72,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.1 From ca7836bf717c34f97719ba8aa3c3f6b07f0a591d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 21 Oct 2023 19:05:45 +0200 Subject: [PATCH 112/413] Remove now obsolete GLIBC version check Signed-off-by: DL6ER --- automated install/basic-install.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index b9ed04a4..443ee85e 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1860,15 +1860,6 @@ get_binary_name() { local machine machine=$(uname -m) - # Get local GLIBC version (leave at "0.0" if no GLIBC, e.g., on musl) - local l_glibc_version="0.0" - if ldd --version 2>&1 | grep -q "GLIBC"; then - l_glibc_version=$(ldd --version | head -n1 | grep -o '[0-9.]*$') - printf "%b %b Detected GLIBC version %s\\n" "${OVER}" "${TICK}" "${l_glibc_version}" - else - printf "%b %b No GLIBC detected\\n" "${OVER}" "${CROSS}" - fi - local l_binary local str="Detecting processor" From 45687d675becf21f23953625c00efdb637531c21 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 21 Oct 2023 19:13:45 +0200 Subject: [PATCH 113/413] Fix the binary detection for ARMv6 and simplify the router overall Signed-off-by: DL6ER --- automated install/basic-install.sh | 37 ++++++++++++++---------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 443ee85e..054d8eef 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1875,29 +1875,27 @@ get_binary_name() { local rev rev=$(echo "${cpu_arch}" | grep -o '[0-9]*') if [[ "${machine}" == "aarch64" ]]; then + # If AArch64 is found (e.g., BCM2711 in Raspberry Pi 4) printf "%b %b Detected AArch64 (64 Bit ARM) architecture\\n" "${OVER}" "${TICK}" - # set the binary to be used l_binary="pihole-FTL-arm64" - elif [[ "${cpu_arch}" == "armv6"* ]]; then + elif [[ "${cpu_arch}" == "v6"* ]]; then + # If ARMv6 is found (e.g., BCM2835 in Raspberry Pi 1 and Zero) printf "%b %b Detected ARMv6 architecture\\n" "${OVER}" "${TICK}" - # set the binary to be used (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) + l_binary="pihole-FTL-armv6" + elif [[ "${cpu_arch}" == "v7"* || "${rev}" -ge 7 ]]; then + # If ARMv7 or higher is found (e.g., BCM2836 in Raspberry PI 2 Mod. B) + # This path is also used for ARMv8 when the OS is in 32bit mode + # (e.g., BCM2837 in Raspberry Pi Model 3B, or BCM2711 in Raspberry Pi 4) + printf "%b %b Detected ARMv7 (or newer) architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" + l_binary="pihole-FTL-armv7" + elif [[ "${rev}" -gt 6 ]]; then + # Otherwise, if ARMv7 is found (e.g., BCM2836 in Raspberry Pi Model 2) + printf "%b %b Detected ARMv7 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" l_binary="pihole-FTL-armv6" else - # If ARMv8 or higher is found (e.g., BCM2837 as found in Raspberry Pi Model 3B) - if [[ "${cpu_arch}" == "v7" || "${rev}" -gt 7 ]]; then - printf "%b %b Detected ARMv7 (or newer) architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" - # set the binary to be used - l_binary="pihole-FTL-armv7" - elif [[ "${rev}" -gt 6 ]]; then - # Otherwise, if ARMv7 is found (e.g., BCM2836 as found in Raspberry Pi Model 2) - printf "%b %b Detected ARMv7 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" - # set the binary to be used - l_binary="pihole-FTL-armv6" - else - # Otherwise, Pi-hole does not support this architecture - printf "%b %b This processor architecture is not supported by Pi-hole (%s)\\n" "${OVER}" "${CROSS}" "${cpu_arch}" - l_binary="" - fi + # Otherwise, Pi-hole does not support this architecture + printf "%b %b This processor architecture is not supported by Pi-hole (%s)\\n" "${OVER}" "${CROSS}" "${cpu_arch}" + l_binary="" fi elif [[ "${machine}" == "x86_64" ]]; then # This gives the processor of packages dpkg installs (for example, "i386") @@ -1912,9 +1910,8 @@ get_binary_name() { printf "%b %b Detected 32bit (i686) architecture\\n" "${OVER}" "${TICK}" l_binary="pihole-FTL-386" else - # 64bit + # 64bit OS printf "%b %b Detected x86_64 architecture\\n" "${OVER}" "${TICK}" - # set the binary to be used l_binary="pihole-FTL-amd64" fi elif [[ "${machine}" == "riscv64" ]]; then From 30bfc7cc9f44cd9b0138ebc2e7400f4e592fbd42 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 21 Oct 2023 19:15:10 +0200 Subject: [PATCH 114/413] Add binutils tot he OS check dependencies as we need it to check if the local system is abel to run any of our precompiled FTL binaries Signed-off-by: DL6ER --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 054d8eef..019cc3e3 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -302,9 +302,9 @@ package_manager_detect() { # Update package cache update_package_cache || exit 1 # Packages required to perform the os_check (stored as an array) - OS_CHECK_DEPS=(grep dnsutils) + OS_CHECK_DEPS=(grep dnsutils binutils) # Packages required to run this install script (stored as an array) - INSTALLER_DEPS=(git iproute2 dialog ca-certificates binutils) + INSTALLER_DEPS=(git iproute2 dialog ca-certificates) # Packages required to run Pi-hole (stored as an array) PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) From 38ecc1693f50764209b8a0eff289064eef0ed881 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 21 Oct 2023 19:28:59 +0200 Subject: [PATCH 115/413] Further simplify the ARMv6 test Signed-off-by: DL6ER --- automated install/basic-install.sh | 8 ++------ test/test_any_automated_install.py | 8 ++++---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 019cc3e3..2d47a548 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1878,20 +1878,16 @@ get_binary_name() { # If AArch64 is found (e.g., BCM2711 in Raspberry Pi 4) printf "%b %b Detected AArch64 (64 Bit ARM) architecture\\n" "${OVER}" "${TICK}" l_binary="pihole-FTL-arm64" - elif [[ "${cpu_arch}" == "v6"* ]]; then + elif [[ "${rev}" -eq 6 ]]; then # If ARMv6 is found (e.g., BCM2835 in Raspberry Pi 1 and Zero) printf "%b %b Detected ARMv6 architecture\\n" "${OVER}" "${TICK}" l_binary="pihole-FTL-armv6" - elif [[ "${cpu_arch}" == "v7"* || "${rev}" -ge 7 ]]; then + elif [[ "${rev}" -ge 7 ]]; then # If ARMv7 or higher is found (e.g., BCM2836 in Raspberry PI 2 Mod. B) # This path is also used for ARMv8 when the OS is in 32bit mode # (e.g., BCM2837 in Raspberry Pi Model 3B, or BCM2711 in Raspberry Pi 4) printf "%b %b Detected ARMv7 (or newer) architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" l_binary="pihole-FTL-armv7" - elif [[ "${rev}" -gt 6 ]]; then - # Otherwise, if ARMv7 is found (e.g., BCM2836 in Raspberry Pi Model 2) - printf "%b %b Detected ARMv7 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" - l_binary="pihole-FTL-armv6" else # Otherwise, Pi-hole does not support this architecture printf "%b %b This processor architecture is not supported by Pi-hole (%s)\\n" "${OVER}" "${CROSS}" "${cpu_arch}" diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 1980f0b6..7d9d15f6 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -269,8 +269,8 @@ def test_FTL_detect_armv6_no_errors(host): mock_command_2( "readelf", { - "-A /bin/sh": ("Tag_CPU_arch: armv6KZ", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv6KZ", "0"), + "-A /bin/sh": ("Tag_CPU_arch: armv6", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv6", "0"), }, host, ) @@ -319,7 +319,7 @@ def test_FTL_detect_armv7l_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv7 architecture (armv7l)") + expected_stdout = tick_box + (" Detected ARMv7 (or newer) architecture") assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -352,7 +352,7 @@ def test_FTL_detect_armv7_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv7 architecture (armv7)") + expected_stdout = tick_box + (" Detected ARMv7 (or newer) architecture") assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout From 369ccf13a846bb64ac7b6d8a454f6bda4cf2a89a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 21 Oct 2023 21:40:21 +0200 Subject: [PATCH 116/413] Move FTL binary availability check after the supported OS check Signed-off-by: DL6ER --- automated install/basic-install.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 2d47a548..1ecde7a9 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2088,16 +2088,6 @@ main() { fi fi - # Check if there is a usable FTL binary available on this architecture - do - # this early on as FTL is a hard dependency for Pi-hole - local funcOutput - funcOutput=$(get_binary_name) #Store output of get_binary_name here - # Abort early if this processor is not supported (get_binary_name returns empty string) - if [[ "${funcOutput}" == "" ]]; then - printf " %b Upgrade/install aborted\\n" "${CROSS}" "${DISTRO_NAME}" - exit 1 - fi - # Check if SELinux is Enforcing and exit before doing anything else checkSelinux @@ -2114,6 +2104,16 @@ main() { # Check that the installed OS is officially supported - display warning if not os_check + # Check if there is a usable FTL binary available on this architecture - do + # this early on as FTL is a hard dependency for Pi-hole + local funcOutput + funcOutput=$(get_binary_name) #Store output of get_binary_name here + # Abort early if this processor is not supported (get_binary_name returns empty string) + if [[ "${funcOutput}" == "" ]]; then + printf " %b Upgrade/install aborted\\n" "${CROSS}" "${DISTRO_NAME}" + exit 1 + fi + # Install packages used by this installation script printf " %b Checking for / installing Required dependencies for this install script...\\n" "${INFO}" install_dependent_packages "${INSTALLER_DEPS[@]}" From 7523c49f624a57f5b0ae87e39328813bf5b72f83 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 22 Oct 2023 08:14:11 +0200 Subject: [PATCH 117/413] Swapping the databases must be the last step before the cleanup. Otherwise, FTL may be reloading from an only partially completed database causing spurious errors and/or warnings. Signed-off-by: DL6ER --- gravity.sh | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/gravity.sh b/gravity.sh index e61ade42..7b7234ee 100755 --- a/gravity.sh +++ b/gravity.sh @@ -73,9 +73,9 @@ generate_gravity_database() { chmod g+w "${piholeDir}" "${gravityDBfile}" } -# Copy data from old to new database file and swap them -gravity_swap_databases() { - local str copyGravity oldAvail +# Build gravity tree +gravity_build_tree() { + local str str="Building tree" echo -ne " ${INFO} ${str}..." @@ -88,7 +88,10 @@ gravity_swap_databases() { return 1 fi echo -e "${OVER} ${TICK} ${str}" +} +# Copy data from old to new database file and swap them +gravity_swap_databases() { str="Swapping databases" echo -ne " ${INFO} ${str}..." @@ -116,11 +119,11 @@ gravity_swap_databases() { # Update timestamp when the gravity table was last updated successfully update_gravity_timestamp() { - output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to update gravity timestamp in database ${gravityDBfile}\\n ${output}" + echo -e "\\n ${CROSS} Unable to update gravity timestamp in database ${gravityTEMPfile}\\n ${output}" return 1 fi return 0 @@ -659,12 +662,12 @@ gravity_Table_Count() { local table="${1}" local str="${2}" local num - num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table};")" + num="$(pihole-FTL sqlite3 "${gravityTEMPfile}" "SELECT COUNT(*) FROM ${table};")" if [[ "${table}" == "gravity" ]]; then local unique - unique="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" + unique="$(pihole-FTL sqlite3 "${gravityTEMPfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" echo -e " ${INFO} Number of ${str}: ${num} (${COL_BOLD}${unique} unique domains${COL_NC})" - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" + pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" else echo -e " ${INFO} Number of ${str}: ${num}" fi @@ -882,22 +885,27 @@ fi # Create local.list gravity_generateLocalList -# Migrate rest of the data from old to new database -if ! gravity_swap_databases; then - echo -e " ${CROSS} Unable to create database. Please contact support." - exit 1 -fi - # Update gravity timestamp update_gravity_timestamp # Ensure proper permissions are set for the database -chown pihole:pihole "${gravityDBfile}" -chmod g+w "${piholeDir}" "${gravityDBfile}" +chown pihole:pihole "${gravityTEMPfile}" +chmod g+w "${piholeDir}" "${gravityTEMPfile}" -# Compute numbers to be displayed +# Build the tree +gravity_build_tree + +# Compute numbers to be displayed (do this after building the tree to get the +# numbers quickly from the tree instead of having to scan the whole database) gravity_ShowCount +# Migrate rest of the data from old to new database +# IMPORTANT: Swapping the databases must be the last step before the cleanup +if ! gravity_swap_databases; then + echo -e " ${CROSS} Unable to create database. Please contact support." + exit 1 +fi + gravity_Cleanup echo "" From 842a9d7778e5de72f99d39b4fe78ed8523c94eb1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 22 Oct 2023 08:32:52 +0200 Subject: [PATCH 118/413] Address review comments Signed-off-by: DL6ER --- automated install/basic-install.sh | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 1ecde7a9..87bf1ea4 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -90,7 +90,6 @@ IPV6_ADDRESS=${IPV6_ADDRESS} QUERY_LOGGING=true WEBPORT=8080 PRIVACY_LEVEL=0 -CACHE_SIZE=10000 if [ -z "${USER}" ]; then USER="$(id -un)" @@ -301,11 +300,11 @@ package_manager_detect() { PKG_COUNT="${PKG_MANAGER} -s -o Debug::NoLocking=true upgrade | grep -c ^Inst || true" # Update package cache update_package_cache || exit 1 - # Packages required to perform the os_check (stored as an array) + # Packages required to perform the os_check and FTL binary detection OS_CHECK_DEPS=(grep dnsutils binutils) - # Packages required to run this install script (stored as an array) + # Packages required to run this install script INSTALLER_DEPS=(git iproute2 dialog ca-certificates) - # Packages required to run Pi-hole (stored as an array) + # Packages required to run Pi-hole PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) # If apt-get is not found, check for rpm. @@ -1856,29 +1855,30 @@ remove_dir() { } get_binary_name() { - # Get the OS architecture (we cannot use uname -m as this may return an incorrect architecture when buildx-compiling with QEMU for arm) + local l_binary local machine machine=$(uname -m) - local l_binary - local str="Detecting processor" printf " %b %s..." "${INFO}" "${str}" - # If the machine is arm or aarch - if [[ "${machine}" == "arm"* || "${machine}" == *"aarch"* ]]; then - # ARM + + # If the machine is aarch64 (armv8) + if [[ "${machine}" == "aarch64" ]]; then + # If AArch64 is found (e.g., BCM2711 in Raspberry Pi 4) + printf "%b %b Detected AArch64 (64 Bit ARM) architecture\\n" "${OVER}" "${TICK}" + l_binary="pihole-FTL-arm64" + elif [[ "${machine}" == "arm"* ]]; then + # ARM 32 bit # Get supported processor from other binaries installed on the system + # We cannot really rely on the output of $(uname -m) above as this may + # return an incorrect architecture when buildx-compiling with QEMU local cpu_arch cpu_arch=$(readelf -A "$(command -v sh)" | grep Tag_CPU_arch | awk '{ print $2 }') # Get the revision from the CPU architecture local rev rev=$(echo "${cpu_arch}" | grep -o '[0-9]*') - if [[ "${machine}" == "aarch64" ]]; then - # If AArch64 is found (e.g., BCM2711 in Raspberry Pi 4) - printf "%b %b Detected AArch64 (64 Bit ARM) architecture\\n" "${OVER}" "${TICK}" - l_binary="pihole-FTL-arm64" - elif [[ "${rev}" -eq 6 ]]; then + if [[ "${rev}" -eq 6 ]]; then # If ARMv6 is found (e.g., BCM2835 in Raspberry Pi 1 and Zero) printf "%b %b Detected ARMv6 architecture\\n" "${OVER}" "${TICK}" l_binary="pihole-FTL-armv6" From fd8fdd3513f7d41afddfe1877d11261c7e4ac0b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 23 Oct 2023 21:36:18 +0200 Subject: [PATCH 119/413] Use suffixed temp file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gravity.sh b/gravity.sh index 7b7234ee..edfe89a9 100755 --- a/gravity.sh +++ b/gravity.sh @@ -141,6 +141,7 @@ database_table_from_file() { # implementations of mktemp support it, e.g. on Alpine tmpFile="$(mktemp -p "${GRAVITY_TMPDIR}")" mv "${tmpFile}" "${tmpFile%.*}.gravity" + tmpFile="${tmpFile%.*}.gravity" local timestamp timestamp="$(date --utc +'%s')" @@ -502,6 +503,7 @@ gravity_DownloadBlocklistFromUrl() { # We don't use '--suffix' here because not all implementations of mktemp support it, e.g. on Alpine listCurlBuffer="$(mktemp -p "${GRAVITY_TMPDIR}")" mv "${listCurlBuffer}" "${listCurlBuffer%.*}.phgpb" + listCurlBuffer="${listCurlBuffer%.*}.phgpb" # Determine if $saveLocation has read permission if [[ -r "${saveLocation}" && $url != "file"* ]]; then From 00d7e998b462949fa0091a4b55ad82f1572b07c4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:03:09 +0200 Subject: [PATCH 120/413] setupVars.conf and pihole-FTL.conf are no more - use pihole.toml instead Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 36 ++++++++++----------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 0e3bbf3d..977bce35 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -77,7 +77,7 @@ PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" -PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole-FTL.conf" +PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml" PIHOLE_CUSTOM_HOSTS_FILE="${PIHOLE_DIRECTORY}/custom.list" PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" @@ -138,7 +138,6 @@ REQUIRED_FILES=("${PIHOLE_CRON_FILE}" "${PIHOLE_RAW_BLOCKLIST_FILES}" "${PIHOLE_LOCAL_HOSTS_FILE}" "${PIHOLE_LOGROTATE_FILE}" -"${PIHOLE_SETUP_VARS_FILE}" "${PIHOLE_FTL_CONF_FILE}" "${PIHOLE_COMMAND}" "${PIHOLE_COLTABLE_FILE}" @@ -165,20 +164,6 @@ show_disclaimer(){ log_write "${DISCLAIMER}" } -source_setup_variables() { - # Display the current test that is running - log_write "\\n${COL_PURPLE}*** [ INITIALIZING ]${COL_NC} Sourcing setup variables" - # If the variable file exists, - if ls "${PIHOLE_SETUP_VARS_FILE}" 1> /dev/null 2>&1; then - log_write "${INFO} Sourcing ${PIHOLE_SETUP_VARS_FILE}..."; - # source it - source ${PIHOLE_SETUP_VARS_FILE} - else - # If it can't, show an error - log_write "${PIHOLE_SETUP_VARS_FILE} ${COL_RED}does not exist or cannot be read.${COL_NC}" - fi -} - make_temporary_log() { # Create a random temporary file for the log TEMPLOG=$(mktemp /tmp/pihole_temp.XXXXXX) @@ -546,15 +531,15 @@ disk_usage() { done } -parse_setup_vars() { - echo_current_diagnostic "Setup variables" +parse_pihole_toml() { + echo_current_diagnostic "Pi-hole configuration" # If the file exists, - if [[ -r "${PIHOLE_SETUP_VARS_FILE}" ]]; then + if [[ -r "${PIHOLE_FTL_CONF_FILE}" ]]; then # parse it - parse_file "${PIHOLE_SETUP_VARS_FILE}" + parse_file "${PIHOLE_FTL_CONF_FILE}" else # If not, show an error - log_write "${CROSS} ${COL_RED}Could not read ${PIHOLE_SETUP_VARS_FILE}.${COL_NC}" + log_write "${CROSS} ${COL_RED}Could not read ${PIHOLE_FTL_CONF_FILE}.${COL_NC}" fi } @@ -1010,8 +995,10 @@ parse_file() { # For each line in the file, for file_lines in "${file_info[@]}"; do if [[ -n "${file_lines}" ]]; then - # don't include the Web password hash - [[ "${file_lines}" =~ ^\#.*$ || ! "${file_lines}" || "${file_lines}" == "WEBPASSWORD="* ]] && continue + # skip empty and comment lines line + [[ "${file_lines}" =~ ^[[:space:]]*\#.*$ || ! "${file_lines}" ]] && continue + # remove the password hash from the output (*"pwhash = "*) + [[ "${file_lines}" == *"pwhash ="* ]] && file_lines=$(echo "${file_lines}" | sed -e 's/\(pwhash = \).*/\1/') # otherwise, display the lines of the file log_write " ${file_lines}" fi @@ -1076,7 +1063,6 @@ list_files_in_dir() { elif [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_DEBUG_LOG}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_RAW_BLOCKLIST_FILES}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_INSTALL_LOG_FILE}" ]] || \ - [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_SETUP_VARS_FILE}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_LOG}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_WEB_SERVER_ACCESS_LOG_FILE}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_LOG_GZIPS}" ]]; then @@ -1484,7 +1470,7 @@ check_dhcp_servers process_status ftl_full_status lighttpd_test_configuration -parse_setup_vars +parse_pihole_toml check_x_headers analyze_ftl_db analyze_gravity_list From ccd8c470a535c5a42085f1e68a91d4b4bff5bf18 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:03:37 +0200 Subject: [PATCH 121/413] check_x_headers is no more Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 977bce35..88723cdc 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -732,35 +732,6 @@ check_networking() { [ -z "${DOCKER_VERSION}" ] && check_required_ports } -# check_x_headers() { -# # The X-Headers allow us to determine from the command line if the Web -# # lighttpd.conf has a directive to show "X-Pi-hole: A black hole for Internet advertisements." -# # in the header of any Pi-holed domain -# # Similarly, it will show "X-Pi-hole: The Pi-hole Web interface is working!" if you view the header returned -# # when accessing the dashboard (i.e curl -I pi.hole/admin/) -# # server is operating correctly -# echo_current_diagnostic "Dashboard headers" -# # Use curl -I to get the header and parse out just the X-Pi-hole one -# local full_curl_output_dashboard -# local dashboard -# full_curl_output_dashboard="$(curl -Is localhost/admin/)" -# dashboard=$(echo "${full_curl_output_dashboard}" | awk '/X-Pi-hole/' | tr -d '\r') -# # Store what the X-Header should be in variables for comparison later -# local dashboard_working -# dashboard_working="X-Pi-hole: The Pi-hole Web interface is working!" - -# # If the X-Header matches what a working system should have, -# if [[ $dashboard == "$dashboard_working" ]]; then -# # then we can show a success -# log_write "$TICK Web interface X-Header: ${COL_GREEN}${dashboard}${COL_NC}" -# else -# # Otherwise, it's a failure since the X-Headers either don't exist or have been modified in some way -# log_write "$CROSS Web interface X-Header: ${COL_RED}X-Header does not match or could not be retrieved.${COL_NC}" - -# log_write "${COL_RED}${full_curl_output_dashboard}${COL_NC}" -# fi -# } - dig_at() { # We need to test if Pi-hole can properly resolve domain names # as it is an essential piece of the software @@ -1471,7 +1442,6 @@ process_status ftl_full_status lighttpd_test_configuration parse_pihole_toml -check_x_headers analyze_ftl_db analyze_gravity_list show_groups From dcc7e96d2f4b963cae8630a6db3e04b7fc7d2802 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:07:11 +0200 Subject: [PATCH 122/413] Remove detection of addresses of PIHOLE_INTERFACE as it is gone, too Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 35 +++------------------------------ 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 88723cdc..517ab4ec 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -550,33 +550,6 @@ parse_locale() { parse_file "${pihole_locale}" } -detect_ip_addresses() { - # First argument should be a 4 or a 6 - local protocol=${1} - # Use ip to show the addresses for the chosen protocol - # Store the values in an array so they can be looped through - # Get the lines that are in the file(s) and store them in an array for parsing later - mapfile -t ip_addr_list < <(ip -"${protocol}" addr show dev "${PIHOLE_INTERFACE}" | awk -F ' ' '{ for(i=1;i<=NF;i++) if ($i ~ '/^inet/') print $(i+1) }') - - # If there is something in the IP address list, - if [[ -n ${ip_addr_list[*]} ]]; then - # Local iterator - local i - # Display the protocol and interface - log_write "${TICK} IPv${protocol} address(es) bound to the ${PIHOLE_INTERFACE} interface:" - # Since there may be more than one IP address, store them in an array - for i in "${!ip_addr_list[@]}"; do - log_write " ${ip_addr_list[$i]}" - done - # Print a blank line just for formatting - log_write "" - else - # If there are no IPs detected, explain that the protocol is not configured - log_write "${CROSS} ${COL_RED}No IPv${protocol} address(es) found on the ${PIHOLE_INTERFACE}${COL_NC} interface.\\n" - return 1 - fi -} - ping_ipv4_or_ipv6() { # Give the first argument a readable name (a 4 or a six should be the argument) local protocol="${1}" @@ -605,9 +578,9 @@ ping_gateway() { while IFS= read -r gateway; do log_write " ${gateway}" - done < <(ip -"${protocol}" route | grep default | grep "${PIHOLE_INTERFACE}" | cut -d ' ' -f 3) + done < <(ip -"${protocol}" route | grep default | cut -d ' ' -f 3) - gateway=$(ip -"${protocol}" route | grep default | grep "${PIHOLE_INTERFACE}" | cut -d ' ' -f 3 | head -n 1) + gateway=$(ip -"${protocol}" route | grep default | cut -d ' ' -f 3 | head -n 1) # If there was at least one gateway if [ -n "${gateway}" ]; then # Let the user know we will ping the gateway for a response @@ -615,7 +588,7 @@ ping_gateway() { # Try to quietly ping the gateway 3 times, with a timeout of 3 seconds, using numeric output only, # on the pihole interface, and tail the last three lines of the output # If pinging the gateway is not successful, - if ! ${cmd} -c 1 -W 2 -n "${gateway}" -I "${PIHOLE_INTERFACE}" >/dev/null; then + if ! ${cmd} -c 1 -W 2 -n "${gateway}" >/dev/null; then # let the user know log_write "${CROSS} ${COL_RED}Gateway did not respond.${COL_NC} ($FAQ_GATEWAY)\\n" # and return an error code @@ -723,8 +696,6 @@ check_networking() { # Runs through several of the functions made earlier; we just clump them # together since they are all related to the networking aspect of things echo_current_diagnostic "Networking" - detect_ip_addresses "4" - detect_ip_addresses "6" ping_gateway "4" ping_gateway "6" # Skip the following check if installed in docker container. Unpriv'ed containers do not have access to the information required From c86e7b3f96ebcd3c99b19a360f2b7410a6847778 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:13:31 +0200 Subject: [PATCH 123/413] Generalize CPU check to a short hardware information output Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 50 ++++++++++++++++----------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 517ab4ec..4c0e3cd0 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -482,34 +482,34 @@ check_firewalld() { fi } -processor_check() { - echo_current_diagnostic "Processor" - # Store the processor type in a variable - PROCESSOR=$(uname -m) - # If it does not contain a value, - if [[ -z "${PROCESSOR}" ]]; then - # we couldn't detect it, so show an error - PROCESSOR=$(lscpu | awk '/Architecture/ {print $2}') - log_write "${CROSS} ${COL_RED}${PROCESSOR}${COL_NC} has not been tested with FTL, but may still work: (${FAQ_FTL_COMPATIBILITY})" +run_and_print_command() { + # Run the command passed as an argument + local cmd="${1}" + # Show the command that is being run + log_write "${INFO} ${cmd}" + # Run the command and store the output in a variable + local output + output=$(${cmd} 2>&1) + # If the command was successful, + if [[ $? -eq 0 ]]; then + # show the output + log_write "${output}" else - # Check if the architecture is currently supported for FTL - case "${PROCESSOR}" in - "amd64" | "x86_64") log_write "${TICK} ${COL_GREEN}${PROCESSOR}${COL_NC}" - ;; - "armv6l") log_write "${TICK} ${COL_GREEN}${PROCESSOR}${COL_NC}" - ;; - "armv6") log_write "${TICK} ${COL_GREEN}${PROCESSOR}${COL_NC}" - ;; - "armv7l") log_write "${TICK} ${COL_GREEN}${PROCESSOR}${COL_NC}" - ;; - "aarch64") log_write "${TICK} ${COL_GREEN}${PROCESSOR}${COL_NC}" - ;; - # Otherwise, show the processor type - *) log_write "${INFO} ${PROCESSOR}"; - esac + # otherwise, show an error + log_write "${CROSS} ${COL_RED}Command failed${COL_NC}" fi } +hardware_check() { + echo_current_diagnostic "System hardware configuration" + # Store the output of the command in a variable + run_and_print_command "lshw -short" + + echo_current_diagnostic "Processor details" + # Store the output of the command in a variable + run_and_print_command "lscpu" +} + disk_usage() { local file_system local hide @@ -1403,7 +1403,7 @@ check_component_versions diagnose_operating_system check_selinux check_firewalld -processor_check +hardware_check disk_usage check_ip_command check_networking From 635b4e952f9c18c1f3ba20280f17b1c1eab20463 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:20:31 +0200 Subject: [PATCH 124/413] Add NOERROR/NXDOMAIN support in the DNS resolution test Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 4c0e3cd0..9225c059 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -781,8 +781,16 @@ dig_at() { if [ -n "${addresses}" ]; then while IFS= read -r local_address ; do # Check if Pi-hole can use itself to block a domain - if local_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${local_address}" +short "${record_type}"); then + if local_dig="$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${local_address}" "${record_type}")"; then # If it can, show success + if [[ "${local_dig}" == *"status: NOERROR"* ]]; then + local_dig="NOERROR" + elif [[ "${local_dig}" == *"status: NXDOMAIN"* ]]; then + local_dig="NXDOMAIN" + else + # Extract the IPv4/6 address from the output + local_dig="$(echo "${local_dig}" | grep -Eo '[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*|([0-9a-f]{0,4}:){1,7}[0-9a-f]{0,4}')" + fi log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} on ${COL_CYAN}${iface}${COL_NC} (${COL_CYAN}${local_address}${COL_NC})" else # Otherwise, show a failure From 15be8eeffad5fd209ff08ca8078bd43a4989274c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:21:08 +0200 Subject: [PATCH 125/413] Remove lighttpd config test Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 9225c059..0e57722e 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -868,20 +868,6 @@ ftl_full_status(){ fi } -lighttpd_test_configuration(){ - # let lighttpd test it's own configuration - local lighttpd_conf_test - echo_current_diagnostic "Lighttpd configuration test" - lighttpd_conf_test=$(lighttpd -tt -f /etc/lighttpd/lighttpd.conf) - if [ -z "${lighttpd_conf_test}" ]; then - # empty output - log_write "${TICK} ${COL_GREEN}No error in lighttpd configuration${COL_NC}" - else - log_write "${CROSS} ${COL_RED}Error in lighttpd configuration${COL_NC}" - log_write " ${lighttpd_conf_test}" - fi -} - make_array_from_file() { local filename="${1}" # The second argument can put a limit on how many line should be read from the file @@ -1419,7 +1405,6 @@ check_name_resolution check_dhcp_servers process_status ftl_full_status -lighttpd_test_configuration parse_pihole_toml analyze_ftl_db analyze_gravity_list From d377cfbc3eb745f112f741f8997d57191f4daa37 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:34:30 +0200 Subject: [PATCH 126/413] Extract the first entry in the answer section from dig's output, replacing any multiple spaces and tabs with a single space Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 0e57722e..53123b9a 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -788,8 +788,9 @@ dig_at() { elif [[ "${local_dig}" == *"status: NXDOMAIN"* ]]; then local_dig="NXDOMAIN" else - # Extract the IPv4/6 address from the output - local_dig="$(echo "${local_dig}" | grep -Eo '[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*|([0-9a-f]{0,4}:){1,7}[0-9a-f]{0,4}')" + # Extract the first entry in the answer section from dig's output, + # replacing any multiple spaces and tabs with a single space + local_dig="$(echo "${local_dig}" | grep -A1 "ANSWER SECTION" | grep -v "ANSWER SECTION" | tr -s " \t" " ")" fi log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} on ${COL_CYAN}${iface}${COL_NC} (${COL_CYAN}${local_address}${COL_NC})" else From 2e9e579bba9122febfa8605452455dc5397f68d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 28 Oct 2023 10:31:54 +0000 Subject: [PATCH 127/413] Bump pytest from 7.4.2 to 7.4.3 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.2 to 7.4.3. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.2...7.4.3) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 27417754..45c7c7c0 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.2 +pytest == 7.4.3 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 tox == 4.11.3 From 6292e65b376ce1d5ad350da2a0b951fc1480f744 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 28 Oct 2023 17:56:37 +0100 Subject: [PATCH 128/413] When setting a blank password, use `webserver.api.password` instead of `webserver.api.pwhash` (fixed in https://github.com/pi-hole/FTL/pull/1702) This prevents the password from being blanked out on the command line when it has been set by an environment variable --- pihole | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pihole b/pihole index 9cc1069f..8e3f1f98 100755 --- a/pihole +++ b/pihole @@ -43,7 +43,7 @@ SetWebPassword() { echo "" if [ "${PASSWORD}" == "" ]; then - setFTLConfigValue "webserver.api.pwhash" "" >/dev/null + setFTLConfigValue "webserver.api.password" "" >/dev/null echo -e " ${TICK} Password Removed" exit 0 fi From ab3b6dfa67b0699a0fa895f2318a7a28f78ddca7 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 28 Oct 2023 19:46:45 +0100 Subject: [PATCH 129/413] No need to >/dev/null because setFTLConfigValue already does this adjust output of setFTLConfigValue to test for the exit code of `pihole-FTL --config` --- advanced/Scripts/utils.sh | 4 ++++ pihole | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 3f9b7031..2fe419e8 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -145,4 +145,8 @@ getFTLConfigValue(){ ####################### setFTLConfigValue(){ pihole-FTL --config "${1}" "${2}" >/dev/null + if [[ $? -eq 5 ]]; then + echo -e " ${CROSS} ${1} set by environment variable. Please unset it to use this function" + exit 5 + fi } diff --git a/pihole b/pihole index 8e3f1f98..279977e4 100755 --- a/pihole +++ b/pihole @@ -43,7 +43,7 @@ SetWebPassword() { echo "" if [ "${PASSWORD}" == "" ]; then - setFTLConfigValue "webserver.api.password" "" >/dev/null + setFTLConfigValue "webserver.api.password" "" echo -e " ${TICK} Password Removed" exit 0 fi @@ -54,7 +54,7 @@ SetWebPassword() { if [ "${PASSWORD}" == "${CONFIRM}" ] ; then # pihole-FTL will automatically hash the password - setFTLConfigValue "webserver.api.password" "${PASSWORD}" >/dev/null + setFTLConfigValue "webserver.api.password" "${PASSWORD}" echo -e " ${TICK} New password set" else echo -e " ${CROSS} Passwords don't match. Your password has not been changed" From 480a8c8d7fd588e3cb35081b6064e60c18643d4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 31 Oct 2023 21:02:26 +0100 Subject: [PATCH 130/413] Remove idn2 as punycode conversion is handled by FTL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/query.sh | 10 ++++------ automated install/basic-install.sh | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 27a33f39..2279df85 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -108,13 +108,11 @@ Main(){ if [ -z "${domain}" ]; then echo "No domain specified"; exit 1 - else - # convert domain to punycode - domain=$(idn2 "${domain}") - - # convert the domain to lowercase - domain=$(echo "${domain}" | tr '[:upper:]' '[:lower:]') fi + # domains are lowercased and converted to punycode by FTL since + # https://github.com/pi-hole/FTL/pull/1715 + # no need to do it here + # Test if the authentication endpoint is available TestAPIAvailability diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 87bf1ea4..ef052dbd 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -305,7 +305,7 @@ package_manager_detect() { # Packages required to run this install script INSTALLER_DEPS=(git iproute2 dialog ca-certificates) # Packages required to run Pi-hole - PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) + PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) # If apt-get is not found, check for rpm. elif is_command rpm ; then @@ -322,7 +322,7 @@ package_manager_detect() { PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src|.riscv64)' | wc -l || true" OS_CHECK_DEPS=(grep bind-utils) INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates binutils) - PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc libcap nmap-ncat jq) + PIHOLE_DEPS=(cronie curl findutils sudo unzip psmisc libcap nmap-ncat jq) # If neither apt-get or yum/dnf package managers were found else From ced1c5d50b5e5beedefed9f2747a21b82b60e833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 17 Dec 2022 16:53:01 +0100 Subject: [PATCH 131/413] Start counting at postion 1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/updatecheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 8bb1888b..66f1a7ab 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -31,7 +31,7 @@ function get_remote_version() { function get_remote_hash(){ - git ls-remote "https://github.com/pi-hole/${1}" --tags "${2}" | awk '{print substr($0, 0,8);}' || return 1 + git ls-remote "https://github.com/pi-hole/${1}" --tags "${2}" | awk '{print substr($0, 1,8);}' || return 1 } # Source the utils file for addOrEditKeyValPair() From bc96d3b0a9635202f56ef42be6ea1ef100a5d27c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 5 Nov 2023 21:15:13 +0100 Subject: [PATCH 132/413] Tweak help text of pihole setpassword MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- pihole | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pihole b/pihole index 279977e4..47da4ddd 100755 --- a/pihole +++ b/pihole @@ -496,7 +496,9 @@ Debugging Options: Options: - setpassword set the password for the web interface + setpassword [pwd] Set the password for the web interface + Without optional argument, password is read interactively. + When specifying a password directly, enclose it in single quotes. -g, updateGravity Update the list of ad-serving domains -h, --help, help Show this help dialog -l, logging Specify whether the Pi-hole log should be used From 72c972175dcb485fefc2de424c4df6eeea0c4d14 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 6 Nov 2023 13:53:08 +0100 Subject: [PATCH 133/413] Remove left-over parts of setupVars processing and only include pihole.toml once Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index c53eada4..8c2388d8 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -544,18 +544,6 @@ disk_usage() { done } -parse_pihole_toml() { - echo_current_diagnostic "Pi-hole configuration" - # If the file exists, - if [[ -r "${PIHOLE_FTL_CONF_FILE}" ]]; then - # parse it - parse_file "${PIHOLE_FTL_CONF_FILE}" - else - # If not, show an error - log_write "${CROSS} ${COL_RED}Could not read ${PIHOLE_FTL_CONF_FILE}.${COL_NC}" - fi -} - parse_locale() { local pihole_locale echo_current_diagnostic "Locale" @@ -1402,10 +1390,6 @@ upload_to_tricorder() { # Run through all the functions we made make_temporary_log initialize_debug -# TODO: Address the reliance on setupVars.conf here. Should debug read pihole.toml directly, or rely on pihole-FTL --config? -# setupVars.conf needs to be sourced before the networking so the values are -# available to the other functions -source_setup_variables check_component_versions # check_critical_program_versions diagnose_operating_system @@ -1419,7 +1403,6 @@ check_name_resolution check_dhcp_servers process_status ftl_full_status -parse_pihole_toml analyze_ftl_db analyze_gravity_list show_groups From 0b4131189d1eb83974967d818334838b41b8be9e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 6 Nov 2023 14:55:34 +0100 Subject: [PATCH 134/413] Migrate Pi-hole created files out of /etc/dnsmasq.d into a pihole owned directory Signed-off-by: DL6ER --- automated install/basic-install.sh | 34 ++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ef5add0e..704a736d 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -91,6 +91,9 @@ QUERY_LOGGING=true WEBPORT=8080 PRIVACY_LEVEL=0 +# Where old configs go to if a v6 migration is performed +V6_CONF_MIGRATION_DIR="/etc/pihole/migration_backup_v6" + if [ -z "${USER}" ]; then USER="$(id -un)" fi @@ -2070,6 +2073,34 @@ copy_to_install_log() { chown pihole:pihole "${installLogLoc}" } +migrate_dnsmasq_configs() { + # Previously, Pi-hole created a number of files in /etc/dnsmasq.d + # During migration, their content is copied into the new single source of + # truth file /etc/pihole/pihole.toml and the old files are moved away to + # avoid conflicts with other services on this system + + # Exit early if this is already Pi-hole v6.0 + # We decide this on the presence of the file /etc/pihole/pihole.toml + if [[ -f /etc/pihole/pihole.toml ]]; then + return 0 + fi + + # Create target directory /etc/pihole/migration_backup_v6 + # and make it owned by pihole:pihole + mkdir -p "${V6_CONF_MIGRATION_DIR} + chown pihole:pihole "${V6_CONF_MIGRATION_DIR} + + # Move all conf files originally created by Pi-hole into this directory + # - 01-pihole.conf + # - 02-pihole-dhcp.conf + # - 04-pihole-static-dhcp.conf + # - 05-pihole-custom-cname.conf + # - 06-rfc6761.conf + + mv /etc/dnsmasq.d/0{1,2,4,5}-pihole*.conf "${V6_MIGRATION_DIR}/ CONF_2>/dev/null || true + mv /etc/dnsmasq.d/06-rfc6761.conf "${V6_MIGRATION_DIR}/ CONF_2>/dev/null || true +} + main() { ######## FIRST CHECK ######## # Must be root to install @@ -2219,6 +2250,9 @@ main() { pihole -a -p "${pw}" fi + # Migrate existing install to v6.0 + migrate_dnsmasq_configs + # Check for and disable systemd-resolved-DNSStubListener before reloading resolved # DNSStubListener needs to remain in place for installer to download needed files, # so this change needs to be made after installation is complete, From de6e61705e7d93d9c0f07ba74f343f43220770de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 6 Nov 2023 21:40:32 +0100 Subject: [PATCH 135/413] Remove local.list and openVPN traces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 1 - gravity.sh | 25 +------------------------ 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 8c2388d8..55659690 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -75,7 +75,6 @@ PIHOLE_CRON_FILE="${CRON_D_DIRECTORY}/pihole" PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" -PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml" PIHOLE_CUSTOM_HOSTS_FILE="${PIHOLE_DIRECTORY}/custom.list" diff --git a/gravity.sh b/gravity.sh index edfe89a9..5c944454 100755 --- a/gravity.sh +++ b/gravity.sh @@ -36,9 +36,6 @@ blacklistFile="${piholeDir}/blacklist.txt" regexFile="${piholeDir}/regex.list" adListFile="${piholeDir}/adlists.list" -localList="${piholeDir}/local.list" -VPNList="/etc/openvpn/ipp.txt" - piholeGitDir="/etc/.pihole" GRAVITYDB=$(getFTLConfigValue files.gravity) gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql" @@ -299,12 +296,7 @@ migrate_to_database() { # Determine if DNS resolution is available before proceeding gravity_CheckDNSResolutionAvailable() { - local lookupDomain="pi.hole" - - # Determine if $localList does not exist, and ensure it is not empty - if [[ ! -e "${localList}" ]] || [[ -s "${localList}" ]]; then - lookupDomain="raw.githubusercontent.com" - fi + local lookupDomain="raw.githubusercontent.com" # Determine if $lookupDomain is resolvable if timeout 4 getent hosts "${lookupDomain}" &> /dev/null; then @@ -686,18 +678,6 @@ gravity_ShowCount() { gravity_Table_Count "vw_regex_whitelist" "regex allowed filters" } -# Create "localhost" entries into hosts format -gravity_generateLocalList() { - # Empty $localList if it already exists, otherwise, create it - echo "### Do not modify this file, it will be overwritten by pihole -g" > "${localList}" - chmod 644 "${localList}" - - # Add additional LAN hosts provided by OpenVPN (if available) - if [[ -f "${VPNList}" ]]; then - awk -F, '{printf $2"\t"$1".vpn\n"}' "${VPNList}" >> "${localList}" - fi -} - # Trap Ctrl-C gravity_Trap() { trap '{ echo -e "\\n\\n ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT @@ -884,9 +864,6 @@ if ! gravity_DownloadBlocklists; then exit 1 fi -# Create local.list -gravity_generateLocalList - # Update gravity timestamp update_gravity_timestamp From e3c6f162d84a3cf78e6e50969424b1e6cebe742f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 7 Nov 2023 08:57:56 +0100 Subject: [PATCH 136/413] Apply suggestions from code review Co-authored-by: Adam Warner Co-authored-by: RD WebDesign Signed-off-by: DL6ER --- automated install/basic-install.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 704a736d..e743a071 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2087,8 +2087,8 @@ migrate_dnsmasq_configs() { # Create target directory /etc/pihole/migration_backup_v6 # and make it owned by pihole:pihole - mkdir -p "${V6_CONF_MIGRATION_DIR} - chown pihole:pihole "${V6_CONF_MIGRATION_DIR} + mkdir -p "${V6_CONF_MIGRATION_DIR}" + chown pihole:pihole "${V6_CONF_MIGRATION_DIR}" # Move all conf files originally created by Pi-hole into this directory # - 01-pihole.conf @@ -2097,8 +2097,8 @@ migrate_dnsmasq_configs() { # - 05-pihole-custom-cname.conf # - 06-rfc6761.conf - mv /etc/dnsmasq.d/0{1,2,4,5}-pihole*.conf "${V6_MIGRATION_DIR}/ CONF_2>/dev/null || true - mv /etc/dnsmasq.d/06-rfc6761.conf "${V6_MIGRATION_DIR}/ CONF_2>/dev/null || true + mv /etc/dnsmasq.d/0{1,2,4,5}-pihole*.conf "${V6_CONF_MIGRATION_DIR}/" 2>/dev/null || true + mv /etc/dnsmasq.d/06-rfc6761.conf "${V6_MIGRATION_DIR}/" 2>/dev/null || true } main() { From e8884083ef1f10229629e80021107f807a7fd584 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 7 Nov 2023 08:58:31 +0100 Subject: [PATCH 137/413] Apply suggestions from code review Signed-off-by: DL6ER --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index e743a071..da0133b1 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2098,7 +2098,7 @@ migrate_dnsmasq_configs() { # - 06-rfc6761.conf mv /etc/dnsmasq.d/0{1,2,4,5}-pihole*.conf "${V6_CONF_MIGRATION_DIR}/" 2>/dev/null || true - mv /etc/dnsmasq.d/06-rfc6761.conf "${V6_MIGRATION_DIR}/" 2>/dev/null || true + mv /etc/dnsmasq.d/06-rfc6761.conf "${V6_CONF_MIGRATION_DIR}/" 2>/dev/null || true } main() { From 766cb26af5884281f343b2cba87cf7cafc207447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 7 Nov 2023 22:24:34 +0100 Subject: [PATCH 138/413] Drop Fedora 36 and add Fedora 39 to the test suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 2 +- test/{_fedora_36.Dockerfile => _fedora_39.Dockerfile} | 2 +- test/{tox.fedora_36.ini => tox.fedora_39.ini} | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename test/{_fedora_36.Dockerfile => _fedora_39.Dockerfile} (97%) rename test/{tox.fedora_36.ini => tox.fedora_39.ini} (80%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8166d253..9d90b2e7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,9 +64,9 @@ jobs: ubuntu_23, centos_8, centos_9, - fedora_36, fedora_37, fedora_38, + fedora_39, ] env: DISTRO: ${{matrix.distro}} diff --git a/test/_fedora_36.Dockerfile b/test/_fedora_39.Dockerfile similarity index 97% rename from test/_fedora_36.Dockerfile rename to test/_fedora_39.Dockerfile index 847767e7..1727a3aa 100644 --- a/test/_fedora_36.Dockerfile +++ b/test/_fedora_39.Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:36 +FROM fedora:39 RUN dnf install -y git initscripts ENV GITDIR /etc/.pihole diff --git a/test/tox.fedora_36.ini b/test/tox.fedora_39.ini similarity index 80% rename from test/tox.fedora_36.ini rename to test/tox.fedora_39.ini index 1896a45f..5c8557c9 100644 --- a/test/tox.fedora_36.ini +++ b/test/tox.fedora_39.ini @@ -1,8 +1,8 @@ [tox] envlist = py3 -[testenv:py3] +[testenv] allowlist_externals = docker deps = -rrequirements.txt -commands = docker buildx build --load --progress plain -f _fedora_36.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _fedora_39.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py From f65b12643345c74d27c1c82a7bf3c177e609cce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 13 Nov 2023 21:12:28 +0100 Subject: [PATCH 139/413] Move custom.list to /hosts/custom.list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 2 +- automated install/basic-install.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 8c2388d8..c346da57 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -78,7 +78,7 @@ PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml" -PIHOLE_CUSTOM_HOSTS_FILE="${PIHOLE_DIRECTORY}/custom.list" +PIHOLE_CUSTOM_HOSTS_FILE="${PIHOLE_DIRECTORY}/hosts/custom.list" PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" # Read the value of an FTL config key. The value is printed to stdout. diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ef5add0e..9b619cbf 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1122,9 +1122,9 @@ installConfigs() { chown pihole:pihole "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" # Install empty custom.list file if it does not exist - if [[ ! -r "${PI_HOLE_CONFIG_DIR}/custom.list" ]]; then - if ! install -o pihole -g pihole -m 660 /dev/null "${PI_HOLE_CONFIG_DIR}/custom.list" &>/dev/null; then - printf " %b Error: Unable to initialize configuration file %s/custom.list\\n" "${COL_LIGHT_RED}" "${PI_HOLE_CONFIG_DIR}" + if [[ ! -r "${PI_HOLE_CONFIG_DIR}/hosts/custom.list" ]]; then + if ! install -D -T -o pihole -g pihole -m 660 /dev/null "${PI_HOLE_CONFIG_DIR}/hosts/custom.list" &>/dev/null; then + printf " %b Error: Unable to initialize configuration file %s/custom.list\\n" "${COL_LIGHT_RED}" "${PI_HOLE_CONFIG_DIR}/hosts" return 1 fi fi From 86aa6b1df64ad263cdc6b48f3d9ce870cf0a5fe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 6 Nov 2023 22:38:01 +0100 Subject: [PATCH 140/413] Improve v6 debug log and remove leftovers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 43 +++++++++------------------------ 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index c346da57..a45e52a9 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -49,7 +49,6 @@ FAQ_HARDWARE_REQUIREMENTS="${COL_CYAN}https://docs.pi-hole.net/main/prerequisite FAQ_HARDWARE_REQUIREMENTS_PORTS="${COL_CYAN}https://docs.pi-hole.net/main/prerequisites/#ports${COL_NC}" FAQ_HARDWARE_REQUIREMENTS_FIREWALLD="${COL_CYAN}https://docs.pi-hole.net/main/prerequisites/#firewalld${COL_NC}" FAQ_GATEWAY="${COL_CYAN}https://discourse.pi-hole.net/t/why-is-a-default-gateway-important-for-pi-hole/3546${COL_NC}" -FAQ_FTL_COMPATIBILITY="${COL_CYAN}https://github.com/pi-hole/FTL#compatibility-list${COL_NC}" # Other URLs we may use FORUMS_URL="${COL_CYAN}https://discourse.pi-hole.net${COL_NC}" @@ -78,7 +77,6 @@ PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml" -PIHOLE_CUSTOM_HOSTS_FILE="${PIHOLE_DIRECTORY}/hosts/custom.list" PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" # Read the value of an FTL config key. The value is printed to stdout. @@ -91,12 +89,12 @@ get_ftl_conf_value() { local default=$2 local value - # Obtain key=... setting from pihole-FTL.conf + # Obtain key=... setting from FTL directly if [[ -e "$PIHOLE_FTL_CONF_FILE" ]]; then # Constructed to return nothing when # a) the setting is not present in the config file, or # b) the setting is commented out (e.g. "#DBFILE=...") - value="$(sed -n -e "s/^\\s*$key=\\s*//p" ${PIHOLE_FTL_CONF_FILE})" + value="$(pihole-FTL --config "${key}")" fi # Test for missing value. Use default value in this case. @@ -107,9 +105,9 @@ get_ftl_conf_value() { echo "$value" } -PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "GRAVITYDB" "${PIHOLE_DIRECTORY}/gravity.db")" +PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "files.gravity" "${PIHOLE_DIRECTORY}/gravity.db")" -PIHOLE_FTL_DB_FILE="$(get_ftl_conf_value "DBFILE" "${PIHOLE_DIRECTORY}/pihole-FTL.db")" +PIHOLE_FTL_DB_FILE="$(get_ftl_conf_value "files.database" "${PIHOLE_DIRECTORY}/pihole-FTL.db")" PIHOLE_COMMAND="${BIN_DIRECTORY}/pihole" PIHOLE_COLTABLE_FILE="${BIN_DIRECTORY}/COL_TABLE" @@ -119,10 +117,8 @@ FTL_PID="${RUN_DIRECTORY}/pihole-FTL.pid" PIHOLE_LOG="${LOG_DIRECTORY}/pihole.log" PIHOLE_LOG_GZIPS="${LOG_DIRECTORY}/pihole.log.[0-9].*" PIHOLE_DEBUG_LOG="${LOG_DIRECTORY}/pihole_debug.log" -PIHOLE_FTL_LOG="$(get_ftl_conf_value "LOGFILE" "${LOG_DIRECTORY}/FTL.log")" - -# PIHOLE_WEB_SERVER_ACCESS_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/access-pihole.log" #TODO: FTL Error log? -# PIHOLE_WEB_SERVER_ERROR_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/error-pihole.log" #TODO: FTL Error log? +PIHOLE_FTL_LOG="$(get_ftl_conf_value "files.log.ftl" "${LOG_DIRECTORY}/FTL.log")" +PIHOLE_WEBSERVER_LOG="$(get_ftl_conf_value "files.log.webserver" "${LOG_DIRECTORY}/webserver.log")" RESOLVCONF="${ETC}/resolv.conf" DNSMASQ_CONF="${ETC}/dnsmasq.conf" @@ -132,8 +128,6 @@ PIHOLE_PROCESSES=( "pihole-FTL" ) # Store the required directories in an array so it can be parsed through REQUIRED_FILES=("${PIHOLE_CRON_FILE}" -# "${WEB_SERVER_CONFIG_FILE}" -# "${WEB_SERVER_CUSTOM_CONFIG_FILE}" "${PIHOLE_INSTALL_LOG_FILE}" "${PIHOLE_RAW_BLOCKLIST_FILES}" "${PIHOLE_LOCAL_HOSTS_FILE}" @@ -146,11 +140,9 @@ REQUIRED_FILES=("${PIHOLE_CRON_FILE}" "${PIHOLE_LOG_GZIPS}" "${PIHOLE_DEBUG_LOG}" "${PIHOLE_FTL_LOG}" -"${PIHOLE_WEB_SERVER_ACCESS_LOG_FILE}" -"${PIHOLE_WEB_SERVER_ERROR_LOG_FILE}" +"${PIHOLE_WEBSERVER_LOG}" "${RESOLVCONF}" "${DNSMASQ_CONF}" -"${PIHOLE_CUSTOM_HOSTS_FILE}" "${PIHOLE_VERSIONS_FILE}") DISCLAIMER="This process collects information from your Pi-hole, and optionally uploads it to a unique and random directory on tricorder.pi-hole.net. @@ -983,12 +975,6 @@ list_files_in_dir() { if [[ "${dir_to_parse}" == "${SHM_DIRECTORY}" ]]; then # SHM file - we do not want to see the content, but we want to see the files and their sizes log_write "$(ls -lh "${dir_to_parse}/")" - elif [[ "${dir_to_parse}" == "${WEB_SERVER_CONFIG_DIRECTORY_FEDORA}" ]]; then - # we want to see all files files in /etc/lighttpd/conf.d - log_write "$(ls -lh "${dir_to_parse}/" 2> /dev/null )" - elif [[ "${dir_to_parse}" == "${WEB_SERVER_CONFIG_DIRECTORY_DEBIAN}" ]]; then - # we want to see all files files in /etc/lighttpd/conf.d - log_write "$(ls -lh "${dir_to_parse}/"/ 2> /dev/null )" fi # Store the files found in an array @@ -1002,7 +988,6 @@ list_files_in_dir() { [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_RAW_BLOCKLIST_FILES}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_INSTALL_LOG_FILE}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_LOG}" ]] || \ - [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_WEB_SERVER_ACCESS_LOG_FILE}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_LOG_GZIPS}" ]]; then : elif [[ "${dir_to_parse}" == "${DNSMASQ_D_DIRECTORY}" ]]; then @@ -1017,8 +1002,8 @@ list_files_in_dir() { log_write "\\n${COL_GREEN}$(ls -lhd "${dir_to_parse}"/"${each_file}")${COL_NC}" # Check if the file we want to view has a limit (because sometimes we just need a little bit of info from the file, not the entire thing) case "${dir_to_parse}/${each_file}" in - # If it's Web server error log, give the first and last 25 lines - "${PIHOLE_WEB_SERVER_ERROR_LOG_FILE}") head_tail_log "${dir_to_parse}/${each_file}" 25 + # If it's Web server log, give the first and last 25 lines + "${PIHOLE_WEBSERVER_LOG}") head_tail_log "${dir_to_parse}/${each_file}" 25 ;; # Same for the FTL log "${PIHOLE_FTL_LOG}") head_tail_log "${dir_to_parse}/${each_file}" 35 @@ -1049,11 +1034,7 @@ show_content_of_pihole_files() { # Show the content of the files in each of Pi-hole's folders show_content_of_files_in_dir "${PIHOLE_DIRECTORY}" show_content_of_files_in_dir "${DNSMASQ_D_DIRECTORY}" - show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY}" - show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY_FEDORA}" - show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY_DEBIAN}" show_content_of_files_in_dir "${CRON_D_DIRECTORY}" - show_content_of_files_in_dir "${WEB_SERVER_LOG_DIRECTORY}" show_content_of_files_in_dir "${LOG_DIRECTORY}" show_content_of_files_in_dir "${SHM_DIRECTORY}" show_content_of_files_in_dir "${ETC}" @@ -1290,10 +1271,10 @@ spinner(){ analyze_pihole_log() { echo_current_diagnostic "Pi-hole log" local pihole_log_permissions - local logging_enabled + local queryLogging - logging_enabled=$(grep -c "^log-queries" /etc/dnsmasq.d/01-pihole.conf) - if [[ "${logging_enabled}" == "0" ]]; then + queryLogging=$(pihole-FTL --config dns.queryLogging) + if [[ "${queryLogging}" == "false" ]]; then # Inform user that logging has been disabled and pihole.log does not contain queries log_write "${INFO} Query logging is disabled" log_write "" From d854eb1a9789a99a24b3ca7b7c101418d47a383c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 7 Nov 2023 21:59:35 +0100 Subject: [PATCH 141/413] Add lshw to Pi-hole deps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9b619cbf..72f00992 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -332,7 +332,7 @@ package_manager_detect() { # Packages required to run this install script INSTALLER_DEPS=(git iproute2 dialog ca-certificates) # Packages required to run Pi-hole - PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) + PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq lshw) # If apt-get is not found, check for rpm. elif is_command rpm ; then @@ -349,7 +349,7 @@ package_manager_detect() { PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src|.riscv64)' | wc -l || true" OS_CHECK_DEPS=(grep bind-utils) INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates binutils) - PIHOLE_DEPS=(cronie curl findutils sudo unzip psmisc libcap nmap-ncat jq) + PIHOLE_DEPS=(cronie curl findutils sudo unzip psmisc libcap nmap-ncat jq lshw) # If neither apt-get or yum/dnf package managers were found else From d3813c4be5eb058ae27ff74167f6794b1c7a2e8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 8 Nov 2023 21:28:05 +0100 Subject: [PATCH 142/413] Update check for required ports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index a45e52a9..a3cfaedd 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -634,10 +634,8 @@ compare_port_to_service_assigned() { check_required_ports() { echo_current_diagnostic "Ports in use" - # Since Pi-hole needs 53 and 4711, check what they are being used by + # Since Pi-hole needs 53,80 and 443 check what they are being used by # so we can detect any issues - local resolver="pihole-FTL" - local web_server="pihole-FTL" local ftl="pihole-FTL" # Create an array for these ports in use ports_in_use=() @@ -656,17 +654,13 @@ check_required_ports() { local port_number port_number="$(echo "${ports_in_use[$i]}" | awk '{print $5}')" # | awk '{gsub(/^.*:/,"",$5);print $5}') - # Use a case statement to determine if the right services are using the right ports - case "$(echo "${port_number}" | rev | cut -d: -f1 | rev)" in - 53) compare_port_to_service_assigned "${resolver}" "${service_name}" "${protocol_type}:${port_number}" - ;; - 80) compare_port_to_service_assigned "${web_server}" "${service_name}" "${protocol_type}:${port_number}" - ;; - 4711) compare_port_to_service_assigned "${ftl}" "${service_name}" "${protocol_type}:${port_number}" - ;; + # Check if the right services are using the right ports + if [[ "$(echo "${port_number}" | rev | cut -d: -f1 | rev)" == @(53|80|443) ]]; then + compare_port_to_service_assigned "${ftl}" "${service_name}" "${protocol_type}:${port_number}" + else # If it's not a default port that Pi-hole needs, just print it out for the user to see - *) log_write " ${protocol_type}:${port_number} is in use by ${service_name:=}"; - esac + log_write " ${protocol_type}:${port_number} is in use by ${service_name:=}"; + fi done } From e7c89ce25fb3a8b4e59f61df63ab00838543ac59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 12 Nov 2023 20:45:45 +0100 Subject: [PATCH 143/413] No need to pass a default value to get_ftl_conf_value() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index a3cfaedd..1a3c2a34 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -80,34 +80,18 @@ PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml" PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" # Read the value of an FTL config key. The value is printed to stdout. -# -# Args: -# 1. The key to read -# 2. The default if the setting or config does not exist get_ftl_conf_value() { local key=$1 - local default=$2 local value - # Obtain key=... setting from FTL directly - if [[ -e "$PIHOLE_FTL_CONF_FILE" ]]; then - # Constructed to return nothing when - # a) the setting is not present in the config file, or - # b) the setting is commented out (e.g. "#DBFILE=...") - value="$(pihole-FTL --config "${key}")" - fi - - # Test for missing value. Use default value in this case. - if [[ -z "$value" ]]; then - value="$default" - fi - + # Obtain setting from FTL directly + value="$(pihole-FTL --config "${key}")" echo "$value" } -PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "files.gravity" "${PIHOLE_DIRECTORY}/gravity.db")" +PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "files.gravity")" -PIHOLE_FTL_DB_FILE="$(get_ftl_conf_value "files.database" "${PIHOLE_DIRECTORY}/pihole-FTL.db")" +PIHOLE_FTL_DB_FILE="$(get_ftl_conf_value "files.database")" PIHOLE_COMMAND="${BIN_DIRECTORY}/pihole" PIHOLE_COLTABLE_FILE="${BIN_DIRECTORY}/COL_TABLE" @@ -117,8 +101,8 @@ FTL_PID="${RUN_DIRECTORY}/pihole-FTL.pid" PIHOLE_LOG="${LOG_DIRECTORY}/pihole.log" PIHOLE_LOG_GZIPS="${LOG_DIRECTORY}/pihole.log.[0-9].*" PIHOLE_DEBUG_LOG="${LOG_DIRECTORY}/pihole_debug.log" -PIHOLE_FTL_LOG="$(get_ftl_conf_value "files.log.ftl" "${LOG_DIRECTORY}/FTL.log")" -PIHOLE_WEBSERVER_LOG="$(get_ftl_conf_value "files.log.webserver" "${LOG_DIRECTORY}/webserver.log")" +PIHOLE_FTL_LOG="$(get_ftl_conf_value "files.log.ftl")" +PIHOLE_WEBSERVER_LOG="$(get_ftl_conf_value "files.log.webserver")" RESOLVCONF="${ETC}/resolv.conf" DNSMASQ_CONF="${ETC}/dnsmasq.conf" @@ -1267,7 +1251,7 @@ analyze_pihole_log() { local pihole_log_permissions local queryLogging - queryLogging=$(pihole-FTL --config dns.queryLogging) + queryLogging="$(get_ftl_conf_value "dns.queryLogging")" if [[ "${queryLogging}" == "false" ]]; then # Inform user that logging has been disabled and pihole.log does not contain queries log_write "${INFO} Query logging is disabled" From 54ab71d8176acdfd1c0db676b8f06090cd4850ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 12 Nov 2023 21:20:03 +0100 Subject: [PATCH 144/413] Don't use hardcoded ports to check for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 1a3c2a34..703c308b 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -618,7 +618,7 @@ compare_port_to_service_assigned() { check_required_ports() { echo_current_diagnostic "Ports in use" - # Since Pi-hole needs 53,80 and 443 check what they are being used by + # Since Pi-hole needs various ports, check what they are being used by # so we can detect any issues local ftl="pihole-FTL" # Create an array for these ports in use @@ -628,6 +628,15 @@ check_required_ports() { ports_in_use+=( "$line" ) done < <( ss --listening --numeric --tcp --udp --processes --no-header ) + local ports_configured + # Get all configured ports + ports_configured="$(pihole-FTL --config "webserver.port")" + # Remove all non-didgits, split into an array at "," + ports_configured="${ports_configured//[!0-9,]/}" + mapfile -d "," -t ports_configured < <(echo "${ports_configured}") + # Add port 53 + ports_configured+=("53") + # Now that we have the values stored, for i in "${!ports_in_use[@]}"; do # loop through them and assign some local variables @@ -639,7 +648,7 @@ check_required_ports() { port_number="$(echo "${ports_in_use[$i]}" | awk '{print $5}')" # | awk '{gsub(/^.*:/,"",$5);print $5}') # Check if the right services are using the right ports - if [[ "$(echo "${port_number}" | rev | cut -d: -f1 | rev)" == @(53|80|443) ]]; then + if [[ ${ports_configured[*]} =~ $(echo "${port_number}" | rev | cut -d: -f1 | rev) ]]; then compare_port_to_service_assigned "${ftl}" "${service_name}" "${protocol_type}:${port_number}" else # If it's not a default port that Pi-hole needs, just print it out for the user to see From b61e8be3b46bae8be8fddb8b4d7315b0afd9801c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 12 Nov 2023 21:36:18 +0100 Subject: [PATCH 145/413] Address reviewer's comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: DL6ER Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 703c308b..931e95a4 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -82,11 +82,9 @@ PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" # Read the value of an FTL config key. The value is printed to stdout. get_ftl_conf_value() { local key=$1 - local value # Obtain setting from FTL directly - value="$(pihole-FTL --config "${key}")" - echo "$value" + pihole-FTL --config "${key}" } PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "files.gravity")" From ab09233753cd522ca84faacbe02d7bdbb7f5c40f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Nov 2023 10:17:34 +0000 Subject: [PATCH 146/413] Bump pytest-testinfra from 9.0.0 to 10.0.0 in /test Bumps [pytest-testinfra](https://github.com/pytest-dev/pytest-testinfra) from 9.0.0 to 10.0.0. - [Release notes](https://github.com/pytest-dev/pytest-testinfra/releases) - [Changelog](https://github.com/pytest-dev/pytest-testinfra/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-testinfra/compare/9.0.0...10.0.0) --- updated-dependencies: - dependency-name: pytest-testinfra dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 45c7c7c0..c3147a10 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.3 pytest-xdist == 3.3.1 -pytest-testinfra == 9.0.0 +pytest-testinfra == 10.0.0 tox == 4.11.3 From f4b67065ccb547c374afc42274207297748deb08 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Nov 2023 10:55:51 +0000 Subject: [PATCH 147/413] Bump pytest-xdist from 3.3.1 to 3.4.0 in /test Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.3.1 to 3.4.0. - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.3.1...v3.4.0) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index c3147a10..74c67fd9 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.3 -pytest-xdist == 3.3.1 +pytest-xdist == 3.4.0 pytest-testinfra == 10.0.0 tox == 4.11.3 From 19d3489bcbe5aca79ec08a38008b6d5a2f1dcaf1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 20:56:23 +0100 Subject: [PATCH 148/413] gravity_DownloadBlocklistFromUrl needs ${domain} but it was declared local before and was not passed as argument, making gravity currently depend on undefined behavior. It seems to be working well in the vast majority of cases, however, it seems we have at least one report where it is not working. Signed-off-by: DL6ER --- gravity.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index edfe89a9..421ec035 100755 --- a/gravity.sh +++ b/gravity.sh @@ -464,7 +464,7 @@ gravity_DownloadBlocklists() { if [[ "${check_url}" =~ ${regex} ]]; then echo -e " ${CROSS} Invalid Target" else - gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" "${adlist_type}" + gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" "${adlist_type}" "${domain}" fi echo "" done @@ -496,7 +496,7 @@ compareLists() { # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { - local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" gravity_type="${6}" + local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" gravity_type="${6}" domain="${7}" local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext # Create temp file to store content on disk instead of RAM From f16cf7178186e33004bcfb44c3838a9fd2912007 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 21:04:46 +0100 Subject: [PATCH 149/413] ${PIHOLE_DNS_1} is gone, use the first server from pihole-FTL --config dns.upstreams instead Signed-off-by: DL6ER --- gravity.sh | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index 421ec035..90cba0f6 100755 --- a/gravity.sh +++ b/gravity.sh @@ -538,12 +538,25 @@ gravity_DownloadBlocklistFromUrl() { fi;; esac + if [[ "${blocked}" == true ]]; then - printf -v ip_addr "%s" "${PIHOLE_DNS_1%#*}" - if [[ ${PIHOLE_DNS_1} != *"#"* ]]; then + # Get first defined upstream server + local upstream + upstream="$(getFTLConfigValue dns.upstreams)" + + # Isolate first upstream server from a string like + # [ 1.2.3.4#1234, 5.6.7.8#5678, ... ] + upstream="${upstream%%,*}" + upstream="${upstream##*[}" + upstream="${upstream%%]*}" + + # Get IP address and port of this upstream server + local ip_addr port + printf -v ip_addr "%s" "${upstream%#*}" + if [[ ${upstream} != *"#"* ]]; then port=53 else - printf -v port "%s" "${PIHOLE_DNS_1#*#}" + printf -v port "%s" "${upstream#*#}" fi ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}" | tail -1) if [[ $(echo "${url}" | awk -F '://' '{print $1}') = "https" ]]; then @@ -551,7 +564,7 @@ gravity_DownloadBlocklistFromUrl() { else port=80 fi bad_list=$(pihole -q -adlist "${domain}" | head -n1 | awk -F 'Match found in ' '{print $2}') - echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${PIHOLE_DNS_1} to download ${url}"; + echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${upstream} to download ${url}"; echo -ne " ${INFO} ${str} Pending..." cmd_ext="--resolve $domain:$port:$ip" fi From 4efcdf11890ca069019c5b9cb3c1206e10a7f73e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 21:06:09 +0100 Subject: [PATCH 150/413] Add missing double quotes to prevent globbing and word splitting Signed-off-by: DL6ER --- gravity.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index 90cba0f6..7a4e52c2 100755 --- a/gravity.sh +++ b/gravity.sh @@ -99,7 +99,7 @@ gravity_swap_databases() { # Number of available blocks on disk availableBlocks=$(stat -f --format "%a" "${gravityDIR}") # Number of blocks, used by gravity.db - gravityBlocks=$(stat --format "%b" ${gravityDBfile}) + gravityBlocks=$(stat --format "%b" "${gravityDBfile}") # Only keep the old database if available disk space is at least twice the size of the existing gravity.db. # Better be safe than sorry... oldAvail=false @@ -604,7 +604,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ "${success}" == true ]]; then if [[ "${httpCode}" == "304" ]]; then # Add domains to database table file - pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" database_adlist_status "${adlistID}" "2" done="true" # Check if $listCurlBuffer is a non-zero length file @@ -614,7 +614,7 @@ gravity_DownloadBlocklistFromUrl() { # Remove curl buffer file after its use rm "${listCurlBuffer}" # Add domains to database table file - pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" # Compare lists, are they identical? compareLists "${adlistID}" "${saveLocation}" done="true" @@ -630,7 +630,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ -r "${saveLocation}" ]]; then echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" # Add domains to database table file - pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" database_adlist_status "${adlistID}" "3" else echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" From df7633bd1b2f4c8e653620ead7c00ab86604428a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 21:08:06 +0100 Subject: [PATCH 151/413] Add missing value for ${gravityDBfile_default} Signed-off-by: DL6ER --- gravity.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/gravity.sh b/gravity.sh index 7a4e52c2..64ba662e 100755 --- a/gravity.sh +++ b/gravity.sh @@ -59,6 +59,7 @@ fi # Set this only after sourcing pihole-FTL.conf as the gravity database path may # have changed gravityDBfile="${GRAVITYDB}" +gravityDBfile_default="/etc/pihole/gravity.db" gravityTEMPfile="${GRAVITYDB}_temp" gravityDIR="$(dirname -- "${gravityDBfile}")" gravityOLDfile="${gravityDIR}/gravity_old.db" From cc333f79ccba8358b87bb38d6f3694ab3bf24809 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 21:10:22 +0100 Subject: [PATCH 152/413] Check if this domain is blocked by Pi-hole but only if the domain is not a local file or empty Signed-off-by: DL6ER --- gravity.sh | 96 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 50 insertions(+), 46 deletions(-) diff --git a/gravity.sh b/gravity.sh index 64ba662e..4aea4d68 100755 --- a/gravity.sh +++ b/gravity.sh @@ -517,57 +517,61 @@ gravity_DownloadBlocklistFromUrl() { str="Status:" echo -ne " ${INFO} ${str} Pending..." blocked=false - case $(getFTLConfigValue dns.blocking.mode) in - "IP-NODATA-AAAA"|"IP") - # Get IP address of this domain - ip="$(dig "${domain}" +short)" - # Check if this IP matches any IP of the system - if [[ -n "${ip}" && $(grep -Ec "inet(|6) ${ip}" <<< "$(ip a)") -gt 0 ]]; then - blocked=true - fi;; - "NXDOMAIN") - if [[ $(dig "${domain}" | grep "NXDOMAIN" -c) -ge 1 ]]; then - blocked=true - fi;; - "NODATA") - if [[ $(dig "${domain}" | grep "NOERROR" -c) -ge 1 ]] && [[ -z $(dig +short "${domain}") ]]; then - blocked=true - fi;; - "NULL"|*) - if [[ $(dig "${domain}" +short | grep "0.0.0.0" -c) -ge 1 ]]; then - blocked=true - fi;; - esac + # Check if this domain is blocked by Pi-hole but only if the domain is not a + # local file or empty + if [[ $url != "file"* ]] && [[ -n "${domain}" ]]; then + case $(getFTLConfigValue dns.blocking.mode) in + "IP-NODATA-AAAA"|"IP") + # Get IP address of this domain + ip="$(dig "${domain}" +short)" + # Check if this IP matches any IP of the system + if [[ -n "${ip}" && $(grep -Ec "inet(|6) ${ip}" <<< "$(ip a)") -gt 0 ]]; then + blocked=true + fi;; + "NXDOMAIN") + if [[ $(dig "${domain}" | grep "NXDOMAIN" -c) -ge 1 ]]; then + blocked=true + fi;; + "NODATA") + if [[ $(dig "${domain}" | grep "NOERROR" -c) -ge 1 ]] && [[ -z $(dig +short "${domain}") ]]; then + blocked=true + fi;; + "NULL"|*) + if [[ $(dig "${domain}" +short | grep "0.0.0.0" -c) -ge 1 ]]; then + blocked=true + fi;; + esac - if [[ "${blocked}" == true ]]; then - # Get first defined upstream server - local upstream - upstream="$(getFTLConfigValue dns.upstreams)" + if [[ "${blocked}" == true ]]; then + # Get first defined upstream server + local upstream + upstream="$(getFTLConfigValue dns.upstreams)" - # Isolate first upstream server from a string like - # [ 1.2.3.4#1234, 5.6.7.8#5678, ... ] - upstream="${upstream%%,*}" - upstream="${upstream##*[}" - upstream="${upstream%%]*}" + # Isolate first upstream server from a string like + # [ 1.2.3.4#1234, 5.6.7.8#5678, ... ] + upstream="${upstream%%,*}" + upstream="${upstream##*[}" + upstream="${upstream%%]*}" - # Get IP address and port of this upstream server - local ip_addr port - printf -v ip_addr "%s" "${upstream%#*}" - if [[ ${upstream} != *"#"* ]]; then - port=53 - else - printf -v port "%s" "${upstream#*#}" + # Get IP address and port of this upstream server + local ip_addr port + printf -v ip_addr "%s" "${upstream%#*}" + if [[ ${upstream} != *"#"* ]]; then + port=53 + else + printf -v port "%s" "${upstream#*#}" + fi + ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}" | tail -1) + if [[ $(echo "${url}" | awk -F '://' '{print $1}') = "https" ]]; then + port=443; + else port=80 + fi + bad_list=$(pihole -q -adlist "${domain}" | head -n1 | awk -F 'Match found in ' '{print $2}') + echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${upstream} to download ${url}"; + echo -ne " ${INFO} ${str} Pending..." + cmd_ext="--resolve $domain:$port:$ip" fi - ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}" | tail -1) - if [[ $(echo "${url}" | awk -F '://' '{print $1}') = "https" ]]; then - port=443; - else port=80 - fi - bad_list=$(pihole -q -adlist "${domain}" | head -n1 | awk -F 'Match found in ' '{print $2}') - echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${upstream} to download ${url}"; - echo -ne " ${INFO} ${str} Pending..." - cmd_ext="--resolve $domain:$port:$ip" fi # shellcheck disable=SC2086 From c785667efea9e579db37c7d450bd3fbe9e7612a0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 23 Nov 2023 09:56:43 +0100 Subject: [PATCH 153/413] Trim leading and trailing spaces and tabs in upstream servers (if any) Signed-off-by: DL6ER --- gravity.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 4aea4d68..20ad6215 100755 --- a/gravity.sh +++ b/gravity.sh @@ -553,6 +553,9 @@ gravity_DownloadBlocklistFromUrl() { upstream="${upstream%%,*}" upstream="${upstream##*[}" upstream="${upstream%%]*}" + # Trim leading and trailing spaces and tabs + upstream="${upstream#"${upstream%%[![:space:]]*}"}" + upstream="${upstream%"${upstream##*[![:space:]]}"}" # Get IP address and port of this upstream server local ip_addr port @@ -567,8 +570,7 @@ gravity_DownloadBlocklistFromUrl() { port=443; else port=80 fi - bad_list=$(pihole -q -adlist "${domain}" | head -n1 | awk -F 'Match found in ' '{print $2}') - echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${upstream} to download ${url}"; + echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by one of your lists. Using DNS server ${upstream} instead"; echo -ne " ${INFO} ${str} Pending..." cmd_ext="--resolve $domain:$port:$ip" fi From a6565bf9a17ad4998b00ca239be9044be7f51674 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 23 Nov 2023 22:07:31 +0100 Subject: [PATCH 154/413] Support special webserver.port ports ending in "s" (secure) and "r" (redirect) Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 43 ++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 449f146f..000c0717 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -21,14 +21,31 @@ TestAPIAvailability() { # as we are running locally, we can get the port value from FTL directly - PORT="$(pihole-FTL --config webserver.port)" - PORT="${PORT%%,*}" + local ports port availabilityResonse + ports="$(pihole-FTL --config webserver.port)" + port="${ports%%,*}" - availabilityResonse=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:${PORT}/api/auth") + # if the port ends with an "s", it is a secure connection + if [ "${port#"${port%?}"}" = "s" ]; then + # remove the "s" from the port + API_PROT="https" + API_PORT="${port%?}" + elif [ "${port#"${port%?}"}" = "r" ]; then + # if the port ends in "r", it is a redirect + API_PROT="http" + # remove the "r" from the port + API_PORT="${port%?}" + else + API_PROT="http" + API_PORT="${port}" + fi - # test if http status code was 200 (OK) or 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 401 ]; then - echo "API not available at: http://localhost:${PORT}/api" + API_URL="${API_PROT}://localhost:${API_PORT}/api" + availabilityResonse=$(curl -skSL -o /dev/null -w "%{http_code}" "${API_URL}/auth") + + # test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + echo "API not available at: ${API_URL}" echo "Exiting." exit 1 fi @@ -54,15 +71,15 @@ Authenthication() { } LoginAPI() { - sessionResponse="$(curl --silent -X POST "http://localhost:${PORT}/api/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" + sessionResponse="$(curl -skSL -X POST "${API_URL}/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" if [ -z "${sessionResponse}" ]; then echo "No response from FTL server. Please check connectivity" exit 1 fi - # obtain validity and session ID from session response - validSession=$(echo "${sessionResponse}"| jq .session.valid 2>/dev/null) - SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null) + # obtain validity and session ID from session response + validSession=$(echo "${sessionResponse}"| jq .session.valid 2>/dev/null) + SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null) } DeleteSession() { @@ -70,7 +87,7 @@ DeleteSession() { # SID is not null (successful authenthication only), delete the session if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then # Try to delete the session. Omit the output, but get the http status code - deleteResponse=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE "http://localhost:${PORT}/api/auth" -H "Accept: application/json" -H "sid: ${SID}") + deleteResponse=$(curl -skSL -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}/auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; @@ -84,14 +101,14 @@ DeleteSession() { GetFTLData() { local data response status # get the data from querying the API as well as the http status code - response=$(curl -s -w "%{http_code}" -X GET "http://localhost:${PORT}/api$1" -H "Accept: application/json" -H "sid: ${SID}" ) + response=$(curl -skSL -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" ) # status are the last 3 characters status=$(printf %s "${response#"${response%???}"}") # data is everything from response without the last 3 characters data=$(printf %s "${response%???}") - if [ "${status}" = 200 ]; then + if [ "${status}" = 200 ] || [ "${status}" = 308 ]; then # response OK echo "${data}" elif [ "${status}" = 000 ]; then From 1276242a4ea3164cf2c24c9727fa7396b1495936 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 25 Nov 2023 10:16:35 +0000 Subject: [PATCH 155/413] Bump pytest-xdist from 3.4.0 to 3.5.0 in /test Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.4.0 to 3.5.0. - [Release notes](https://github.com/pytest-dev/pytest-xdist/releases) - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.4.0...v3.5.0) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 74c67fd9..799e3fad 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.3 -pytest-xdist == 3.4.0 +pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.11.3 From 974fea592df99d97332763b6f9cf9812a6c907a4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 Nov 2023 00:05:55 +0100 Subject: [PATCH 156/413] Iterate over ports, skip redirected ports Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 64 +++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 000c0717..b50e416a 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -25,26 +25,48 @@ TestAPIAvailability() { ports="$(pihole-FTL --config webserver.port)" port="${ports%%,*}" - # if the port ends with an "s", it is a secure connection - if [ "${port#"${port%?}"}" = "s" ]; then - # remove the "s" from the port - API_PROT="https" - API_PORT="${port%?}" - elif [ "${port#"${port%?}"}" = "r" ]; then - # if the port ends in "r", it is a redirect - API_PROT="http" - # remove the "r" from the port - API_PORT="${port%?}" - else - API_PROT="http" - API_PORT="${port}" - fi + # Iterate over comma separated list of ports + while [ "${port}" != "${ports}" ]; do + # if the port ends with an "s", it is a secure connection + if [ "${port#"${port%?}"}" = "s" ]; then + # remove the "s" from the port + API_PROT="https" + API_PORT="${port%?}" + elif [ "${port#"${port%?}"}" = "r" ]; then + # Ignore this port + API_PORT="0" + else + API_PROT="http" + API_PORT="${port}" + fi - API_URL="${API_PROT}://localhost:${API_PORT}/api" - availabilityResonse=$(curl -skSL -o /dev/null -w "%{http_code}" "${API_URL}/auth") + if [ ! "${API_PORT}" = "0" ]; then + # If the port is of form "ip:port", we need to remove everything before + # the last ":" in the string, e.g., "[::]:80" -> "80" + if [ "${API_PORT#*:}" != "${API_PORT}" ]; then + API_PORT="${API_PORT##*:}" + fi - # test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + API_URL="${API_PROT}://localhost:${API_PORT}/api" + availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}/auth") + + # test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + API_PORT="0" + else + # API is available at this port/protocol combination + break + fi + fi + + # remove the first port from the list + ports="${ports#*,}" + # get the next port + port="${ports%%,*}" + done + + # if API_PORT is 0, no working API port was found + if [ "${API_PORT}" = "0" ]; then echo "API not available at: ${API_URL}" echo "Exiting." exit 1 @@ -71,7 +93,7 @@ Authenthication() { } LoginAPI() { - sessionResponse="$(curl -skSL -X POST "${API_URL}/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" + sessionResponse="$(curl -skS -X POST "${API_URL}/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" if [ -z "${sessionResponse}" ]; then echo "No response from FTL server. Please check connectivity" @@ -87,7 +109,7 @@ DeleteSession() { # SID is not null (successful authenthication only), delete the session if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then # Try to delete the session. Omit the output, but get the http status code - deleteResponse=$(curl -skSL -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}/auth" -H "Accept: application/json" -H "sid: ${SID}") + deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}/auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; @@ -101,7 +123,7 @@ DeleteSession() { GetFTLData() { local data response status # get the data from querying the API as well as the http status code - response=$(curl -skSL -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" ) + response=$(curl -skS -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" ) # status are the last 3 characters status=$(printf %s "${response#"${response%???}"}") From 6016131280ce5f24cc53fa7989026f593f4d2d4f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 Nov 2023 22:59:49 +0100 Subject: [PATCH 157/413] Ensure we also check the last port Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index b50e416a..2952fb43 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -26,16 +26,18 @@ TestAPIAvailability() { port="${ports%%,*}" # Iterate over comma separated list of ports - while [ "${port}" != "${ports}" ]; do + while [ -n "${ports}" ]; do # if the port ends with an "s", it is a secure connection if [ "${port#"${port%?}"}" = "s" ]; then # remove the "s" from the port API_PROT="https" API_PORT="${port%?}" elif [ "${port#"${port%?}"}" = "r" ]; then - # Ignore this port + # Ignore this port, the client may not be able to follow the + # redirected target when FTL is not used as local resolver API_PORT="0" else + # otherwise it is an insecure (plain HTTP) connection API_PROT="http" API_PORT="${port}" fi @@ -50,8 +52,9 @@ TestAPIAvailability() { API_URL="${API_PROT}://localhost:${API_PORT}/api" availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}/auth") - # test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) + # Test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + # API is not available at this port/protocol combination API_PORT="0" else # API is available at this port/protocol combination @@ -59,9 +62,9 @@ TestAPIAvailability() { fi fi - # remove the first port from the list + # If the loop has not been broken, remove the first port from the list + # and get the next port ports="${ports#*,}" - # get the next port port="${ports%%,*}" done @@ -192,3 +195,6 @@ secretRead() { # restore original terminal settings stty "${stty_orig}" } + + +TestAPIAvailability From 96bf07863f6bdd4d9cad91d5444fef5f85d47b61 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 1 Dec 2023 09:10:06 +0100 Subject: [PATCH 158/413] Use CHAOS TXT local.api.txt instead of trying to parse pihole-FTL --config webserver.ports Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 90 +++++++++++++++++++-------------------- advanced/Scripts/query.sh | 6 +-- 2 files changed, 47 insertions(+), 49 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 2952fb43..46da37cd 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -21,62 +21,60 @@ TestAPIAvailability() { # as we are running locally, we can get the port value from FTL directly - local ports port availabilityResonse - ports="$(pihole-FTL --config webserver.port)" - port="${ports%%,*}" + local chaos_api_list availabilityResonse - # Iterate over comma separated list of ports - while [ -n "${ports}" ]; do - # if the port ends with an "s", it is a secure connection - if [ "${port#"${port%?}"}" = "s" ]; then - # remove the "s" from the port - API_PROT="https" - API_PORT="${port%?}" - elif [ "${port#"${port%?}"}" = "r" ]; then - # Ignore this port, the client may not be able to follow the - # redirected target when FTL is not used as local resolver - API_PORT="0" + # Query the API URLs from FTL using CHAOS TXT local.api.ftl + # The result is a space-separated enumeration of full URLs + # e.g., "http://localhost:80/api" "https://localhost:443/api" + chaos_api_list="$(dig +short chaos txt local.api.ftl @127.0.0.1)" + + # If the query was not successful, the variable is empty + if [ -z "${chaos_api_list}" ]; then + echo "API not available. Please check connectivity" + exit 1 + fi + + # Iterate over space-separated list of URLs + while [ -n "${chaos_api_list}" ]; do + # Get the first URL + API_URL="${chaos_api_list%% *}" + # Strip leading and trailing quotes + API_URL="${API_URL%\"}" + API_URL="${API_URL#\"}" + + # Test if the API is available at this URL + availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}auth") + + # Test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + # API is not available at this port/protocol combination + API_PORT="" else - # otherwise it is an insecure (plain HTTP) connection - API_PROT="http" - API_PORT="${port}" + # API is available at this URL combination + break fi - if [ ! "${API_PORT}" = "0" ]; then - # If the port is of form "ip:port", we need to remove everything before - # the last ":" in the string, e.g., "[::]:80" -> "80" - if [ "${API_PORT#*:}" != "${API_PORT}" ]; then - API_PORT="${API_PORT##*:}" - fi + # Remove the first URL from the list + local last_api_list + last_api_list="${chaos_api_list}" + chaos_api_list="${chaos_api_list#* }" - API_URL="${API_PROT}://localhost:${API_PORT}/api" - availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}/auth") - - # Test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then - # API is not available at this port/protocol combination - API_PORT="0" - else - # API is available at this port/protocol combination - break - fi + # If the list did not change, we are at the last element + if [ "${last_api_list}" = "${chaos_api_list}" ]; then + # Remove the last element + chaos_api_list="" fi - - # If the loop has not been broken, remove the first port from the list - # and get the next port - ports="${ports#*,}" - port="${ports%%,*}" done - # if API_PORT is 0, no working API port was found - if [ "${API_PORT}" = "0" ]; then + # if API_PORT is empty, no working API port was found + if [ -n "${API_PORT}" ]; then echo "API not available at: ${API_URL}" echo "Exiting." exit 1 fi } -Authenthication() { +Authentication() { # Try to authenticate LoginAPI @@ -96,7 +94,7 @@ Authenthication() { } LoginAPI() { - sessionResponse="$(curl -skS -X POST "${API_URL}/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" + sessionResponse="$(curl -skS -X POST "${API_URL}auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" if [ -z "${sessionResponse}" ]; then echo "No response from FTL server. Please check connectivity" @@ -108,11 +106,11 @@ LoginAPI() { } DeleteSession() { - # if a valid Session exists (no password required or successful authenthication) and - # SID is not null (successful authenthication only), delete the session + # if a valid Session exists (no password required or successful Authentication) and + # SID is not null (successful Authentication only), delete the session if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then # Try to delete the session. Omit the output, but get the http status code - deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}/auth" -H "Accept: application/json" -H "sid: ${SID}") + deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 2279df85..62d29d5b 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -121,14 +121,14 @@ Main(){ # or b) for the /search endpoint (webserver.api.searchAPIauth) no authentication is required. # Therefore, we try to query directly without authentication but do authenticat if 401 is returned - data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}") + data=$(GetFTLData "search/${domain}?N=${max_results}&partial=${partial}") if [ "${data}" = 401 ]; then # Unauthenticated, so authenticate with the FTL server required - Authenthication + Authentication # send query again - data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}") + data=$(GetFTLData "search/${domain}?N=${max_results}&partial=${partial}") fi GenerateOutput "${data}" From 29d010dc2ccc04cd6563d8154415e2070b5bd9a7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 1 Dec 2023 10:21:02 +0100 Subject: [PATCH 159/413] Use files.gravity_tmp as temporary directory for the intermediate lists Signed-off-by: DL6ER --- gravity.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index edfe89a9..8dc8edd8 100755 --- a/gravity.sh +++ b/gravity.sh @@ -41,6 +41,7 @@ VPNList="/etc/openvpn/ipp.txt" piholeGitDir="/etc/.pihole" GRAVITYDB=$(getFTLConfigValue files.gravity) +GRAVITY_TMPDIR=$(getFTLConfigValue files.gravity_tmp) gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql" gravityDBcopy="${piholeGitDir}/advanced/Templates/gravity_copy.sql" @@ -48,9 +49,7 @@ domainsExtension="domains" curl_connect_timeout=10 -# Set up tmp dir variable in case it's not configured -: "${GRAVITY_TMPDIR:=/tmp}" - +# Check gravity temp directory if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then echo -e " ${COL_LIGHT_RED}Gravity temporary directory does not exist or is not a writeable directory, falling back to /tmp. ${COL_NC}" GRAVITY_TMPDIR="/tmp" From f5fe550a2e27e5b81ffbe55d3f209c890dda3b94 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Dec 2023 10:35:21 +0000 Subject: [PATCH 160/413] Bump tox from 4.11.3 to 4.11.4 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.11.3 to 4.11.4. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.11.3...4.11.4) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 799e3fad..bfc6d027 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.3 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.11.3 +tox == 4.11.4 From 32a741b5c74bab31ed5c02f5ba33f65f631fea7f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 2 Dec 2023 22:42:36 +0100 Subject: [PATCH 161/413] We do not follow 308 but FTL also doesn't suggest it Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 46da37cd..b7bc2a86 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -25,7 +25,7 @@ TestAPIAvailability() { # Query the API URLs from FTL using CHAOS TXT local.api.ftl # The result is a space-separated enumeration of full URLs - # e.g., "http://localhost:80/api" "https://localhost:443/api" + # e.g., "http://localhost:80/api/" "https://localhost:443/api/" chaos_api_list="$(dig +short chaos txt local.api.ftl @127.0.0.1)" # If the query was not successful, the variable is empty @@ -45,8 +45,8 @@ TestAPIAvailability() { # Test if the API is available at this URL availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}auth") - # Test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + # Test if http status code was 200 (OK) or 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 401 ]; then # API is not available at this port/protocol combination API_PORT="" else @@ -131,7 +131,7 @@ GetFTLData() { # data is everything from response without the last 3 characters data=$(printf %s "${response%???}") - if [ "${status}" = 200 ] || [ "${status}" = 308 ]; then + if [ "${status}" = 200 ]; then # response OK echo "${data}" elif [ "${status}" = 000 ]; then @@ -193,6 +193,3 @@ secretRead() { # restore original terminal settings stty "${stty_orig}" } - - -TestAPIAvailability From 2681835f94ec072e183d6cd6de267f25b93fb0a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 6 Dec 2023 00:21:04 +0100 Subject: [PATCH 162/413] Treat FTL return data as strings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/api.sh | 2 +- advanced/Scripts/query.sh | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 449f146f..bf6e3654 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -93,7 +93,7 @@ GetFTLData() { if [ "${status}" = 200 ]; then # response OK - echo "${data}" + printf %s "${data}" elif [ "${status}" = 000 ]; then # connection lost echo "000" diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 2279df85..789efe75 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -46,14 +46,14 @@ GenerateOutput(){ data="${1}" # construct a new json for the list results where each object contains the domain and the related type - lists_data=$(echo "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]') + lists_data=$(printf %s "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]') # construct a new json for the gravity results where each object contains the adlist URL and the related domains - gravity_data=$(echo "${data}" | jq '.search.gravity | group_by(.address) | map({ address: (.[0].address), domains: [.[] | .domain] })') + gravity_data=$(printf %s "${data}" | jq '.search.gravity | group_by(.address) | map({ address: (.[0].address), domains: [.[] | .domain] })') # number of objects in each json - num_gravity=$(echo "${gravity_data}" | jq length ) - num_lists=$(echo "${lists_data}" | jq length ) + num_gravity=$(printf %s "${gravity_data}" | jq length ) + num_lists=$(printf %s "${lists_data}" | jq length ) if [ "${partial}" = true ]; then search_type_str="partially" @@ -66,7 +66,7 @@ GenerateOutput(){ if [ "${num_lists}" -gt 0 ]; then # Convert the data to a csv, each line is a "domain,type" string # not using jq's @csv here as it quotes each value individually - lists_data_csv=$(echo "${lists_data}" | jq --raw-output '.[] | [.domain, .type] | join(",")' ) + lists_data_csv=$(printf %s "${lists_data}" | jq --raw-output '.[] | [.domain, .type] | join(",")' ) # Generate output for each csv line, separating line in a domain and type substring at the ',' echo "${lists_data_csv}" | while read -r line; do @@ -79,7 +79,7 @@ GenerateOutput(){ if [ "${num_gravity}" -gt 0 ]; then # Convert the data to a csv, each line is a "URL,domain,domain,...." string # not using jq's @csv here as it quotes each value individually - gravity_data_csv=$(echo "${gravity_data}" | jq --raw-output '.[] | [.address, .domains[]] | join(",")' ) + gravity_data_csv=$(printf %s "${gravity_data}" | jq --raw-output '.[] | [.address, .domains[]] | join(",")' ) # Generate line-by-line output for each csv line echo "${gravity_data_csv}" | while read -r line; do From 159817b7e286c4015030bfe61c6100b7d5b2c028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 6 Dec 2023 20:35:26 +0100 Subject: [PATCH 163/413] Use development-v6 as ftl branch for binary test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/test_any_automated_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 7d9d15f6..62588c0a 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -465,7 +465,7 @@ def test_FTL_development_binary_installed_and_responsive_no_errors(host): source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) - echo "development" > /etc/pihole/ftlbranch + echo "development-v6" > /etc/pihole/ftlbranch binary="pihole-FTL${funcOutput##*pihole-FTL}" theRest="${funcOutput%pihole-FTL*}" FTLdetect "${binary}" "${theRest}" From 8a71e4253e0733fec4a4cb805ea219ff8ec8dab2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Dec 2023 17:39:18 +0000 Subject: [PATCH 164/413] Bump actions/stale from 8.0.0 to 9.0.0 Bumps [actions/stale](https://github.com/actions/stale) from 8.0.0 to 9.0.0. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v8.0.0...v9.0.0) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yml | 2 +- .github/workflows/stale_pr.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index d9de09d2..74ee9bef 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -17,7 +17,7 @@ jobs: issues: write steps: - - uses: actions/stale@v8.0.0 + - uses: actions/stale@v9.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 30 diff --git a/.github/workflows/stale_pr.yml b/.github/workflows/stale_pr.yml index 2db2a25d..96650818 100644 --- a/.github/workflows/stale_pr.yml +++ b/.github/workflows/stale_pr.yml @@ -17,7 +17,7 @@ jobs: pull-requests: write steps: - - uses: actions/stale@v8.0.0 + - uses: actions/stale@v9.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} # Do not automatically mark PR/issue as stale From e49d7fa5f1dd19a80c798318a270e80f0a6d9415 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Dec 2023 17:39:42 +0000 Subject: [PATCH 165/413] Bump actions/setup-python from 4.7.1 to 5.0.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.7.1 to 5.0.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.7.1...v5.0.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8166d253..ec4da6bb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,7 +75,7 @@ jobs: uses: actions/checkout@v4.1.1 - name: Set up Python 3.10 - uses: actions/setup-python@v4.7.1 + uses: actions/setup-python@v5.0.0 with: python-version: "3.10" From a87d1bbc4fcf107753324f551ac54bfe7bc45a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 14:22:43 +0100 Subject: [PATCH 166/413] Remove pihole.sudo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole.sudo | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 advanced/Templates/pihole.sudo diff --git a/advanced/Templates/pihole.sudo b/advanced/Templates/pihole.sudo deleted file mode 100644 index 708309be..00000000 --- a/advanced/Templates/pihole.sudo +++ /dev/null @@ -1,9 +0,0 @@ -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Allows the WebUI to use Pi-hole commands -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. -# From 54186a63eee39a2e420dda8d170807e3492bbc3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 14:33:13 +0100 Subject: [PATCH 167/413] Remove test for sudo file as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/test_any_automated_install.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 62588c0a..c53070dc 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -174,10 +174,6 @@ def test_installPihole_fresh_install_readableFiles(host): ) actual_rc = host.run(check_man).rc assert exit_status_success == actual_rc - # check not readable sudoers file - check_sudo = test_cmd.format("r", "/etc/sudoers.d/pihole", piholeuser) - actual_rc = host.run(check_sudo).rc - assert exit_status_success != actual_rc # check not readable cron file check_sudo = test_cmd.format("x", "/etc/cron.d/", piholeuser) actual_rc = host.run(check_sudo).rc From 274d4c263c2ec4c8ef3d3b4a55bae45aa3fb694b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 21:23:39 +0100 Subject: [PATCH 168/413] Add bash-completion to PIHOLE_DEPS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 72f00992..f5bf15e6 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -332,7 +332,7 @@ package_manager_detect() { # Packages required to run this install script INSTALLER_DEPS=(git iproute2 dialog ca-certificates) # Packages required to run Pi-hole - PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq lshw) + PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq lshw bash-completion) # If apt-get is not found, check for rpm. elif is_command rpm ; then @@ -349,7 +349,7 @@ package_manager_detect() { PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src|.riscv64)' | wc -l || true" OS_CHECK_DEPS=(grep bind-utils) INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates binutils) - PIHOLE_DEPS=(cronie curl findutils sudo unzip psmisc libcap nmap-ncat jq lshw) + PIHOLE_DEPS=(cronie curl findutils sudo unzip psmisc libcap nmap-ncat jq lshw bash-completion) # If neither apt-get or yum/dnf package managers were found else From fe7299323c389ca86fe941ecadaee6ce082695a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 21:40:03 +0100 Subject: [PATCH 169/413] Update bash_completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/bash-completion/pihole | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 305a3f5b..b2740724 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -1,5 +1,5 @@ _pihole() { - local cur prev opts opts_admin opts_checkout opts_debug opts_interface opts_logging opts_privacy opts_query opts_update opts_version + local cur prev opts opts_checkout opts_debug opts_logging opts_query opts_update opts_version COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" @@ -7,17 +7,13 @@ _pihole() { case "${prev}" in "pihole") - opts="admin blacklist checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" + opts="blacklist checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) ;; "whitelist"|"blacklist"|"wildcard"|"regex") opts_lists="\--delmode \--noreload \--quiet \--list \--nuke" COMPREPLY=( $(compgen -W "${opts_lists}" -- ${cur}) ) ;; - "admin") - opts_admin="celsius fahrenheit interface kelvin password privacylevel" - COMPREPLY=( $(compgen -W "${opts_admin}" -- ${cur}) ) - ;; "checkout") opts_checkout="core ftl web master dev" COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) @@ -31,7 +27,7 @@ _pihole() { COMPREPLY=( $(compgen -W "${opts_logging}" -- ${cur}) ) ;; "query") - opts_query="-adlist -all -exact" + opts_query="--partial --all" COMPREPLY=( $(compgen -W "${opts_query}" -- ${cur}) ) ;; "updatePihole"|"-up") @@ -41,23 +37,7 @@ _pihole() { "version") opts_version="\--admin \--current \--ftl \--hash \--latest \--pihole" COMPREPLY=( $(compgen -W "${opts_version}" -- ${cur}) ) - ;; - "interface") - if ( [[ "$prev2" == "admin" ]] || [[ "$prev2" == "-a" ]] ); then - opts_interface="$(cat /proc/net/dev | cut -d: -s -f1)" - COMPREPLY=( $(compgen -W "${opts_interface}" -- ${cur}) ) - else - return 1 - fi - ;; - "privacylevel") - if ( [[ "$prev2" == "admin" ]] || [[ "$prev2" == "-a" ]] ); then - opts_privacy="0 1 2 3" - COMPREPLY=( $(compgen -W "${opts_privacy}" -- ${cur}) ) - else - return 1 - fi - ;; + ;; "core"|"admin"|"ftl") if [[ "$prev2" == "checkout" ]]; then opts_checkout="master dev" From 6d02d4056f6abd63ff3d9479979502b6523f715e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 22:01:30 +0100 Subject: [PATCH 170/413] Rename option --admin to --web in version function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/version.sh | 16 ++++++++-------- advanced/bash-completion/pihole | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index e3b4a6ae..2983c04e 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -24,7 +24,7 @@ fi getLocalVersion() { case ${1} in "Pi-hole" ) echo "${CORE_VERSION:=N/A}";; - "web" ) echo "${WEB_VERSION:=N/A}";; + "web" ) echo "${WEB_VERSION:=N/A}";; "FTL" ) echo "${FTL_VERSION:=N/A}";; esac } @@ -32,7 +32,7 @@ getLocalVersion() { getLocalHash() { case ${1} in "Pi-hole" ) echo "${CORE_HASH:=N/A}";; - "web" ) echo "${WEB_HASH:=N/A}";; + "web" ) echo "${WEB_HASH:=N/A}";; "FTL" ) echo "${FTL_HASH:=N/A}";; esac } @@ -40,7 +40,7 @@ getLocalHash() { getRemoteHash(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_HASH:=N/A}";; - "web" ) echo "${GITHUB_WEB_HASH:=N/A}";; + "web" ) echo "${GITHUB_WEB_HASH:=N/A}";; "FTL" ) echo "${GITHUB_FTL_HASH:=N/A}";; esac } @@ -48,7 +48,7 @@ getRemoteHash(){ getRemoteVersion(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_VERSION:=N/A}";; - "web" ) echo "${GITHUB_WEB_VERSION:=N/A}";; + "web" ) echo "${GITHUB_WEB_VERSION:=N/A}";; "FTL" ) echo "${GITHUB_FTL_VERSION:=N/A}";; esac } @@ -56,7 +56,7 @@ getRemoteVersion(){ getLocalBranch(){ case ${1} in "Pi-hole" ) echo "${CORE_BRANCH:=N/A}";; - "web" ) echo "${WEB_BRANCH:=N/A}";; + "web" ) echo "${WEB_BRANCH:=N/A}";; "FTL" ) echo "${FTL_BRANCH:=N/A}";; esac } @@ -114,11 +114,11 @@ defaultOutput() { helpFunc() { echo "Usage: pihole -v [repo | option] [option] Example: 'pihole -v -p -l' -Show Pi-hole, Admin Console & FTL versions +Show Pi-hole, Web Console & FTL versions Repositories: -p, --pihole Only retrieve info regarding Pi-hole repository - -a, --admin Only retrieve info regarding web repository + -w, --web Only retrieve info regarding web repository -f, --ftl Only retrieve info regarding FTL repository Options: @@ -131,7 +131,7 @@ Options: case "${1}" in "-p" | "--pihole" ) shift; versionOutput "Pi-hole" "$@";; - "-a" | "--admin" ) shift; versionOutput "web" "$@";; + "-w" | "--web" ) shift; versionOutput "web" "$@";; "-f" | "--ftl" ) shift; versionOutput "FTL" "$@";; "-h" | "--help" ) helpFunc;; * ) defaultOutput "$@";; diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index b2740724..4fe8f83a 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -35,7 +35,7 @@ _pihole() { COMPREPLY=( $(compgen -W "${opts_update}" -- ${cur}) ) ;; "version") - opts_version="\--admin \--current \--ftl \--hash \--latest \--pihole" + opts_version="\--web \--current \--ftl \--hash \--latest \--pihole" COMPREPLY=( $(compgen -W "${opts_version}" -- ${cur}) ) ;; "core"|"admin"|"ftl") From c90a27c5096d53f4be21044fdd71cb380806bc3f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 9 Dec 2023 23:06:50 +0100 Subject: [PATCH 171/413] Add "-ni" to all sqlite3 invocations Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 34 ++++++++-------- advanced/Scripts/list.sh | 20 +++++----- advanced/Scripts/piholeARPTable.sh | 4 +- advanced/Scripts/piholeDebug.sh | 12 +++--- advanced/Scripts/piholeLogFlush.sh | 2 +- gravity.sh | 39 +++++++++---------- 6 files changed, 55 insertions(+), 56 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index e36d9b1e..7c4deaa7 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -19,13 +19,13 @@ upgrade_gravityDB(){ auditFile="${piholeDir}/auditlog.list" # Get database version - version="$(pihole-FTL sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" + version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" if [[ "$version" == "1" ]]; then # This migration script upgrades the gravity.db file by # adding the domain_audit table echo -e " ${INFO} Upgrading gravity database from version 1 to 2" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/1_to_2.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/1_to_2.sql" version=2 # Store audit domains in database table @@ -40,28 +40,28 @@ upgrade_gravityDB(){ # renaming the regex table to regex_blacklist, and # creating a new regex_whitelist table + corresponding linking table and views echo -e " ${INFO} Upgrading gravity database from version 2 to 3" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/2_to_3.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/2_to_3.sql" version=3 fi if [[ "$version" == "3" ]]; then # This migration script unifies the formally separated domain # lists into a single table with a UNIQUE domain constraint echo -e " ${INFO} Upgrading gravity database from version 3 to 4" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/3_to_4.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/3_to_4.sql" version=4 fi if [[ "$version" == "4" ]]; then # This migration script upgrades the gravity and list views # implementing necessary changes for per-client blocking echo -e " ${INFO} Upgrading gravity database from version 4 to 5" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/4_to_5.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/4_to_5.sql" version=5 fi if [[ "$version" == "5" ]]; then # This migration script upgrades the adlist view # to return an ID used in gravity.sh echo -e " ${INFO} Upgrading gravity database from version 5 to 6" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/5_to_6.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/5_to_6.sql" version=6 fi if [[ "$version" == "6" ]]; then @@ -69,7 +69,7 @@ upgrade_gravityDB(){ # which is automatically associated to all clients not # having their own group assignments echo -e " ${INFO} Upgrading gravity database from version 6 to 7" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/6_to_7.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/6_to_7.sql" version=7 fi if [[ "$version" == "7" ]]; then @@ -77,21 +77,21 @@ upgrade_gravityDB(){ # to ensure uniqueness on the group name # We also add date_added and date_modified columns echo -e " ${INFO} Upgrading gravity database from version 7 to 8" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/7_to_8.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/7_to_8.sql" version=8 fi if [[ "$version" == "8" ]]; then # This migration fixes some issues that were introduced # in the previous migration script. echo -e " ${INFO} Upgrading gravity database from version 8 to 9" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/8_to_9.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/8_to_9.sql" version=9 fi if [[ "$version" == "9" ]]; then # This migration drops unused tables and creates triggers to remove # obsolete groups assignments when the linked items are deleted echo -e " ${INFO} Upgrading gravity database from version 9 to 10" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/9_to_10.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/9_to_10.sql" version=10 fi if [[ "$version" == "10" ]]; then @@ -101,44 +101,44 @@ upgrade_gravityDB(){ # to keep the copying process generic (needs the same columns in both the # source and the destination databases). echo -e " ${INFO} Upgrading gravity database from version 10 to 11" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/10_to_11.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/10_to_11.sql" version=11 fi if [[ "$version" == "11" ]]; then # Rename group 0 from "Unassociated" to "Default" echo -e " ${INFO} Upgrading gravity database from version 11 to 12" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/11_to_12.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/11_to_12.sql" version=12 fi if [[ "$version" == "12" ]]; then # Add column date_updated to adlist table echo -e " ${INFO} Upgrading gravity database from version 12 to 13" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/12_to_13.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/12_to_13.sql" version=13 fi if [[ "$version" == "13" ]]; then # Add columns number and status to adlist table echo -e " ${INFO} Upgrading gravity database from version 13 to 14" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/13_to_14.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/13_to_14.sql" version=14 fi if [[ "$version" == "14" ]]; then # Changes the vw_adlist created in 5_to_6 echo -e " ${INFO} Upgrading gravity database from version 14 to 15" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/14_to_15.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/14_to_15.sql" version=15 fi if [[ "$version" == "15" ]]; then # Add column abp_entries to adlist table echo -e " ${INFO} Upgrading gravity database from version 15 to 16" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/15_to_16.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/15_to_16.sql" version=16 fi if [[ "$version" == "16" ]]; then # Add antigravity table # Add column type to adlist table (to support adlist types) echo -e " ${INFO} Upgrading gravity database from version 16 to 17" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/16_to_17.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/16_to_17.sql" version=17 fi } diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index b76a7ef7..76558e58 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -150,18 +150,18 @@ AddDomain() { domain="$1" # Is the domain in the list we want to add it to? - num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")" + num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")" requestedListname="$(GetListnameFromTypeId "${typeId}")" if [[ "${num}" -ne 0 ]]; then - existingTypeId="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")" + existingTypeId="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")" if [[ "${existingTypeId}" == "${typeId}" ]]; then if [[ "${verbose}" == true ]]; then echo -e " ${INFO} ${1} already exists in ${requestedListname}, no need to add!" fi else existingListname="$(GetListnameFromTypeId "${existingTypeId}")" - pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';" if [[ "${verbose}" == true ]]; then echo -e " ${INFO} ${1} already exists in ${existingListname}, it has been moved to ${requestedListname}!" fi @@ -177,10 +177,10 @@ AddDomain() { # Insert only the domain here. The enabled and date_added fields will be filled # with their default values (enabled = true, date_added = current timestamp) if [[ -z "${comment}" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});" else # also add comment when variable has been set through the "--comment" option - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');" fi } @@ -189,7 +189,7 @@ RemoveDomain() { domain="$1" # Is the domain in the list we want to remove it from? - num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")" + num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")" requestedListname="$(GetListnameFromTypeId "${typeId}")" @@ -206,14 +206,14 @@ RemoveDomain() { fi reload=true # Remove it from the current list - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};" } Displaylist() { local count num_pipes domain enabled status nicedate requestedListname requestedListname="$(GetListnameFromTypeId "${typeId}")" - data="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)" + data="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)" if [[ -z $data ]]; then echo -e "Not showing empty list" @@ -251,10 +251,10 @@ Displaylist() { } NukeList() { - count=$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};") + count=$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};") listname="$(GetListnameFromTypeId "${typeId}")" if [ "$count" -gt 0 ];then - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};" echo " ${TICK} Removed ${count} domain(s) from the ${listname}" else echo " ${INFO} ${listname} already empty. Nothing to do!" diff --git a/advanced/Scripts/piholeARPTable.sh b/advanced/Scripts/piholeARPTable.sh index 5daa025d..b92dd124 100755 --- a/advanced/Scripts/piholeARPTable.sh +++ b/advanced/Scripts/piholeARPTable.sh @@ -39,7 +39,7 @@ flushARP(){ # Truncate network_addresses table in pihole-FTL.db # This needs to be done before we can truncate the network table due to # foreign key constraints - if ! output=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then + if ! output=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then echo -e "${OVER} ${CROSS} Failed to truncate network_addresses table" echo " Database location: ${DBFILE}" echo " Output: ${output}" @@ -47,7 +47,7 @@ flushARP(){ fi # Truncate network table in pihole-FTL.db - if ! output=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM network" 2>&1); then + if ! output=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM network" 2>&1); then echo -e "${OVER} ${CROSS} Failed to truncate network table" echo " Database location: ${DBFILE}" echo " Output: ${output}" diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 931e95a4..aa0e61c4 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -718,7 +718,7 @@ dig_at() { # This helps emulate queries to different domains that a user might query # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains local random_url - random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") + random_url=$(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") # Fallback if no non-ABP style domains were found if [ -z "${random_url}" ]; then random_url="flurry.com" @@ -1064,7 +1064,7 @@ show_db_entries() { IFS=$'\r\n' local entries=() mapfile -t entries < <(\ - pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" \ + pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" \ -cmd ".headers on" \ -cmd ".mode column" \ -cmd ".width ${widths}" \ @@ -1089,7 +1089,7 @@ show_FTL_db_entries() { IFS=$'\r\n' local entries=() mapfile -t entries < <(\ - pihole-FTL sqlite3 "${PIHOLE_FTL_DB_FILE}" \ + pihole-FTL sqlite3 -ni "${PIHOLE_FTL_DB_FILE}" \ -cmd ".headers on" \ -cmd ".mode column" \ -cmd ".width ${widths}" \ @@ -1155,7 +1155,7 @@ analyze_gravity_list() { fi show_db_entries "Info table" "SELECT property,value FROM info" "20 40" - gravity_updated_raw="$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT value FROM info where property = 'updated'")" + gravity_updated_raw="$(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT value FROM info where property = 'updated'")" gravity_updated="$(date -d @"${gravity_updated_raw}")" log_write " Last gravity run finished at: ${COL_CYAN}${gravity_updated}${COL_NC}" log_write "" @@ -1163,7 +1163,7 @@ analyze_gravity_list() { OLD_IFS="$IFS" IFS=$'\r\n' local gravity_sample=() - mapfile -t gravity_sample < <(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10") + mapfile -t gravity_sample < <(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10") log_write " ${COL_CYAN}----- First 10 Gravity Domains -----${COL_NC}" for line in "${gravity_sample[@]}"; do @@ -1195,7 +1195,7 @@ database_integrity_check(){ log_write "${INFO} Checking foreign key constraints of ${database} ... (this can take several minutes)" unset result - result="$(pihole-FTL sqlite3 "${database}" -cmd ".headers on" -cmd ".mode column" "PRAGMA foreign_key_check" 2>&1 & spinner)" + result="$(pihole-FTL sqlite3 -ni "${database}" -cmd ".headers on" -cmd ".mode column" "PRAGMA foreign_key_check" 2>&1 & spinner)" if [[ -z ${result} ]]; then log_write "${TICK} No foreign key errors in ${database}" else diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 14542e4b..4d97fec5 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -63,7 +63,7 @@ else fi fi # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history) - deleted=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1") + deleted=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1") # Restart pihole-FTL to force reloading history sudo pihole restartdns diff --git a/gravity.sh b/gravity.sh index 8dc8edd8..f51103ff 100755 --- a/gravity.sh +++ b/gravity.sh @@ -48,7 +48,6 @@ gravityDBcopy="${piholeGitDir}/advanced/Templates/gravity_copy.sql" domainsExtension="domains" curl_connect_timeout=10 - # Check gravity temp directory if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then echo -e " ${COL_LIGHT_RED}Gravity temporary directory does not exist or is not a writeable directory, falling back to /tmp. ${COL_NC}" @@ -64,7 +63,7 @@ gravityOLDfile="${gravityDIR}/gravity_old.db" # Generate new SQLite3 file from schema template generate_gravity_database() { - if ! pihole-FTL sqlite3 "${gravityDBfile}" < "${gravityDBschema}"; then + if ! pihole-FTL sqlite3 -ni "${gravityDBfile}" < "${gravityDBschema}"; then echo -e " ${CROSS} Unable to create ${gravityDBfile}" return 1 fi @@ -79,7 +78,7 @@ gravity_build_tree() { echo -ne " ${INFO} ${str}..." # The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once - output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -118,7 +117,7 @@ gravity_swap_databases() { # Update timestamp when the gravity table was last updated successfully update_gravity_timestamp() { - output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -163,7 +162,7 @@ database_table_from_file() { # Get MAX(id) from domainlist when INSERTing into this table if [[ "${table}" == "domainlist" ]]; then - rowid="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")" + rowid="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")" if [[ -z "$rowid" ]]; then rowid=0 fi @@ -193,7 +192,7 @@ database_table_from_file() { # Store domains in database table specified by ${table} # Use printf as .mode and .import need to be on separate lines # see https://unix.stackexchange.com/a/445615/83260 - output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 -ni "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -213,7 +212,7 @@ database_table_from_file() { # Check if a column with name ${2} exists in gravity table with name ${1} gravity_column_exists() { - output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) if [[ "${output}" == "1" ]]; then return 0 # Bash 0 is success fi @@ -228,7 +227,7 @@ database_adlist_number() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${2}" "${3}" "${1}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${2}" "${3}" "${1}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -244,7 +243,7 @@ database_adlist_status() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -362,9 +361,9 @@ gravity_DownloadBlocklists() { # Retrieve source URLs from gravity database # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true) - mapfile -t sources <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" - mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)" - mapfile -t sourceTypes <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2> /dev/null)" + mapfile -t sources <<< "$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" + mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)" + mapfile -t sourceTypes <<< "$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2> /dev/null)" # Parse source domains from $sources mapfile -t sourceDomains <<< "$( @@ -393,7 +392,7 @@ gravity_DownloadBlocklists() { str="Preparing new gravity database" echo -ne " ${INFO} ${str}..." rm "${gravityTEMPfile}" > /dev/null 2>&1 - output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -413,7 +412,7 @@ gravity_DownloadBlocklists() { copyGravity="${copyGravity//"${gravityDBfile_default}"/"${gravityDBfile}"}" fi - output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -663,12 +662,12 @@ gravity_Table_Count() { local table="${1}" local str="${2}" local num - num="$(pihole-FTL sqlite3 "${gravityTEMPfile}" "SELECT COUNT(*) FROM ${table};")" + num="$(pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM ${table};")" if [[ "${table}" == "gravity" ]]; then local unique - unique="$(pihole-FTL sqlite3 "${gravityTEMPfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" + unique="$(pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" echo -e " ${INFO} Number of ${str}: ${num} (${COL_BOLD}${unique} unique domains${COL_NC})" - pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" + pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" else echo -e " ${INFO} Number of ${str}: ${num}" fi @@ -749,7 +748,7 @@ database_recovery() { local str="Checking integrity of existing gravity database (this can take a while)" local option="${1}" echo -ne " ${INFO} ${str}..." - result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)" + result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)" if [[ ${result} = "ok" ]]; then echo -e "${OVER} ${TICK} ${str} - no errors found" @@ -757,7 +756,7 @@ database_recovery() { str="Checking foreign keys of existing gravity database (this can take a while)" echo -ne " ${INFO} ${str}..." unset result - result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)" + result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)" if [[ -z ${result} ]]; then echo -e "${OVER} ${TICK} ${str} - no errors found" if [[ "${option}" != "force" ]]; then @@ -776,7 +775,7 @@ database_recovery() { echo -ne " ${INFO} ${str}..." # We have to remove any possibly existing recovery database or this will fail rm -f "${gravityDBfile}.recovered" > /dev/null 2>&1 - if result="$(pihole-FTL sqlite3 "${gravityDBfile}" ".recover" | pihole-FTL sqlite3 "${gravityDBfile}.recovered" 2>&1)"; then + if result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" ".recover" | pihole-FTL sqlite3 -ni "${gravityDBfile}.recovered" 2>&1)"; then echo -e "${OVER} ${TICK} ${str} - success" mv "${gravityDBfile}" "${gravityDBfile}.old" mv "${gravityDBfile}.recovered" "${gravityDBfile}" From fe4d934a406cf4fa45a72668170868bd57448537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 22:31:36 +0100 Subject: [PATCH 172/413] Simplify pihole -v MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/version.sh | 138 ++++++++---------------------------- pihole | 5 +- 2 files changed, 31 insertions(+), 112 deletions(-) diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index 2983c04e..ff7f0815 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -8,6 +8,10 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. +# Ignore warning about `local` being undefinded in POSIX +# shellcheck disable=SC3043 +# https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions + # Source the versions file poupulated by updatechecker.sh cachedVersions="/etc/pihole/versions" @@ -21,118 +25,34 @@ else . "$cachedVersions" fi -getLocalVersion() { - case ${1} in - "Pi-hole" ) echo "${CORE_VERSION:=N/A}";; - "web" ) echo "${WEB_VERSION:=N/A}";; - "FTL" ) echo "${FTL_VERSION:=N/A}";; - esac -} +main() { + local details + details=$1 -getLocalHash() { - case ${1} in - "Pi-hole" ) echo "${CORE_HASH:=N/A}";; - "web" ) echo "${WEB_HASH:=N/A}";; - "FTL" ) echo "${FTL_HASH:=N/A}";; - esac -} - -getRemoteHash(){ - case ${1} in - "Pi-hole" ) echo "${GITHUB_CORE_HASH:=N/A}";; - "web" ) echo "${GITHUB_WEB_HASH:=N/A}";; - "FTL" ) echo "${GITHUB_FTL_HASH:=N/A}";; - esac -} - -getRemoteVersion(){ - case ${1} in - "Pi-hole" ) echo "${GITHUB_CORE_VERSION:=N/A}";; - "web" ) echo "${GITHUB_WEB_VERSION:=N/A}";; - "FTL" ) echo "${GITHUB_FTL_VERSION:=N/A}";; - esac -} - -getLocalBranch(){ - case ${1} in - "Pi-hole" ) echo "${CORE_BRANCH:=N/A}";; - "web" ) echo "${WEB_BRANCH:=N/A}";; - "FTL" ) echo "${FTL_BRANCH:=N/A}";; - esac -} - -versionOutput() { - - [ "$2" = "-c" ] || [ "$2" = "--current" ] || [ -z "$2" ] && current=$(getLocalVersion "${1}") && branch=$(getLocalBranch "${1}") - [ "$2" = "-l" ] || [ "$2" = "--latest" ] || [ -z "$2" ] && latest=$(getRemoteVersion "${1}") - if [ "$2" = "--hash" ]; then - [ "$3" = "-c" ] || [ "$3" = "--current" ] || [ -z "$3" ] && curHash=$(getLocalHash "${1}") && branch=$(getLocalBranch "${1}") - [ "$3" = "-l" ] || [ "$3" = "--latest" ] || [ -z "$3" ] && latHash=$(getRemoteHash "${1}") && branch=$(getLocalBranch "${1}") - fi - - # We do not want to show the branch name when we are on master, - # blank out the variable in this case - if [ "$branch" = "master" ]; then - branch="" + if [ "${details}" = true ]; then + echo "Core" + echo " Version is ${CORE_VERSION:=N/A} (Latest: ${GITHUB_CORE_VERSION:=N/A})" + echo " Branch is ${CORE_BRANCH:=N/A}" + echo " Hash is ${CORE_HASH:=N/A} (Latest: ${GITHUB_CORE_HASH:=N/A})" + echo "Web" + echo " Version is ${WEB_VERSION:=N/A} (Latest: ${GITHUB_WEB_VERSION:=N/A})" + echo " Branch is ${WEB_BRANCH:=N/A}" + echo " Hash is ${WEB_HASH:=N/A} (Latest: ${GITHUB_WEB_HASH:=N/A})" + echo "FTL" + echo " Version is ${FTL_VERSION:=N/A} (Latest: ${GITHUB_FTL_VERSION:=N/A})" + echo " Branch is ${FTL_BRANCH:=N/A}" + echo " Hash is ${FTL_HASH:=N/A} (Latest: ${GITHUB_FTL_HASH:=N/A})" else - branch="$branch " + echo "Core version is ${CORE_VERSION:=N/A} (Latest: ${GITHUB_CORE_VERSION:=N/A})" + echo "Web version is ${WEB_VERSION:=N/A} (Latest: ${GITHUB_WEB_VERSION:=N/A})" + echo "FTL version is ${FTL_VERSION:=N/A} (Latest: ${GITHUB_FTL_VERSION:=N/A})" fi - - if [ -n "$current" ] && [ -n "$latest" ]; then - output="${1} version is $branch$current (Latest: $latest)" - elif [ -n "$current" ] && [ -z "$latest" ]; then - output="Current ${1} version is $branch$current" - elif [ -z "$current" ] && [ -n "$latest" ]; then - output="Latest ${1} version is $latest" - elif [ -n "$curHash" ] && [ -n "$latHash" ]; then - output="Local ${1} hash is $curHash (Remote: $latHash)" - elif [ -n "$curHash" ] && [ -z "$latHash" ]; then - output="Current local ${1} hash is $curHash" - elif [ -z "$curHash" ] && [ -n "$latHash" ]; then - output="Latest remote ${1} hash is $latHash" - elif [ -z "$curHash" ] && [ -z "$latHash" ]; then - output="Hashes for ${1} not available" - else - errorOutput - return 1 - fi - - [ -n "$output" ] && echo " $output" } -errorOutput() { - echo " Invalid Option! Try 'pihole -v --help' for more information." - exit 1 -} - -defaultOutput() { - versionOutput "Pi-hole" "$@" - versionOutput "web" "$@" - versionOutput "FTL" "$@" -} - -helpFunc() { - echo "Usage: pihole -v [repo | option] [option] -Example: 'pihole -v -p -l' -Show Pi-hole, Web Console & FTL versions - -Repositories: - -p, --pihole Only retrieve info regarding Pi-hole repository - -w, --web Only retrieve info regarding web repository - -f, --ftl Only retrieve info regarding FTL repository - -Options: - -c, --current Return the current version - -l, --latest Return the latest version - --hash Return the GitHub hash from your local repositories - -h, --help Show this help dialog" - exit 0 -} - -case "${1}" in - "-p" | "--pihole" ) shift; versionOutput "Pi-hole" "$@";; - "-w" | "--web" ) shift; versionOutput "web" "$@";; - "-f" | "--ftl" ) shift; versionOutput "FTL" "$@";; - "-h" | "--help" ) helpFunc;; - * ) defaultOutput "$@";; +# Process options (if present) +case "$1" in + "-vv" ) details=true;; + * ) details=false;; esac + +main "${details}" diff --git a/pihole b/pihole index 47da4ddd..ef93146c 100755 --- a/pihole +++ b/pihole @@ -140,7 +140,6 @@ uninstallFunc() { } versionFunc() { - shift exec "${PI_HOLE_SCRIPT_DIR}"/version.sh "$@" } @@ -508,7 +507,7 @@ Options: -up, updatePihole Update Pi-hole subsystems Add '--check-only' to exit script before update is performed. -v, version Show installed versions of Pi-hole, Web Interface & FTL - Add '-h' for more info on version usage + Use -vv for detailed information. uninstall Uninstall Pi-hole from your system status Display the running status of Pi-hole subsystems enable Enable Pi-hole subsystems @@ -531,7 +530,7 @@ fi need_root=1 case "${1}" in "-h" | "help" | "--help" ) helpFunc;; - "-v" | "version" ) versionFunc "$@";; + "-v" | "-vv" | "version" ) versionFunc "$@";; "-c" | "chronometer" ) chronometerFunc "$@";; "-q" | "query" ) queryFunc "$@";; "status" ) statusFunc "$2";; From b333e30162a07287bcf313ec9da01e1644c72d52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 22:55:01 +0100 Subject: [PATCH 173/413] Remove version options from bash completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/bash-completion/pihole | 4 ---- 1 file changed, 4 deletions(-) diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 4fe8f83a..89e02d2f 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -34,10 +34,6 @@ _pihole() { opts_update="--check-only" COMPREPLY=( $(compgen -W "${opts_update}" -- ${cur}) ) ;; - "version") - opts_version="\--web \--current \--ftl \--hash \--latest \--pihole" - COMPREPLY=( $(compgen -W "${opts_version}" -- ${cur}) ) - ;; "core"|"admin"|"ftl") if [[ "$prev2" == "checkout" ]]; then opts_checkout="master dev" From c3c31a1a60dcf68ae3b08a2c474485fdbd3c9cb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 23:09:58 +0100 Subject: [PATCH 174/413] Print version details automatically if not on master MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/version.sh | 16 ++++++++-------- pihole | 5 ++--- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index ff7f0815..2422ea6d 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -27,7 +27,13 @@ fi main() { local details - details=$1 + details=false + + # Automatically show detailed information if + # at least one of the components is not on master branch + if [ ! "${CORE_BRANCH}" = "master" ] || [ ! "${WEB_BRANCH}" = "master" ] || [ ! "${FTL_BRANCH}" = "master" ] ; then + details=true + fi if [ "${details}" = true ]; then echo "Core" @@ -49,10 +55,4 @@ main() { fi } -# Process options (if present) -case "$1" in - "-vv" ) details=true;; - * ) details=false;; -esac - -main "${details}" +main diff --git a/pihole b/pihole index ef93146c..7c84771c 100755 --- a/pihole +++ b/pihole @@ -140,7 +140,7 @@ uninstallFunc() { } versionFunc() { - exec "${PI_HOLE_SCRIPT_DIR}"/version.sh "$@" + exec "${PI_HOLE_SCRIPT_DIR}"/version.sh } restartDNS() { @@ -507,7 +507,6 @@ Options: -up, updatePihole Update Pi-hole subsystems Add '--check-only' to exit script before update is performed. -v, version Show installed versions of Pi-hole, Web Interface & FTL - Use -vv for detailed information. uninstall Uninstall Pi-hole from your system status Display the running status of Pi-hole subsystems enable Enable Pi-hole subsystems @@ -530,7 +529,7 @@ fi need_root=1 case "${1}" in "-h" | "help" | "--help" ) helpFunc;; - "-v" | "-vv" | "version" ) versionFunc "$@";; + "-v" | "version" ) versionFunc;; "-c" | "chronometer" ) chronometerFunc "$@";; "-q" | "query" ) queryFunc "$@";; "status" ) statusFunc "$2";; From 0cfcdc4b50e16612bf7572f290c1960c0f39e873 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 16 Dec 2023 10:08:32 +0000 Subject: [PATCH 175/413] Bump github/codeql-action from 2 to 3 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2 to 3. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v2...v3) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6544db61..fc821194 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -29,12 +29,12 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: 'python' - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 From 0066c6fbffa889525751478468b1cfba77272e20 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Dec 2023 05:29:11 +0100 Subject: [PATCH 176/413] Make IDs of anti-/gravity lists available in vw_(anti)gravity Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 6 +++++ .../database_migration/gravity/17_to_18.sql | 25 +++++++++++++++++++ advanced/Templates/gravity.db.sql | 6 ++--- gravity.sh | 2 +- 4 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 advanced/Scripts/database_migration/gravity/17_to_18.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 7c4deaa7..b263b40d 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -141,4 +141,10 @@ upgrade_gravityDB(){ pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/16_to_17.sql" version=17 fi + if [[ "$version" == "17" ]]; then + # Add adlist.id to vw_gravity and vw_antigravity + echo -e " ${INFO} Upgrading gravity database from version 17 to 18" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/17_to_18.sql" + version=18 + fi } diff --git a/advanced/Scripts/database_migration/gravity/17_to_18.sql b/advanced/Scripts/database_migration/gravity/17_to_18.sql new file mode 100644 index 00000000..00171a9a --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/17_to_18.sql @@ -0,0 +1,25 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +DROP VIEW vw_gravity; +CREATE VIEW vw_gravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id + FROM gravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = gravity.adlist_id + LEFT JOIN adlist ON adlist.id = gravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1); + +DROP VIEW vw_antigravity; +CREATE VIEW vw_antigravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id + FROM antigravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id + LEFT JOIN adlist ON adlist.id = antigravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; + +UPDATE info SET value = 18 WHERE property = 'version'; + +COMMIT; diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 46f26ba7..097b0a78 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -65,7 +65,7 @@ CREATE TABLE info value TEXT NOT NULL ); -INSERT INTO "info" VALUES('version','17'); +INSERT INTO "info" VALUES('version','18'); CREATE TABLE domain_audit ( @@ -144,14 +144,14 @@ CREATE VIEW vw_regex_blacklist AS SELECT domain, domainlist.id AS id, domainlist AND domainlist.type = 3 ORDER BY domainlist.id; -CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id +CREATE VIEW vw_gravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id FROM gravity LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = gravity.adlist_id LEFT JOIN adlist ON adlist.id = gravity.adlist_id LEFT JOIN "group" ON "group".id = adlist_by_group.group_id WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1); -CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id +CREATE VIEW vw_antigravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id FROM antigravity LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id LEFT JOIN adlist ON adlist.id = antigravity.adlist_id diff --git a/gravity.sh b/gravity.sh index f51103ff..ebe7e740 100755 --- a/gravity.sh +++ b/gravity.sh @@ -78,7 +78,7 @@ gravity_build_tree() { echo -ne " ${INFO} ${str}..." # The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once - output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) + output=$( { time pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } ) status="$?" if [[ "${status}" -ne 0 ]]; then From 8e8c7ecad212d45548dc89f408d2545d332a37a5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Dec 2023 05:41:43 +0100 Subject: [PATCH 177/413] Run gravity upgrade on checkout/update Signed-off-by: DL6ER --- automated install/basic-install.sh | 5 +++++ gravity.sh | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f5bf15e6..5863e818 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2225,6 +2225,11 @@ main() { # but before starting or restarting the ftl service disable_resolved_stublistener + # Check if gravity database needs to be upgraded. If so, do it without rebuilding + # gravity altogether. This may be a very long running task needlessly blocking + # the update process. + /opt/pihole/gravity.sh --upgrade + printf " %b Restarting services...\\n" "${INFO}" # Start services diff --git a/gravity.sh b/gravity.sh index ebe7e740..216a67a5 100755 --- a/gravity.sh +++ b/gravity.sh @@ -78,7 +78,7 @@ gravity_build_tree() { echo -ne " ${INFO} ${str}..." # The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once - output=$( { time pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -831,6 +831,7 @@ for var in "$@"; do case "${var}" in "-f" | "--force" ) forceDelete=true;; "-r" | "--repair" ) repairSelector "$3";; + "-u" | "--upgrade" ) upgrade_gravityDB "${gravityDBfile}" "${piholeDir}"; exit 0;; "-h" | "--help" ) helpFunc;; esac done From 1b87ec067ef8d19df0cc6af9f4aa30ba69696d3c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Dec 2023 06:12:18 +0100 Subject: [PATCH 178/413] Exit early if the database does not exist (e.g. in some CI tests) Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity-db.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index b263b40d..7b356b3a 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -18,6 +18,11 @@ upgrade_gravityDB(){ piholeDir="${2}" auditFile="${piholeDir}/auditlog.list" + # Exit early if the database does not exist (e.g. in CI tests) + if [[ ! -f "${database}" ]]; then + return + fi + # Get database version version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" From 0c6363572b2ddde8c44f3bd5d06dfff92f929380 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Dec 2023 22:00:49 +0100 Subject: [PATCH 179/413] Append the interface to the gateway address if it is a link-local address Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index aa0e61c4..a78f94ee 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -547,17 +547,24 @@ ping_gateway() { ping_ipv4_or_ipv6 "${protocol}" # Check if we are using IPv4 or IPv6 # Find the default gateways using IPv4 or IPv6 - local gateway + local gateway gateway_addr gateway_iface log_write "${INFO} Default IPv${protocol} gateway(s):" while IFS= read -r gateway; do - log_write " ${gateway}" - done < <(ip -"${protocol}" route | grep default | cut -d ' ' -f 3) + log_write " $(cut -d ' ' -f 3 <<< "${gateway}")%$(cut -d ' ' -f 5 <<< "${gateway}")" + done < <(ip -"${protocol}" route | grep default) - gateway=$(ip -"${protocol}" route | grep default | cut -d ' ' -f 3 | head -n 1) + gateway_addr=$(ip -"${protocol}" route | grep default | cut -d ' ' -f 3 | head -n 1) + gateway_iface=$(ip -"${protocol}" route | grep default | cut -d ' ' -f 5 | head -n 1) # If there was at least one gateway - if [ -n "${gateway}" ]; then + if [ -n "${gateway_addr}" ]; then + # Append the interface to the gateway address if it is a link-local address + if [[ "${gateway_addr}" =~ ^fe80 ]]; then + gateway="${gateway_addr}%${gateway_iface}" + else + gateway="${gateway_addr}" + fi # Let the user know we will ping the gateway for a response log_write " * Pinging first gateway ${gateway}..." # Try to quietly ping the gateway 3 times, with a timeout of 3 seconds, using numeric output only, From 00340136bdc517cc71d4458fb5ccd8d784724767 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 26 Dec 2023 21:35:24 +0100 Subject: [PATCH 180/413] Extend %iface logic to the dig test. Also fix intentation in this function Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 39 +++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index a78f94ee..2c3ebb14 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -764,24 +764,29 @@ dig_at() { # Removes CIDR and everything thereafter (e.g., scope properties) addresses="$(ip address show dev "${iface}" | sed "/${sed_selector} /!d;s/^.*${sed_selector} //g;s/\/.*$//g;")" if [ -n "${addresses}" ]; then - while IFS= read -r local_address ; do + while IFS= read -r local_address ; do + # If ${local_address} is an IPv6 link-local address, append the interface name to it + if [[ "${local_address}" =~ ^fe80 ]]; then + local_address="${local_address}%${iface}" + fi + # Check if Pi-hole can use itself to block a domain - if local_dig="$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${local_address}" "${record_type}")"; then - # If it can, show success - if [[ "${local_dig}" == *"status: NOERROR"* ]]; then - local_dig="NOERROR" - elif [[ "${local_dig}" == *"status: NXDOMAIN"* ]]; then - local_dig="NXDOMAIN" - else - # Extract the first entry in the answer section from dig's output, - # replacing any multiple spaces and tabs with a single space - local_dig="$(echo "${local_dig}" | grep -A1 "ANSWER SECTION" | grep -v "ANSWER SECTION" | tr -s " \t" " ")" - fi - log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} on ${COL_CYAN}${iface}${COL_NC} (${COL_CYAN}${local_address}${COL_NC})" - else - # Otherwise, show a failure - log_write "${CROSS} ${COL_RED}Failed to resolve${COL_NC} ${random_url} on ${COL_RED}${iface}${COL_NC} (${COL_RED}${local_address}${COL_NC})" - fi + if local_dig="$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${local_address}" "${record_type}")"; then + # If it can, show success + if [[ "${local_dig}" == *"status: NOERROR"* ]]; then + local_dig="NOERROR" + elif [[ "${local_dig}" == *"status: NXDOMAIN"* ]]; then + local_dig="NXDOMAIN" + else + # Extract the first entry in the answer section from dig's output, + # replacing any multiple spaces and tabs with a single space + local_dig="$(echo "${local_dig}" | grep -A1 "ANSWER SECTION" | grep -v "ANSWER SECTION" | tr -s " \t" " ")" + fi + log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} on ${COL_CYAN}${iface}${COL_NC} (${COL_CYAN}${local_address}${COL_NC})" + else + # Otherwise, show a failure + log_write "${CROSS} ${COL_RED}Failed to resolve${COL_NC} ${random_url} on ${COL_RED}${iface}${COL_NC} (${COL_RED}${local_address}${COL_NC})" + fi done <<< "${addresses}" else log_write "${TICK} No IPv${protocol} address available on ${COL_CYAN}${iface}${COL_NC}" From d2828310f240aed098e5b3d787f5d63f47575a7a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 28 Dec 2023 11:12:20 +0100 Subject: [PATCH 181/413] Modify DELETE triggers to delete BEFORE instead of AFTER to prevent possible foreign key constraint violations Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 7 +++++ .../database_migration/gravity/18_to_19.sql | 27 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 advanced/Scripts/database_migration/gravity/18_to_19.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 7b356b3a..e99f1df2 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -152,4 +152,11 @@ upgrade_gravityDB(){ pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/17_to_18.sql" version=18 fi + if [[ "$version" == "18" ]]; then + # Modify DELETE triggers to delete BEFORE instead of AFTER to prevent + # foreign key constraint violations + echo -e " ${INFO} Upgrading gravity database from version 18 to 19" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/18_to_19.sql" + version=19 + fi } diff --git a/advanced/Scripts/database_migration/gravity/18_to_19.sql b/advanced/Scripts/database_migration/gravity/18_to_19.sql new file mode 100644 index 00000000..c85a4d57 --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/18_to_19.sql @@ -0,0 +1,27 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +DROP TRIGGER tr_domainlist_delete; +CREATE TRIGGER tr_domainlist_delete BEFORE DELETE ON domainlist + BEGIN + DELETE FROM domainlist_by_group WHERE domainlist_id = OLD.id; + END; + +DROP TRIGGER tr_adlist_delete; +CREATE TRIGGER tr_adlist_delete BEFORE DELETE ON adlist + BEGIN + DELETE FROM adlist_by_group WHERE adlist_id = OLD.id; + END; + +DROP TRIGGER tr_client_delete; +CREATE TRIGGER tr_client_delete BEFORE DELETE ON client + BEGIN + DELETE FROM client_by_group WHERE client_id = OLD.id; + END; + +UPDATE info SET value = 19 WHERE property = 'version'; + +COMMIT; From 9c3578856f1c47887d7af5f1bc972ce44f5316ec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Jan 2024 10:02:39 +0000 Subject: [PATCH 182/413] Bump pytest from 7.4.3 to 7.4.4 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.3 to 7.4.4. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.3...7.4.4) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index bfc6d027..0873d097 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.3 +pytest == 7.4.4 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.11.4 From 220c0675ef67adea60161fdd38f409736bcaaac0 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 7 Jan 2024 21:03:48 +0000 Subject: [PATCH 183/413] As we do in other tests, specify `development-v6` branch of FTL is downloaded for the tests. We need to change this to `develoment` once v6 is released Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index c53070dc..ab301a6e 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -239,6 +239,7 @@ def test_FTL_detect_aarch64_no_errors(host): mock_command("uname", {"-m": ("aarch64", "0")}, host) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -272,6 +273,7 @@ def test_FTL_detect_armv6_no_errors(host): ) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -305,6 +307,7 @@ def test_FTL_detect_armv7l_no_errors(host): ) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -338,6 +341,7 @@ def test_FTL_detect_armv7_no_errors(host): ) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -371,6 +375,7 @@ def test_FTL_detect_armv8a_no_errors(host): ) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -393,6 +398,7 @@ def test_FTL_detect_x86_64_no_errors(host): """ detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -415,6 +421,7 @@ def test_FTL_detect_unknown_no_errors(host): mock_command("uname", {"-m": ("mips", "0")}, host) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -442,6 +449,7 @@ def test_FTL_download_aarch64_no_errors(host): ) download_binary = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user FTLinstall "pihole-FTL-aarch64-linux-gnu" @@ -458,6 +466,7 @@ def test_FTL_development_binary_installed_and_responsive_no_errors(host): """ host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) From df9c60e3511f7871c1392f19bebbef1112be64ac Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 8 Jan 2024 18:26:59 +0000 Subject: [PATCH 184/413] Reduce code duplication in FTL arch detect tests. Use parametrize to run the same test with different parameters for each arch we need to test. Also include detection of unusupported in this test. Create FTL_BRANCH constant to be used in functions (less places to remember to change it) Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 253 +++++------------------------ 1 file changed, 37 insertions(+), 216 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index ab301a6e..0930f0af 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -12,6 +12,8 @@ from .conftest import ( run_script, ) +FTL_BRANCH = "development-v6" + def test_supported_package_manager(host): """ @@ -80,11 +82,7 @@ def test_installPihole_fresh_install_readableFiles(host): host.run("command -v dnf > /dev/null && dnf install -y man") host.run("command -v yum > /dev/null && yum install -y man") # Workaround to get FTLv6 installed until it reaches master branch - host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - """ - ) + host.run('echo "' + FTL_BRANCH + '" > /etc/pihole/ftlbranch') install = host.run( """ export TERM=xterm @@ -231,49 +229,37 @@ def test_update_package_cache_failure_no_errors(host): assert "Error: Unable to update package cache." in updateCache.stdout -def test_FTL_detect_aarch64_no_errors(host): +@pytest.mark.parametrize( + "arch,detected_string,supported", + [ + ("aarch64", "AArch64 (64 Bit ARM)", True), + ("armv6", "ARMv6", True), + ("armv7l", "ARMv7 (or newer)", True), + ("armv7", "ARMv7 (or newer)", True), + ("armv8a", "ARMv7 (or newer)", True), + ("x86_64", "x86_64", True), + ("riscv64", "riscv64", True), + ("mips", "mips", False), + ], +) +def test_FTL_detect_no_errors(host, arch, detected_string, supported): """ - confirms only aarch64 package is downloaded for FTL engine + confirms only correct package is downloaded for FTL engine """ - # mock uname to return aarch64 platform - mock_command("uname", {"-m": ("aarch64", "0")}, host) - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected AArch64 (64 Bit ARM) architecture" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv6_no_errors(host): - """ - confirms only armv6 package is downloaded for FTL engine - """ - # mock uname to return armv6 platform - mock_command("uname", {"-m": ("armv6", "0")}, host) - # mock readelf to respond with armv6l CPU architecture + # mock uname to return passed platform + mock_command("uname", {"-m": (arch, "0")}, host) + # mock readelf to respond with passed CPU architecture mock_command_2( "readelf", { - "-A /bin/sh": ("Tag_CPU_arch: armv6", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv6", "0"), + "-A /bin/sh": ("Tag_CPU_arch: " + arch, "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: " + arch, "0"), }, host, ) + host.run('echo "' + FTL_BRANCH + '" > /etc/pihole/ftlbranch') detectPlatform = host.run( """ - echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -282,195 +268,30 @@ def test_FTL_detect_armv6_no_errors(host): FTLdetect "${binary}" "${theRest}" """ ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv6 architecture" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv7l_no_errors(host): - """ - confirms only armv7l package is downloaded for FTL engine - """ - # mock uname to return armv7l platform - mock_command("uname", {"-m": ("armv7l", "0")}, host) - # mock readelf to respond with armv7l CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv7l", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv7l", "0"), - }, - host, - ) - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv7 (or newer) architecture") - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv7_no_errors(host): - """ - confirms only armv7 package is downloaded for FTL engine - """ - # mock uname to return armv7 platform - mock_command("uname", {"-m": ("armv7", "0")}, host) - # mock readelf to respond with armv7 CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv7", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv7", "0"), - }, - host, - ) - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv7 (or newer) architecture") - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv8a_no_errors(host): - """ - confirms only armv8a package is downloaded for FTL engine - """ - # mock uname to return armv8a platform - mock_command("uname", {"-m": ("armv8a", "0")}, host) - # mock readelf to respond with armv8a CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv8a", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv8a", "0"), - }, - host, - ) - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv7 (or newer) architecture (armv8a)" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_x86_64_no_errors(host): - """ - confirms only x86_64 package is downloaded for FTL engine - """ - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected x86_64 architecture" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_unknown_no_errors(host): - """confirms only generic package is downloaded for FTL engine""" - # mock uname to return generic platform - mock_command("uname", {"-m": ("mips", "0")}, host) - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = "Not able to detect architecture (unknown: mips)" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_download_aarch64_no_errors(host): - """ - confirms only aarch64 package is downloaded for FTL engine - """ - # mock dialog answers and ensure installer dependencies - mock_command("dialog", {"*": ("", "0")}, host) - host.run( - """ - source /opt/pihole/basic-install.sh - package_manager_detect - install_dependent_packages ${INSTALLER_DEPS[@]} - """ - ) - download_binary = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - FTLinstall "pihole-FTL-aarch64-linux-gnu" - """ - ) - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in download_binary.stdout - assert "error" not in download_binary.stdout.lower() + if supported: + expected_stdout = info_box + " FTL Checks..." + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Detected " + detected_string + " architecture" + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Downloading and Installing FTL" + assert expected_stdout in detectPlatform.stdout + else: + expected_stdout = ( + "Not able to detect architecture (unknown: " + detected_string + ")" + ) + assert expected_stdout in detectPlatform.stdout def test_FTL_development_binary_installed_and_responsive_no_errors(host): """ confirms FTL development binary is copied and functional in installed location """ + host.run('echo "' + FTL_BRANCH + '" > /etc/pihole/ftlbranch') host.run( """ - echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) - echo "development-v6" > /etc/pihole/ftlbranch binary="pihole-FTL${funcOutput##*pihole-FTL}" theRest="${funcOutput%pihole-FTL*}" FTLdetect "${binary}" "${theRest}" From debab10792c7bee8cd6ba18b6c3798c4f03c8a65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 7 Jan 2024 21:37:58 +0100 Subject: [PATCH 185/413] Use 204 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/api.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index fe73a608..18a48ce7 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -113,9 +113,8 @@ DeleteSession() { deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in - "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; + "204") printf "%b" "Session successfully deleted.\n";; "401") printf "%b" "Logout attempt without a valid session. Unauthorized!\n";; - "410") printf "%b" "Session successfully deleted.\n";; esac; fi From b662fd6f0101193a37b79377b077ffa37c33dfc2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 13 Jan 2024 10:43:13 +0000 Subject: [PATCH 186/413] Bump tox from 4.11.4 to 4.12.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.11.4 to 4.12.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.11.4...4.12.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 0873d097..59fd13cc 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.4 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.11.4 +tox == 4.12.0 From 47f06dfd71824f199af2a9b757f6ac06f9724ef4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 15 Jan 2024 20:53:34 +0100 Subject: [PATCH 187/413] Do not store remote version in versions file if on custom branch. It's always wrong here Signed-off-by: DL6ER --- advanced/Scripts/updatecheck.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 66f1a7ab..d2a338a7 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -26,7 +26,12 @@ function get_local_hash() { } function get_remote_version() { - curl -s "https://api.github.com/repos/pi-hole/${1}/releases/latest" 2> /dev/null | jq --raw-output .tag_name || return 1 + # if ${2} is = "master" we need to use the "latest" endpoint, otherwise, we simply return null + if [[ "${2}" == "master" ]]; then + curl -s "https://api.github.com/repos/pi-hole/${1}/releases/latest" 2> /dev/null | jq --raw-output .tag_name || return 1 + else + echo "null" + fi } @@ -73,7 +78,7 @@ addOrEditKeyValPair "${VERSION_FILE}" "CORE_BRANCH" "${CORE_BRANCH}" CORE_HASH="$(get_local_hash /etc/.pihole)" addOrEditKeyValPair "${VERSION_FILE}" "CORE_HASH" "${CORE_HASH}" -GITHUB_CORE_VERSION="$(get_remote_version pi-hole)" +GITHUB_CORE_VERSION="$(get_remote_version pi-hole "${CORE_BRANCH}")" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_CORE_VERSION" "${GITHUB_CORE_VERSION}" GITHUB_CORE_HASH="$(get_remote_hash pi-hole "${CORE_BRANCH}")" @@ -91,7 +96,7 @@ addOrEditKeyValPair "${VERSION_FILE}" "WEB_BRANCH" "${WEB_BRANCH}" WEB_HASH="$(get_local_hash /var/www/html/admin)" addOrEditKeyValPair "${VERSION_FILE}" "WEB_HASH" "${WEB_HASH}" -GITHUB_WEB_VERSION="$(get_remote_version web)" +GITHUB_WEB_VERSION="$(get_remote_version web "${WEB_BRANCH}")" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_VERSION" "${GITHUB_WEB_VERSION}" GITHUB_WEB_HASH="$(get_remote_hash web "${WEB_BRANCH}")" @@ -108,7 +113,7 @@ addOrEditKeyValPair "${VERSION_FILE}" "FTL_BRANCH" "${FTL_BRANCH}" FTL_HASH="$(pihole-FTL --hash)" addOrEditKeyValPair "${VERSION_FILE}" "FTL_HASH" "${FTL_HASH}" -GITHUB_FTL_VERSION="$(get_remote_version FTL)" +GITHUB_FTL_VERSION="$(get_remote_version FTL "${FTL_BRANCH}")" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_FTL_VERSION" "${GITHUB_FTL_VERSION}" GITHUB_FTL_HASH="$(get_remote_hash FTL "${FTL_BRANCH}")" From 60de50bb73f3df5f78c68726c3cd31ba1d5193ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Jan 2024 10:49:27 +0000 Subject: [PATCH 188/413] Bump tox from 4.12.0 to 4.12.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.12.0 to 4.12.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.12.0...4.12.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 59fd13cc..dea4ace9 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.4 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.12.0 +tox == 4.12.1 From 935a4ce0b3262fa3b3d468f018771a5001ffe145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 28 Jan 2024 16:56:57 +0100 Subject: [PATCH 189/413] Also remove Fedora 37 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 3 +-- test/_fedora_37.Dockerfile | 18 ------------------ test/tox.fedora_37.ini | 8 -------- 3 files changed, 1 insertion(+), 28 deletions(-) delete mode 100644 test/_fedora_37.Dockerfile delete mode 100644 test/tox.fedora_37.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9d90b2e7..eadfc930 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,8 +64,7 @@ jobs: ubuntu_23, centos_8, centos_9, - fedora_37, - fedora_38, + fedora_38, fedora_39, ] env: diff --git a/test/_fedora_37.Dockerfile b/test/_fedora_37.Dockerfile deleted file mode 100644 index b4f939ba..00000000 --- a/test/_fedora_37.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM fedora:37 -RUN dnf install -y git initscripts - -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole - -RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole -ADD . $GITDIR -RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR - -RUN true && \ - chmod +x $SCRIPTDIR/* - -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net - -#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/tox.fedora_37.ini b/test/tox.fedora_37.ini deleted file mode 100644 index 9c8752cc..00000000 --- a/test/tox.fedora_37.ini +++ /dev/null @@ -1,8 +0,0 @@ -[tox] -envlist = py3 - -[testenv] -allowlist_externals = docker -deps = -rrequirements.txt -commands = docker buildx build --load --progress plain -f _fedora_37.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py From 07b448d784d3f5e11e7a18e32eeab58aa0a5d25a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Jan 2024 17:15:36 +0100 Subject: [PATCH 190/413] Also check for IPv6 address for configured DNS servers Signed-off-by: DL6ER --- automated install/basic-install.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f5bf15e6..4bb4106b 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -817,12 +817,12 @@ If you want to specify a port other than 53, separate it with a hash.\ printf -v PIHOLE_DNS_1 "%s" "${piholeDNS%%,*}" printf -v PIHOLE_DNS_2 "%s" "${piholeDNS##*,}" - # If the first DNS value is invalid or empty, this if statement will be true and we will set PIHOLE_DNS_1="Invalid" - if ! valid_ip "${PIHOLE_DNS_1}" || [[ ! "${PIHOLE_DNS_1}" ]]; then + # If the first DNS value is invalid (neither IPv4 nor IPv6) or empty, set PIHOLE_DNS_1="Invalid" + if ! valid_ip "${PIHOLE_DNS_1}" && ! valid_ip6 "${PIHOLE_DNS_1}" || [[ -z "${PIHOLE_DNS_1}" ]]; then PIHOLE_DNS_1=${strInvalid} fi - # If the second DNS value is invalid or empty, this if statement will be true and we will set PIHOLE_DNS_2="Invalid" - if ! valid_ip "${PIHOLE_DNS_2}" && [[ "${PIHOLE_DNS_2}" ]]; then + # If the second DNS value is invalid but not empty, set PIHOLE_DNS_2="Invalid" + if ! valid_ip "${PIHOLE_DNS_2}" && ! valid_ip6 "${PIHOLE_DNS_2}" && [[ -n "${PIHOLE_DNS_2}" ]]; then PIHOLE_DNS_2=${strInvalid} fi # If either of the DNS servers are invalid, From 31a8f150b2e3a5411871f1163c7cc7a433e7ecc5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 30 Jan 2024 19:09:14 +0100 Subject: [PATCH 191/413] Add checking for availability of ftl.pi-hole.net when using FTL from a custom branch. If the server is down (or the user is offline, cannot resolve the domain, etc.), we fail early and hard instead of possibly corrupting the installation Signed-off-by: DL6ER --- advanced/Scripts/piholeCheckout.sh | 22 +++++++++++----- advanced/Scripts/update.sh | 7 ++++- automated install/basic-install.sh | 41 +++++++++++++++++++++--------- 3 files changed, 51 insertions(+), 19 deletions(-) diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index 39d39b1c..41fd8606 100755 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -164,7 +164,9 @@ checkout() { path="${2}/${binary}" oldbranch="$(pihole-FTL -b)" - if check_download_exists "$path"; then + check_download_exists "$path" + local ret=$? + if [ $ret -eq 0 ]; then echo " ${TICK} Branch ${2} exists" echo "${2}" > /etc/pihole/ftlbranch chmod 644 /etc/pihole/ftlbranch @@ -175,11 +177,19 @@ checkout() { # Update local and remote versions via updatechecker /opt/pihole/updatecheck.sh else - echo " ${CROSS} Requested branch \"${2}\" is not available" - ftlbranches=( $(git ls-remote https://github.com/pi-hole/ftl | grep 'heads' | sed 's/refs\/heads\///;s/ //g' | awk '{print $2}') ) - echo -e " ${INFO} Available branches for FTL are:" - for e in "${ftlbranches[@]}"; do echo " - $e"; done - exit 1 + if [[ $ret -eq 1 ]]; then + echo " ${CROSS} Requested branch \"${2}\" is not available" + ftlbranches=( $(git ls-remote https://github.com/pi-hole/ftl | grep 'heads' | sed 's/refs\/heads\///;s/ //g' | awk '{print $2}') ) + echo -e " ${INFO} Available branches for FTL are:" + for e in "${ftlbranches[@]}"; do echo " - $e"; done + exit 1 + elif [[ $ret -eq 2 ]]; then + printf " %b Unable to download from ftl.pi-hole.net. Please check your Internet connection and try again later.\\n" "${CROSS}" + exit 1 + else + printf " %b Unknown error. Please contact Pi-hole Support\\n" "${CROSS}" + exit 1 + fi fi else diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index 9dae66df..8a35ef2e 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -144,7 +144,7 @@ main() { local binary binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL) - if FTLcheckUpdate "${binary}" > /dev/null; then + if FTLcheckUpdate "${binary}"; then FTL_update=true echo -e " ${INFO} FTL:\\t\\t${COL_YELLOW}update available${COL_NC}" else @@ -155,8 +155,13 @@ main() { 2) echo -e " ${INFO} FTL:\\t\\t${COL_LIGHT_RED}Branch is not available.${COL_NC}\\n\\t\\t\\tUse ${COL_LIGHT_GREEN}pihole checkout ftl [branchname]${COL_NC} to switch to a valid branch." ;; + 3) + echo -e " ${INFO} FTL:\\t\\t${COL_LIGHT_RED}Something has gone wrong, cannot reach download server${COL_NC}" + exit 1 + ;; *) echo -e " ${INFO} FTL:\\t\\t${COL_LIGHT_RED}Something has gone wrong, contact support${COL_NC}" + exit 1 esac FTL_update=false fi diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f5bf15e6..af7ca151 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1687,12 +1687,19 @@ update_dialogs() { } check_download_exists() { + # Check if the download exists and we can reach the server status=$(curl --head --silent "https://ftl.pi-hole.net/${1}" | head -n 1) - if grep -q "404" <<< "$status"; then - return 1 - else + + # Check the status code + if grep -q "200" <<< "$status"; then return 0 + elif grep -q "404" <<< "$status"; then + return 1 fi + + # Other error or no status code at all, e.g., no Internet, server not + # available/reachable, ... + return 2 } fully_fetch_repo() { @@ -1957,10 +1964,8 @@ get_binary_name() { } FTLcheckUpdate() { - #In the next section we check to see if FTL is already installed (in case of pihole -r). - #If the installed version matches the latest version, then check the installed sha1sum of the binary vs the remote sha1sum. If they do not match, then download - printf " %b Checking for existing FTL binary...\\n" "${INFO}" - + # In the next section we check to see if FTL is already installed (in case of pihole -r). + # If the installed version matches the latest version, then check the installed sha1sum of the binary vs the remote sha1sum. If they do not match, then download local ftlLoc ftlLoc=$(command -v pihole-FTL 2>/dev/null) @@ -1979,14 +1984,24 @@ FTLcheckUpdate() { local localSha1 if [[ ! "${ftlBranch}" == "master" ]]; then - #Check whether or not the binary for this FTL branch actually exists. If not, then there is no update! + # Check whether or not the binary for this FTL branch actually exists. If not, then there is no update! local path path="${ftlBranch}/${binary}" # shellcheck disable=SC1090 - if ! check_download_exists "$path"; then - printf " %b Branch \"%s\" is not available.\\n" "${INFO}" "${ftlBranch}" - printf " %b Use %bpihole checkout ftl [branchname]%b to switch to a valid branch.\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${COL_NC}" - return 2 + check_download_exists "$path" + local ret=$? + if [ $ret -ne 0 ]; then + if [[ $ret -eq 1 ]]; then + printf " %b Branch \"%s\" is not available.\\n" "${INFO}" "${ftlBranch}" + printf " %b Use %bpihole checkout ftl [branchname]%b to switch to a valid branch.\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${COL_NC}" + return 2 + elif [[ $ret -eq 2 ]]; then + printf " %b Unable to download from ftl.pi-hole.net. Please check your Internet connection and try again later.\\n" "${CROSS}" + return 3 + else + printf " %b Unknown error. Please contact Pi-hole Support\\n" "${CROSS}" + return 4 + fi fi if [[ ${ftlLoc} ]]; then @@ -2011,12 +2026,14 @@ FTLcheckUpdate() { FTLversion=$(/usr/bin/pihole-FTL tag) local FTLlatesttag + # Get the latest version from the GitHub API if ! FTLlatesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep --color=never -i Location: | awk -F / '{print $NF}' | tr -d '[:cntrl:]'); then # There was an issue while retrieving the latest version printf " %b Failed to retrieve latest FTL release metadata" "${CROSS}" return 3 fi + # Check if the installed version matches the latest version if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then return 0 else From 6e946f76d6e0166d6f2d0e86ed7ab01686faf6bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 3 Feb 2024 10:21:48 +0000 Subject: [PATCH 192/413] Bump pytest from 7.4.4 to 8.0.0 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.4 to 8.0.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.4...8.0.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index dea4ace9..2b8dfb1b 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.4 +pytest == 8.0.0 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.12.1 From b5ab8ac1980442f19d474f630f47da6dd851f151 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 11 Feb 2024 16:54:22 +0100 Subject: [PATCH 193/413] Change UNIQUEness constraint from (address) to (address, type) in the adlist table. This will allow certain adlists to be associated to different groups. A possible scenario is an adlist meant to block a specific service (e.g. Twitter, Youtube, etc.). It can then either be used to ensure these services are really blocked on the devices of group A but will never be blocked on devices of group B. Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 097b0a78..42060443 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -27,7 +27,7 @@ CREATE TABLE domainlist CREATE TABLE adlist ( id INTEGER PRIMARY KEY AUTOINCREMENT, - address TEXT UNIQUE NOT NULL, + address TEXT NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), @@ -37,7 +37,8 @@ CREATE TABLE adlist invalid_domains INTEGER NOT NULL DEFAULT 0, status INTEGER NOT NULL DEFAULT 0, abp_entries INTEGER NOT NULL DEFAULT 0, - type INTEGER NOT NULL DEFAULT 0 + type INTEGER NOT NULL DEFAULT 0, + UNIQUE(address, type) ); CREATE TABLE adlist_by_group From bcb712b6e3f408b0080768b9c2c77c2a4c7c6dbd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 11 Feb 2024 21:07:15 +0100 Subject: [PATCH 194/413] Fix accidential double -ni from one of the last PRs Signed-off-by: DL6ER --- gravity.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gravity.sh b/gravity.sh index 592aa801..4d785d8a 100755 --- a/gravity.sh +++ b/gravity.sh @@ -114,7 +114,7 @@ gravity_swap_databases() { # Update timestamp when the gravity table was last updated successfully update_gravity_timestamp() { - output=$({ printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 -ni -ni "${gravityTEMPfile}"; } 2>&1) + output=$({ printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -352,8 +352,8 @@ gravity_DownloadBlocklists() { # Retrieve source URLs from gravity database # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true) - mapfile -t sources <<<"$(pihole-FTL sqlite3 -ni -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2>/dev/null)" - mapfile -t sourceIDs <<<"$(pihole-FTL sqlite3 -ni -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2>/dev/null)" + mapfile -t sources <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2>/dev/null)" + mapfile -t sourceIDs <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2>/dev/null)" mapfile -t sourceTypes <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2>/dev/null)" # Parse source domains from $sources @@ -667,12 +667,12 @@ gravity_Table_Count() { local table="${1}" local str="${2}" local num - num="$(pihole-FTL sqlite3 -ni -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM ${table};")" + num="$(pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM ${table};")" if [[ "${table}" == "gravity" ]]; then local unique - unique="$(pihole-FTL sqlite3 -ni -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" + unique="$(pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" echo -e " ${INFO} Number of ${str}: ${num} (${COL_BOLD}${unique} unique domains${COL_NC})" - pihole-FTL sqlite3 -ni -ni "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" + pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" else echo -e " ${INFO} Number of ${str}: ${num}" fi From 75fadb9b55fbb1454bbddd60d0ed99924200d2d4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 13 Feb 2024 08:55:26 +0100 Subject: [PATCH 195/413] Adlists need to be grouped by both address and type to differentiate between anti-/gravity lists Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 581d2069..df7db893 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -48,7 +48,7 @@ GenerateOutput() { lists_data=$(printf %s "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]') # construct a new json for the gravity results where each object contains the adlist URL and the related domains - gravity_data=$(printf %s "${data}" | jq '.search.gravity | group_by(.address) | map({ address: (.[0].address), domains: [.[] | .domain] })') + gravity_data=$(printf %s "${data}" | jq '.search.gravity | group_by(.address,.type) | map({ address: (.[0].address), domains: [.[] | .domain] })') # number of objects in each json num_gravity=$(printf %s "${gravity_data}" | jq length) From 9ff43040ec52d9bc1adefa2f4a69775835dfc67c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 13 Feb 2024 09:58:23 +0100 Subject: [PATCH 196/413] Add list type in pihole -q Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index df7db893..493c75ea 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -41,14 +41,14 @@ Options: GenerateOutput() { local data gravity_data lists_data num_gravity num_lists search_type_str - local gravity_data_csv lists_data_csv line current_domain + local gravity_data_csv lists_data_csv line current_domain url type color data="${1}" # construct a new json for the list results where each object contains the domain and the related type lists_data=$(printf %s "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]') # construct a new json for the gravity results where each object contains the adlist URL and the related domains - gravity_data=$(printf %s "${data}" | jq '.search.gravity | group_by(.address,.type) | map({ address: (.[0].address), domains: [.[] | .domain] })') + gravity_data=$(printf %s "${data}" | jq '.search.gravity | group_by(.address,.type) | map({ address: (.[0].address), type: (.[0].type), domains: [.[] | .domain] })') # number of objects in each json num_gravity=$(printf %s "${gravity_data}" | jq length) @@ -78,15 +78,27 @@ GenerateOutput() { if [ "${num_gravity}" -gt 0 ]; then # Convert the data to a csv, each line is a "URL,domain,domain,...." string # not using jq's @csv here as it quotes each value individually - gravity_data_csv=$(printf %s "${gravity_data}" | jq --raw-output '.[] | [.address, .domains[]] | join(",")') + gravity_data_csv=$(printf %s "${gravity_data}" | jq --raw-output '.[] | [.address, .type, .domains[]] | join(",")') # Generate line-by-line output for each csv line echo "${gravity_data_csv}" | while read -r line; do + # Get first part of the line, the URL + url=${line%%,*} + + # cut off URL, leaving "type,domain,domain,...." + line=${line#*,} + type=${line%%,*} + # type == "block" -> red, type == "allow" -> green + if [ "${type}" = "block" ]; then + color="${COL_RED}" + else + color="${COL_GREEN}" + fi # print adlist URL - printf "%s\n\n" " - ${COL_BLUE}${line%%,*}${COL_NC}" + printf "%s (%s)\n\n" " - ${COL_BLUE}${url}${COL_NC}" "${color}${type}${COL_NC}" - # cut off URL, leaving "domain,domain,...." + # cut off type, leaving "domain,domain,...." line=${line#*,} # print each domain and remove it from the string until nothing is left while [ ${#line} -gt 0 ]; do From b322f1e98bf1ec64b705add4f089cda22ae8e5e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 17 Feb 2024 10:50:41 +0000 Subject: [PATCH 197/413] Bump tox from 4.12.1 to 4.13.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.12.1 to 4.13.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.12.1...4.13.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 2b8dfb1b..a6826838 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.0.0 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.12.1 +tox == 4.13.0 From 3ed29f494b93be8b03bc9739ad5f03a7cf578a25 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 24 Feb 2024 12:45:22 +0000 Subject: [PATCH 198/413] Bump pytest from 8.0.0 to 8.0.1 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.0.0 to 8.0.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.0.0...8.0.1) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index a6826838..3e341719 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.0.0 +pytest == 8.0.1 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.13.0 From 07e6c0d250febddf706e5b414466b88a6c89b288 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 24 Feb 2024 12:49:07 +0000 Subject: [PATCH 199/413] Bump pytest-testinfra from 10.0.0 to 10.1.0 in /test Bumps [pytest-testinfra](https://github.com/pytest-dev/pytest-testinfra) from 10.0.0 to 10.1.0. - [Release notes](https://github.com/pytest-dev/pytest-testinfra/releases) - [Changelog](https://github.com/pytest-dev/pytest-testinfra/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-testinfra/compare/10.0.0...10.1.0) --- updated-dependencies: - dependency-name: pytest-testinfra dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 3e341719..2a559b05 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 8.0.1 pytest-xdist == 3.5.0 -pytest-testinfra == 10.0.0 +pytest-testinfra == 10.1.0 tox == 4.13.0 From fd1372df3e72509e16c92bfcb9b4489d9b232fc6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Mar 2024 10:01:31 +0000 Subject: [PATCH 200/413] Bump pytest from 8.0.1 to 8.0.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.0.1 to 8.0.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.0.1...8.0.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 2a559b05..ffb05813 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.0.1 +pytest == 8.0.2 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 tox == 4.13.0 From bfd8b572cbc20a294a2886f1cfb4e2315d10ca98 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Mar 2024 10:07:37 +0000 Subject: [PATCH 201/413] Bump tox from 4.13.0 to 4.14.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.13.0 to 4.14.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.13.0...4.14.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index ffb05813..d4415e2b 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.0.2 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 -tox == 4.13.0 +tox == 4.14.1 From c02401b81e43febae7365f960c2d530afc714e24 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 9 Mar 2024 11:17:33 +0100 Subject: [PATCH 202/413] Highlight "### CHANGED" strings in the debug log of pihole.toml Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index f5a57278..7c558127 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -865,8 +865,6 @@ make_array_from_file() { local limit=${2} # A local iterator for testing if we are at the limit above local i=0 - # Set the array to be empty so we can start fresh when the function is used - local file_content=() # If the file is a directory if [[ -d "${filename}" ]]; then # do nothing since it cannot be parsed @@ -878,11 +876,14 @@ make_array_from_file() { new_line=$(echo "${line}" | sed -e 's/^\s*#.*$//' -e '/^$/d') # If the line still has content (a non-zero value) if [[ -n "${new_line}" ]]; then - # Put it into the array - file_content+=("${new_line}") - else - # Otherwise, it's a blank line or comment, so do nothing - : + + # If the string contains "### CHANGED", highlight this part in red + if [[ "${new_line}" == *"### CHANGED"* ]]; then + new_line="${new_line//### CHANGED/${COL_RED}### CHANGED${COL_NC}}" + fi + + # Finally, write this line to the log + log_write " ${new_line}" fi # Increment the iterator +1 i=$((i+1)) @@ -894,12 +895,6 @@ make_array_from_file() { break fi done < "${filename}" - # Now the we have made an array of the file's content - for each_line in "${file_content[@]}"; do - # Print each line - # At some point, we may want to check the file line-by-line, so that's the reason for an array - log_write " ${each_line}" - done fi } From 2fd0de4743b134ad1a0be6bea119301d16b98ef5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 10 Mar 2024 08:43:37 +0100 Subject: [PATCH 203/413] Verify that we actually downloaded a valid checksum before comparing it to the local one. This covers situations where downloading the checksum from remote might have failed Signed-off-by: DL6ER --- automated install/basic-install.sh | 38 +++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 24abb7e7..eca7d8b0 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2002,9 +2002,11 @@ FTLcheckUpdate() { local localSha1 if [[ ! "${ftlBranch}" == "master" ]]; then - # Check whether or not the binary for this FTL branch actually exists. If not, then there is no update! + # This is not the master branch local path path="${ftlBranch}/${binary}" + + # Check whether or not the binary for this FTL branch actually exists. If not, then there is no update! # shellcheck disable=SC1090 check_download_exists "$path" local ret=$? @@ -2023,12 +2025,20 @@ FTLcheckUpdate() { fi if [[ ${ftlLoc} ]]; then - # We already have a pihole-FTL binary downloaded. - # Alt branches don't have a tagged version against them, so just confirm the checksum of the local vs remote to decide whether we download or not + # We already have a pihole-FTL binary installed, check if it's the + # same as the remote one + # Alt branches don't have a tagged version against them, so just + # confirm the checksum of the local vs remote to decide whether we + # download or not remoteSha1=$(curl -sSL --fail "https://ftl.pi-hole.net/${ftlBranch}/${binary}.sha1" | cut -d ' ' -f 1) - localSha1=$(sha1sum "$(command -v pihole-FTL)" | cut -d ' ' -f 1) + localSha1=$(sha1sum "${ftlLoc}" | cut -d ' ' -f 1) - if [[ "${remoteSha1}" != "${localSha1}" ]]; then + # Check we downloaded a valid checksum (no 404 or other error like + # no DNS resolution) + if [[ ! "${remoteSha1}" =~ ^[a-f0-9]{40}$ ]]; then + printf " %b Remote checksum not available, trying to download binary from ftl.pi-hole.net.\\n" "${CROSS}" + return 0 + elif [[ "${remoteSha1}" != "${localSha1}" ]]; then printf " %b Checksums do not match, downloading from ftl.pi-hole.net.\\n" "${INFO}" return 0 else @@ -2039,7 +2049,10 @@ FTLcheckUpdate() { return 0 fi else + # This is the master branch if [[ ${ftlLoc} ]]; then + # We already have a pihole-FTL binary installed, check if it's the + # same as the remote one local FTLversion FTLversion=$(/usr/bin/pihole-FTL tag) local FTLlatesttag @@ -2053,15 +2066,24 @@ FTLcheckUpdate() { # Check if the installed version matches the latest version if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then + # If the installed version does not match the latest version, then download return 0 else + # If the installed version matches the latest version, then + # check the installed sha1sum of the binary vs the remote + # sha1sum. If they do not match, then download printf " %b Latest FTL Binary already installed (%s). Confirming Checksum...\\n" "${INFO}" "${FTLlatesttag}" remoteSha1=$(curl -sSL --fail "https://github.com/pi-hole/FTL/releases/download/${FTLversion%$'\r'}/${binary}.sha1" | cut -d ' ' -f 1) - localSha1=$(sha1sum "$(command -v pihole-FTL)" | cut -d ' ' -f 1) + localSha1=$(sha1sum "${ftlLoc}" | cut -d ' ' -f 1) - if [[ "${remoteSha1}" != "${localSha1}" ]]; then - printf " %b Corruption detected...\\n" "${INFO}" + # Check we downloaded a valid checksum (no 404 or other error like + # no DNS resolution) + if [[ ! "${remoteSha1}" =~ ^[a-f0-9]{40}$ ]]; then + printf " %b Remote checksum not available, trying to redownload binary...\\n" "${CROSS}" + return 0 + elif [[ "${remoteSha1}" != "${localSha1}" ]]; then + printf " %b Corruption detected, redownloading binary...\\n" "${CROSS}" return 0 else printf " %b Checksum correct. No need to download!\\n" "${INFO}" From 82a83c497dd772cf6f044ec50b4471af102bff37 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 10 Mar 2024 21:18:13 +0100 Subject: [PATCH 204/413] Reduce code-duplication Signed-off-by: DL6ER --- automated install/basic-install.sh | 59 +++++++++++++----------------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index eca7d8b0..d057cb82 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2030,21 +2030,9 @@ FTLcheckUpdate() { # Alt branches don't have a tagged version against them, so just # confirm the checksum of the local vs remote to decide whether we # download or not - remoteSha1=$(curl -sSL --fail "https://ftl.pi-hole.net/${ftlBranch}/${binary}.sha1" | cut -d ' ' -f 1) - localSha1=$(sha1sum "${ftlLoc}" | cut -d ' ' -f 1) - - # Check we downloaded a valid checksum (no 404 or other error like - # no DNS resolution) - if [[ ! "${remoteSha1}" =~ ^[a-f0-9]{40}$ ]]; then - printf " %b Remote checksum not available, trying to download binary from ftl.pi-hole.net.\\n" "${CROSS}" - return 0 - elif [[ "${remoteSha1}" != "${localSha1}" ]]; then - printf " %b Checksums do not match, downloading from ftl.pi-hole.net.\\n" "${INFO}" - return 0 - else - printf " %b Checksum of installed binary matches remote. No need to download!\\n" "${INFO}" - return 1 - fi + printf " %b FTL binary already installed. Confirming Checksum...\\n" "${INFO}" + checkSumFile="https://ftl.pi-hole.net/${ftlBranch}/${binary}.sha1" + # Continue further down... else return 0 fi @@ -2066,34 +2054,39 @@ FTLcheckUpdate() { # Check if the installed version matches the latest version if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then - # If the installed version does not match the latest version, then download + # If the installed version does not match the latest version, + # then download return 0 else # If the installed version matches the latest version, then # check the installed sha1sum of the binary vs the remote # sha1sum. If they do not match, then download - printf " %b Latest FTL Binary already installed (%s). Confirming Checksum...\\n" "${INFO}" "${FTLlatesttag}" - - remoteSha1=$(curl -sSL --fail "https://github.com/pi-hole/FTL/releases/download/${FTLversion%$'\r'}/${binary}.sha1" | cut -d ' ' -f 1) - localSha1=$(sha1sum "${ftlLoc}" | cut -d ' ' -f 1) - - # Check we downloaded a valid checksum (no 404 or other error like - # no DNS resolution) - if [[ ! "${remoteSha1}" =~ ^[a-f0-9]{40}$ ]]; then - printf " %b Remote checksum not available, trying to redownload binary...\\n" "${CROSS}" - return 0 - elif [[ "${remoteSha1}" != "${localSha1}" ]]; then - printf " %b Corruption detected, redownloading binary...\\n" "${CROSS}" - return 0 - else - printf " %b Checksum correct. No need to download!\\n" "${INFO}" - return 1 - fi + printf " %b Latest FTL binary already installed (%s). Confirming Checksum...\\n" "${INFO}" "${FTLlatesttag}" + checkSumFile="https://github.com/pi-hole/FTL/releases/download/${FTLversion%$'\r'}/${binary}.sha1" + # Continue further down... fi else return 0 fi fi + + # If we reach this point, we need to check the checksum of the local vs + # remote to decide whether we download or not + remoteSha1=$(curl -sSL --fail "${checkSumFile}" | cut -d ' ' -f 1) + localSha1=$(sha1sum "${ftlLoc}" | cut -d ' ' -f 1) + + # Check we downloaded a valid checksum (no 404 or other error like + # no DNS resolution) + if [[ ! "${remoteSha1}" =~ ^[a-f0-9]{40}$ ]]; then + printf " %b Remote checksum not available, trying to redownload binary...\\n" "${CROSS}" + return 0 + elif [[ "${remoteSha1}" != "${localSha1}" ]]; then + printf " %b Corruption detected, redownloading binary...\\n" "${CROSS}" + return 0 + fi + + printf " %b Checksum correct. No need to download!\\n" "${INFO}" + return 1 } # Detect suitable FTL binary platform From 58a1e8582611f70249ef5592286b5423c5106436 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 16 Mar 2024 10:21:49 +0000 Subject: [PATCH 205/413] Bump actions/checkout from 4.1.1 to 4.1.2 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.1 to 4.1.2. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.1...v4.1.2) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index fc821194..9cfd8a61 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 095d7358..c6a581ff 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index e52d4ae9..9b35a974 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2a2b50dc..bf52d46f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 - name: Check scripts in repository are executable run: | @@ -71,7 +71,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 - name: Set up Python 3.10 uses: actions/setup-python@v5.0.0 From 472602ffb9d84bcc4b26a088d880eda04c2d60de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 23 Mar 2024 10:52:31 +0000 Subject: [PATCH 206/413] Bump tox from 4.14.1 to 4.14.2 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.14.1 to 4.14.2. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.14.1...4.14.2) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index d4415e2b..f8d5d94c 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.0.2 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 -tox == 4.14.1 +tox == 4.14.2 From 9f9e5dffc259d7509bf5f1fdfb59be9c8da0cc57 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Mar 2024 20:27:30 +0000 Subject: [PATCH 207/413] Bump pytest from 8.0.2 to 8.1.1 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.0.2 to 8.1.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.0.2...8.1.1) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index f8d5d94c..9426dd2c 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.0.2 +pytest == 8.1.1 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 tox == 4.14.2 From 398f90f7f436c6773f003692857e62aabf8a6d6d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 28 Mar 2024 09:12:51 +0100 Subject: [PATCH 208/413] Non-matching checksums are not always corruption. Actually, they will instead be caused by binaries updated on the remote branch. This is most seen with frequently channging branchs such as development-v6 at this time. Signed-off-by: DL6ER --- automated install/basic-install.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index d057cb82..41cc2161 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2030,7 +2030,7 @@ FTLcheckUpdate() { # Alt branches don't have a tagged version against them, so just # confirm the checksum of the local vs remote to decide whether we # download or not - printf " %b FTL binary already installed. Confirming Checksum...\\n" "${INFO}" + printf " %b FTL binary already installed, verifying integrity...\\n" "${INFO}" checkSumFile="https://ftl.pi-hole.net/${ftlBranch}/${binary}.sha1" # Continue further down... else @@ -2061,7 +2061,7 @@ FTLcheckUpdate() { # If the installed version matches the latest version, then # check the installed sha1sum of the binary vs the remote # sha1sum. If they do not match, then download - printf " %b Latest FTL binary already installed (%s). Confirming Checksum...\\n" "${INFO}" "${FTLlatesttag}" + printf " %b Latest FTL binary already installed (%s), verifying integrity...\\n" "${INFO}" "${FTLlatesttag}" checkSumFile="https://github.com/pi-hole/FTL/releases/download/${FTLversion%$'\r'}/${binary}.sha1" # Continue further down... fi @@ -2078,14 +2078,14 @@ FTLcheckUpdate() { # Check we downloaded a valid checksum (no 404 or other error like # no DNS resolution) if [[ ! "${remoteSha1}" =~ ^[a-f0-9]{40}$ ]]; then - printf " %b Remote checksum not available, trying to redownload binary...\\n" "${CROSS}" + printf " %b Remote checksum not available, trying to redownload...\\n" "${CROSS}" return 0 elif [[ "${remoteSha1}" != "${localSha1}" ]]; then - printf " %b Corruption detected, redownloading binary...\\n" "${CROSS}" + printf " %b Remote binary is different, downloading...\\n" "${CROSS}" return 0 fi - printf " %b Checksum correct. No need to download!\\n" "${INFO}" + printf " %b Local binary up-to-date. No need to download!\\n" "${INFO}" return 1 } From b595b3b5f495223ec04d8fa159c5fcb3ded8b44d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 28 Mar 2024 15:41:45 +0100 Subject: [PATCH 209/413] Port most recent changes of PRs #5615, #5618 amd #5620 to v6 Signed-off-by: DL6ER --- gravity.sh | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index d49af29d..bc0ff23d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -488,6 +488,7 @@ compareLists() { gravity_DownloadBlocklistFromUrl() { local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" gravity_type="${6}" domain="${7}" local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext + local file_path permissions ip_addr port blocked=false download=true # Create temp file to store content on disk instead of RAM # We don't use '--suffix' here because not all implementations of mktemp support it, e.g. on Alpine @@ -591,8 +592,40 @@ gravity_DownloadBlocklistFromUrl() { fi fi - # shellcheck disable=SC2086 - httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2>/dev/null) + # If we are going to "download" a local file, we first check if the target + # file has a+r permission. We explicitly check for all+read because we want + # to make sure that the file is readable by everyone and not just the user + # running the script. + if [[ $url == "file://"* ]]; then + # Get the file path + file_path=$(echo "$url" | cut -d'/' -f3-) + # Check if the file exists and is a regular file (or a symlink to one) + if [[ ! -e $file_path ]]; then + # Output that the file does not exist + echo -e "${OVER} ${CROSS} ${file_path} does not exist" + download=false + elif [[ ! -f $file_path ]]; then + # Output that the file is not a regular file + echo -e "${OVER} ${CROSS} ${file_path} is not a regular file" + download=false + else + # Check if the file has a+r permissions + permissions=$(stat -Lc "%a" "$file_path") + if [[ $permissions == *4 || $permissions == *5 || $permissions == *6 || $permissions == *7 ]]; then + # Output that we are using the local file + echo -e "${OVER} ${INFO} Using local file ${file_path}" + else + # Output that the file does not have the correct permissions + echo -e "${OVER} ${CROSS} Cannot read file (file needs to have o+r permission)" + download=false + fi + fi + fi + + if [[ "${download}" == true ]]; then + # shellcheck disable=SC2086 + httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2>/dev/null) + fi case $url in # Did we "download" a local file? @@ -601,7 +634,7 @@ gravity_DownloadBlocklistFromUrl() { echo -e "${OVER} ${TICK} ${str} Retrieval successful" success=true else - echo -e "${OVER} ${CROSS} ${str} Not found / empty list" + echo -e "${OVER} ${CROSS} ${str} Retrieval failed / empty list" fi ;; # Did we "download" a remote file? From 67ddb64bcca77f942158e26e14816cd1bb88600b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 30 Mar 2024 10:10:21 +0000 Subject: [PATCH 210/413] Bump eps1lon/actions-label-merge-conflict from 2.1.0 to 3.0.0 Bumps [eps1lon/actions-label-merge-conflict](https://github.com/eps1lon/actions-label-merge-conflict) from 2.1.0 to 3.0.0. - [Release notes](https://github.com/eps1lon/actions-label-merge-conflict/releases) - [Changelog](https://github.com/eps1lon/actions-label-merge-conflict/blob/main/CHANGELOG.md) - [Commits](https://github.com/eps1lon/actions-label-merge-conflict/compare/v2.1.0...v3.0.0) --- updated-dependencies: - dependency-name: eps1lon/actions-label-merge-conflict dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/merge-conflict.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge-conflict.yml b/.github/workflows/merge-conflict.yml index d86e9cd1..d9ceacca 100644 --- a/.github/workflows/merge-conflict.yml +++ b/.github/workflows/merge-conflict.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check if PRs are have merge conflicts - uses: eps1lon/actions-label-merge-conflict@v2.1.0 + uses: eps1lon/actions-label-merge-conflict@v3.0.0 with: dirtyLabel: "PR: Merge Conflict" repoToken: "${{ secrets.GITHUB_TOKEN }}" From 91e0d668e0a552ca08e14de67937bd0a0986e9d7 Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 30 Mar 2024 20:03:15 +0100 Subject: [PATCH 211/413] Code style: Use explicit form of CLI arguments Co-authored-by: Dan Schaper Signed-off-by: Dominik --- gravity.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index bc0ff23d..e37c8104 100755 --- a/gravity.sh +++ b/gravity.sh @@ -598,7 +598,7 @@ gravity_DownloadBlocklistFromUrl() { # running the script. if [[ $url == "file://"* ]]; then # Get the file path - file_path=$(echo "$url" | cut -d'/' -f3-) + file_path=$(echo "$url" | cut --delimiter='/' --fields=3-) # Check if the file exists and is a regular file (or a symlink to one) if [[ ! -e $file_path ]]; then # Output that the file does not exist @@ -610,7 +610,7 @@ gravity_DownloadBlocklistFromUrl() { download=false else # Check if the file has a+r permissions - permissions=$(stat -Lc "%a" "$file_path") + permissions=$(stat --dereference --format="%a" "$file_path") if [[ $permissions == *4 || $permissions == *5 || $permissions == *6 || $permissions == *7 ]]; then # Output that we are using the local file echo -e "${OVER} ${INFO} Using local file ${file_path}" @@ -624,7 +624,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ "${download}" == true ]]; then # shellcheck disable=SC2086 - httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2>/dev/null) + httpCode=$(curl --connect-timeout ${curl_connect_timeout} --silent --location ${compression} ${cmd_ext} ${heisenbergCompensator} --write-out "%{http_code}" "${url}" --output "${listCurlBuffer}" 2>/dev/null) fi case $url in From f01362caf15141b95e85a287dff58b414143a7cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 13:17:20 +0000 Subject: [PATCH 212/413] Bump actions/setup-python from 5.0.0 to 5.1.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.0.0 to 5.1.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.0.0...v5.1.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bf52d46f..52acc09a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,7 +74,7 @@ jobs: uses: actions/checkout@v4.1.2 - name: Set up Python 3.10 - uses: actions/setup-python@v5.0.0 + uses: actions/setup-python@v5.1.0 with: python-version: "3.10" From 5d77c2b362e5748ef87accb6c63c07bff6edd81d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Apr 2024 10:06:19 +0000 Subject: [PATCH 213/413] Bump actions/checkout from 4.1.2 to 4.1.3 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.2 to 4.1.3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.2...v4.1.3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 9cfd8a61..ee0b7dae 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index c6a581ff..bbd275b8 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.3 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 9b35a974..c04b24be 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.3 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 52acc09a..b0454998 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.3 - name: Check scripts in repository are executable run: | @@ -71,7 +71,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.3 - name: Set up Python 3.10 uses: actions/setup-python@v5.1.0 From 91d95e2c8ad2d64c5289b85eccd79ea128391b3f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Apr 2024 10:21:11 +0000 Subject: [PATCH 214/413] Bump actions/checkout from 4.1.3 to 4.1.4 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.3 to 4.1.4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.3...v4.1.4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index ee0b7dae..51920384 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.3 + uses: actions/checkout@v4.1.4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index bbd275b8..51289101 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.3 + uses: actions/checkout@v4.1.4 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index c04b24be..8e024b10 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.3 + uses: actions/checkout@v4.1.4 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b0454998..5eda11ce 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.3 + uses: actions/checkout@v4.1.4 - name: Check scripts in repository are executable run: | @@ -71,7 +71,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.3 + uses: actions/checkout@v4.1.4 - name: Set up Python 3.10 uses: actions/setup-python@v5.1.0 From 09ce0003405945b6e63744a14f9023d5acc1e7d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Apr 2024 10:28:36 +0000 Subject: [PATCH 215/413] Bump tox from 4.14.2 to 4.15.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.14.2 to 4.15.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.14.2...4.15.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 9426dd2c..0269d946 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.1.1 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 -tox == 4.14.2 +tox == 4.15.0 From 78c755b53849ce1a7b21a6935b9bfce704e83bab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Apr 2024 11:03:12 +0000 Subject: [PATCH 216/413] Bump pytest from 8.1.1 to 8.1.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.1.1 to 8.1.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.1.1...8.1.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 0269d946..1b27d448 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.1.1 +pytest == 8.1.2 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 tox == 4.15.0 From bf0c3c0e0fa184a4ae171c03be2be451f0413d35 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 4 May 2024 10:04:20 +0000 Subject: [PATCH 217/413] Bump pytest from 8.1.2 to 8.2.0 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.1.2 to 8.2.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.1.2...8.2.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 1b27d448..721029ed 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.1.2 +pytest == 8.2.0 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 tox == 4.15.0 From 24ade13c0245e769c7ddb7e909a788044c9bf409 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 4 May 2024 10:52:00 +0000 Subject: [PATCH 218/413] Bump pytest-xdist from 3.5.0 to 3.6.1 in /test Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.5.0 to 3.6.1. - [Release notes](https://github.com/pytest-dev/pytest-xdist/releases) - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.5.0...v3.6.1) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 721029ed..76af80a3 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 8.2.0 -pytest-xdist == 3.5.0 +pytest-xdist == 3.6.1 pytest-testinfra == 10.1.0 tox == 4.15.0 From cd30772c43a86c043ed3a1f5c862e1efd6c28aa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 11 May 2024 14:21:51 +0200 Subject: [PATCH 219/413] Add Ubuntu 24.04 and Fedora 40, remove Fedora 38 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 3 ++- ...ora_38.Dockerfile => _fedora_40.Dockerfile} | 2 +- test/_ubuntu_24.Dockerfile | 18 ++++++++++++++++++ test/{tox.fedora_38.ini => tox.fedora_40.ini} | 2 +- test/tox.ubuntu_24.ini | 8 ++++++++ 5 files changed, 30 insertions(+), 3 deletions(-) rename test/{_fedora_38.Dockerfile => _fedora_40.Dockerfile} (97%) create mode 100644 test/_ubuntu_24.Dockerfile rename test/{tox.fedora_38.ini => tox.fedora_40.ini} (84%) create mode 100644 test/tox.ubuntu_24.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5eda11ce..71bd04bd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -62,10 +62,11 @@ jobs: ubuntu_20, ubuntu_22, ubuntu_23, + ubuntu_24, centos_8, centos_9, - fedora_38, fedora_39, + fedora_40, ] env: DISTRO: ${{matrix.distro}} diff --git a/test/_fedora_38.Dockerfile b/test/_fedora_40.Dockerfile similarity index 97% rename from test/_fedora_38.Dockerfile rename to test/_fedora_40.Dockerfile index 76f69771..6d00072f 100644 --- a/test/_fedora_38.Dockerfile +++ b/test/_fedora_40.Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:38 +FROM fedora:40 RUN dnf install -y git initscripts ENV GITDIR /etc/.pihole diff --git a/test/_ubuntu_24.Dockerfile b/test/_ubuntu_24.Dockerfile new file mode 100644 index 00000000..2b048361 --- /dev/null +++ b/test/_ubuntu_24.Dockerfile @@ -0,0 +1,18 @@ +FROM buildpack-deps:24.04-scm + +ENV GITDIR /etc/.pihole +ENV SCRIPTDIR /opt/pihole + +RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole +ADD . $GITDIR +RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ +ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV DEBIAN_FRONTEND=noninteractive + +RUN true && \ + chmod +x $SCRIPTDIR/* + +ENV SKIP_INSTALL true +ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net + +#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/tox.fedora_38.ini b/test/tox.fedora_40.ini similarity index 84% rename from test/tox.fedora_38.ini rename to test/tox.fedora_40.ini index d596092c..149630d7 100644 --- a/test/tox.fedora_38.ini +++ b/test/tox.fedora_40.ini @@ -4,5 +4,5 @@ envlist = py3 [testenv] allowlist_externals = docker deps = -rrequirements.txt -commands = docker buildx build --load --progress plain -f _fedora_38.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _fedora_40.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py diff --git a/test/tox.ubuntu_24.ini b/test/tox.ubuntu_24.ini new file mode 100644 index 00000000..dbd278d6 --- /dev/null +++ b/test/tox.ubuntu_24.ini @@ -0,0 +1,8 @@ +[tox] +envlist = py3 + +[testenv:py3] +allowlist_externals = docker +deps = -rrequirements.txt +commands = docker buildx build --load --progress plain -f _ubuntu_24.Dockerfile -t pytest_pihole:test_container ../ + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py From f4c9278b86608f9d10496d9c6f647ed3e7145a4a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 11 May 2024 15:45:15 +0000 Subject: [PATCH 220/413] Bump eps1lon/actions-label-merge-conflict from 3.0.0 to 3.0.1 Bumps [eps1lon/actions-label-merge-conflict](https://github.com/eps1lon/actions-label-merge-conflict) from 3.0.0 to 3.0.1. - [Release notes](https://github.com/eps1lon/actions-label-merge-conflict/releases) - [Changelog](https://github.com/eps1lon/actions-label-merge-conflict/blob/main/CHANGELOG.md) - [Commits](https://github.com/eps1lon/actions-label-merge-conflict/compare/v3.0.0...v3.0.1) --- updated-dependencies: - dependency-name: eps1lon/actions-label-merge-conflict dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/merge-conflict.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge-conflict.yml b/.github/workflows/merge-conflict.yml index d9ceacca..491ede68 100644 --- a/.github/workflows/merge-conflict.yml +++ b/.github/workflows/merge-conflict.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check if PRs are have merge conflicts - uses: eps1lon/actions-label-merge-conflict@v3.0.0 + uses: eps1lon/actions-label-merge-conflict@v3.0.1 with: dirtyLabel: "PR: Merge Conflict" repoToken: "${{ secrets.GITHUB_TOKEN }}" From 3ad90c10066852fccd7fdc1535f2b4c39f7b5eee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 11 May 2024 15:47:44 +0000 Subject: [PATCH 221/413] Bump actions/checkout from 4.1.4 to 4.1.5 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.4 to 4.1.5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.4...v4.1.5) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 51920384..910c0b24 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 51289101..ab647ea2 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 8e024b10..f6361037 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 71bd04bd..fec902df 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 - name: Check scripts in repository are executable run: | @@ -72,7 +72,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 - name: Set up Python 3.10 uses: actions/setup-python@v5.1.0 From 5270336679030dce6f4c3d01201014e178113136 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 May 2024 10:56:18 +0000 Subject: [PATCH 222/413] Bump actions/checkout from 4.1.5 to 4.1.6 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.5 to 4.1.6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.5...v4.1.6) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 910c0b24..dd97f2ee 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index ab647ea2..3e9ccbc3 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index f6361037..d215e80f 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fec902df..568c4669 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 - name: Check scripts in repository are executable run: | @@ -72,7 +72,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 - name: Set up Python 3.10 uses: actions/setup-python@v5.1.0 From d30a7612abe969da015ad93b651cfff3e6e08c88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 25 May 2024 11:02:34 +0000 Subject: [PATCH 223/413] Bump pytest from 8.2.0 to 8.2.1 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.2.0 to 8.2.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.2.0...8.2.1) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 76af80a3..3ece8551 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.2.0 +pytest == 8.2.1 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.0 tox == 4.15.0 From 7b19b650d48854d0c96bda78f58d76c90d25b2e1 Mon Sep 17 00:00:00 2001 From: Jack'lul Date: Sun, 26 May 2024 08:38:03 +0200 Subject: [PATCH 224/413] Fix version check for release Docker images Signed-off-by: Jack'lul --- advanced/Scripts/updatecheck.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index eda6c403..b325ee9c 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -54,7 +54,8 @@ chmod 644 "${VERSION_FILE}" # if /pihole.docker.tag file exists, we will use it's value later in this script DOCKER_TAG=$(cat /pihole.docker.tag 2>/dev/null) -regex='^([0-9]+\.){1,2}(\*|[0-9]+)(-.*)?$|(^nightly$)|(^dev.*$)' +release_regex='^([0-9]+\.){1,2}(\*|[0-9]+)(-.*)?$' +regex=$release_regex'|(^nightly$)|(^dev.*$)' if [[ ! "${DOCKER_TAG}" =~ $regex ]]; then # DOCKER_TAG does not match the pattern (see https://regex101.com/r/RsENuz/1), so unset it. unset DOCKER_TAG @@ -121,6 +122,12 @@ addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_FTL_HASH" "${GITHUB_FTL_HASH}" if [[ "${DOCKER_TAG}" ]]; then addOrEditKeyValPair "${VERSION_FILE}" "DOCKER_VERSION" "${DOCKER_TAG}" - GITHUB_DOCKER_VERSION="$(get_remote_version docker-pi-hole)" + # Remote version check only if the tag is a valid release version + docker_branch="" + if [[ "${DOCKER_TAG}" =~ $release_regex ]]; then + docker_branch="master" + fi + + GITHUB_DOCKER_VERSION="$(get_remote_version docker-pi-hole "${docker_branch}")" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_DOCKER_VERSION" "${GITHUB_DOCKER_VERSION}" fi From 4add164d43bfbaee81c563857b93bfb350d5a32b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Jun 2024 10:20:46 +0000 Subject: [PATCH 225/413] Bump eps1lon/actions-label-merge-conflict from 3.0.1 to 3.0.2 Bumps [eps1lon/actions-label-merge-conflict](https://github.com/eps1lon/actions-label-merge-conflict) from 3.0.1 to 3.0.2. - [Release notes](https://github.com/eps1lon/actions-label-merge-conflict/releases) - [Changelog](https://github.com/eps1lon/actions-label-merge-conflict/blob/main/CHANGELOG.md) - [Commits](https://github.com/eps1lon/actions-label-merge-conflict/compare/v3.0.1...v3.0.2) --- updated-dependencies: - dependency-name: eps1lon/actions-label-merge-conflict dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/merge-conflict.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge-conflict.yml b/.github/workflows/merge-conflict.yml index 491ede68..4e56ae3e 100644 --- a/.github/workflows/merge-conflict.yml +++ b/.github/workflows/merge-conflict.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check if PRs are have merge conflicts - uses: eps1lon/actions-label-merge-conflict@v3.0.1 + uses: eps1lon/actions-label-merge-conflict@v3.0.2 with: dirtyLabel: "PR: Merge Conflict" repoToken: "${{ secrets.GITHUB_TOKEN }}" From 628e1bbe413774c16422f944f1f80eaadae4220f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Jun 2024 10:22:49 +0000 Subject: [PATCH 226/413] Bump pytest-testinfra from 10.1.0 to 10.1.1 in /test Bumps [pytest-testinfra](https://github.com/pytest-dev/pytest-testinfra) from 10.1.0 to 10.1.1. - [Release notes](https://github.com/pytest-dev/pytest-testinfra/releases) - [Changelog](https://github.com/pytest-dev/pytest-testinfra/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-testinfra/compare/10.1.0...10.1.1) --- updated-dependencies: - dependency-name: pytest-testinfra dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 3ece8551..90c15130 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 8.2.1 pytest-xdist == 3.6.1 -pytest-testinfra == 10.1.0 +pytest-testinfra == 10.1.1 tox == 4.15.0 From e232361b2d779dc42bd08fa4ffb436a8f2989a21 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 7 Jun 2024 19:02:43 +0200 Subject: [PATCH 227/413] Add CAP_SYS_TIME to FTL's ambient capabilities Signed-off-by: DL6ER --- advanced/Templates/pihole-FTL.service | 2 +- advanced/Templates/pihole-FTL.systemd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service index 460339ae..009401fc 100644 --- a/advanced/Templates/pihole-FTL.service +++ b/advanced/Templates/pihole-FTL.service @@ -37,7 +37,7 @@ start() { # Run pre-start script, which pre-creates all expected files with correct permissions sh "${PI_HOLE_SCRIPT_DIR}/pihole-FTL-prestart.sh" - if setcap CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_NICE,CAP_IPC_LOCK,CAP_CHOWN+eip "/usr/bin/pihole-FTL"; then + if setcap CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_NICE,CAP_IPC_LOCK,CAP_CHOWN,CAP_SYS_TIME+eip "/usr/bin/pihole-FTL"; then su -s /bin/sh -c "/usr/bin/pihole-FTL" pihole else echo "Warning: Starting pihole-FTL as root because setting capabilities is not supported on this system" diff --git a/advanced/Templates/pihole-FTL.systemd b/advanced/Templates/pihole-FTL.systemd index 2a114199..0a3d270e 100644 --- a/advanced/Templates/pihole-FTL.systemd +++ b/advanced/Templates/pihole-FTL.systemd @@ -18,7 +18,7 @@ StartLimitIntervalSec=60s [Service] User=pihole PermissionsStartOnly=true -AmbientCapabilities=CAP_NET_BIND_SERVICE CAP_NET_RAW CAP_NET_ADMIN CAP_SYS_NICE CAP_IPC_LOCK CAP_CHOWN +AmbientCapabilities=CAP_NET_BIND_SERVICE CAP_NET_RAW CAP_NET_ADMIN CAP_SYS_NICE CAP_IPC_LOCK CAP_CHOWN CAP_SYS_TIME ExecStartPre=/opt/pihole/pihole-FTL-prestart.sh ExecStart=/usr/bin/pihole-FTL -f From 2fdb6559dcfb0b036b9aef4d9a64e363965053cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 8 Jun 2024 10:15:56 +0000 Subject: [PATCH 228/413] Bump tox from 4.15.0 to 4.15.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.15.0 to 4.15.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.15.0...4.15.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 90c15130..5cfd7e68 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.2.1 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.15.0 +tox == 4.15.1 From 0742c017a894cc4c69f14c02fa6e55c107468f4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 8 Jun 2024 10:57:29 +0000 Subject: [PATCH 229/413] Bump pytest from 8.2.1 to 8.2.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.2.1 to 8.2.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.2.1...8.2.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 5cfd7e68..557d6497 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.2.1 +pytest == 8.2.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 tox == 4.15.1 From cfc0d0717342342fa5f45df2ec29bc0f3a39685e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 8 Jun 2024 12:57:45 +0200 Subject: [PATCH 230/413] Remove CentOS8 from test suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 1 - test/_centos_8.Dockerfile | 18 ------------------ test/tox.centos_8.ini | 8 -------- 3 files changed, 27 deletions(-) delete mode 100644 test/_centos_8.Dockerfile delete mode 100644 test/tox.centos_8.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 568c4669..5a5ee85f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,7 +63,6 @@ jobs: ubuntu_22, ubuntu_23, ubuntu_24, - centos_8, centos_9, fedora_39, fedora_40, diff --git a/test/_centos_8.Dockerfile b/test/_centos_8.Dockerfile deleted file mode 100644 index a07a67e9..00000000 --- a/test/_centos_8.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM quay.io/centos/centos:stream8 -RUN yum install -y git initscripts - -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole - -RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole -ADD . $GITDIR -RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR - -RUN true && \ - chmod +x $SCRIPTDIR/* - -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net - -#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/tox.centos_8.ini b/test/tox.centos_8.ini deleted file mode 100644 index 85ae1ffb..00000000 --- a/test/tox.centos_8.ini +++ /dev/null @@ -1,8 +0,0 @@ -[tox] -envlist = py3 - -[testenv:py3] -allowlist_externals = docker -deps = -rrequirements.txt -commands = docker buildx build --load --progress plain -f _centos_8.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py From 5d2cb552d3e6309330eef6a02cd5d99b83f36247 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 15 Jun 2024 10:25:08 +0000 Subject: [PATCH 231/413] Bump actions/checkout from 4.1.6 to 4.1.7 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.6 to 4.1.7. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.6...v4.1.7) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index dd97f2ee..0a5b59b9 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 3e9ccbc3..2218ea02 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index d215e80f..fe24c9b5 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5a5ee85f..1da17f0a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 - name: Check scripts in repository are executable run: | @@ -71,7 +71,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 - name: Set up Python 3.10 uses: actions/setup-python@v5.1.0 From 520641fa5e26e54f92d53d8e4c4d16d2b702fbfd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 19 Jun 2024 22:18:11 +0200 Subject: [PATCH 232/413] Try to use the CLI password for logins (if enabled and readable by the current user) Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 12 +++++++++--- advanced/Scripts/query.sh | 16 ++++------------ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 18a48ce7..efffa25a 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -75,9 +75,15 @@ TestAPIAvailability() { } Authentication() { - # Try to authenticate - LoginAPI + # Try to read the CLI password (if enabled and readable by the current user) + if [ -r /etc/pihole/cli_pw ]; then + password=$(cat /etc/pihole/cli_pw) + # Try to authenticate using the CLI password + LoginAPI + fi + + # If this did not work, ask the user for the password while [ "${validSession}" = false ] || [ -z "${validSession}" ] ; do echo "Authentication failed. Please enter your Pi-hole password" @@ -105,7 +111,7 @@ LoginAPI() { SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null) } -DeleteSession() { +LogoutAPI() { # if a valid Session exists (no password required or successful Authentication) and # SID is not null (successful Authentication only), delete the session if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 493c75ea..123eee21 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -128,22 +128,14 @@ Main() { # Test if the authentication endpoint is available TestAPIAvailability - # Users can configure FTL in a way, that for accessing a) all endpoints (webserver.api.localAPIauth) - # or b) for the /search endpoint (webserver.api.searchAPIauth) no authentication is required. - # Therefore, we try to query directly without authentication but do authenticat if 401 is returned + # Authenticate with FTL + Authentication + # send query again data=$(GetFTLData "search/${domain}?N=${max_results}&partial=${partial}") - if [ "${data}" = 401 ]; then - # Unauthenticated, so authenticate with the FTL server required - Authentication - - # send query again - data=$(GetFTLData "search/${domain}?N=${max_results}&partial=${partial}") - fi - GenerateOutput "${data}" - DeleteSession + LogoutAPI } # Process all options (if present) From a2951cd3b8c220d09e7e70a24a747cda1a9bf129 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 19 Jun 2024 22:19:54 +0200 Subject: [PATCH 233/413] Hide successful login/logout messages to avoid cluttering the terminal Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index efffa25a..3d35742f 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -94,9 +94,6 @@ Authentication() { LoginAPI done - # Loop exited, authentication was successful - echo "Authentication successful." - } LoginAPI() { @@ -119,7 +116,6 @@ LogoutAPI() { deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in - "204") printf "%b" "Session successfully deleted.\n";; "401") printf "%b" "Logout attempt without a valid session. Unauthorized!\n";; esac; fi From 4df7cee6c20bf3bbcf3ad806345f3b8dc82aced3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 19 Jun 2024 22:21:43 +0200 Subject: [PATCH 234/413] Add partial matching hint if exact matching found nothing Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 4 ++-- advanced/Scripts/query.sh | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 3d35742f..4162eff1 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -74,7 +74,7 @@ TestAPIAvailability() { fi } -Authentication() { +LoginAPI() { # Try to read the CLI password (if enabled and readable by the current user) if [ -r /etc/pihole/cli_pw ]; then password=$(cat /etc/pihole/cli_pw) @@ -96,7 +96,7 @@ Authentication() { } -LoginAPI() { +Authentication() { sessionResponse="$(curl -skS -X POST "${API_URL}auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" if [ -z "${sessionResponse}" ]; then diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 123eee21..c76e890e 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -112,6 +112,12 @@ GenerateOutput() { printf "\n\n" done fi + + # If no exact results were found, suggest using partial matching + if [ "${num_lists}" -eq 0 ] && [ "${num_gravity}" -eq 0 ] && [ "${partial}" = false ]; then + printf "%s\n" "Hint: Try partial matching with" + printf "%s\n\n" " ${COL_GREEN}pihole -q --partial ${domain}${COL_NC}" + fi } Main() { @@ -129,12 +135,14 @@ Main() { TestAPIAvailability # Authenticate with FTL - Authentication + LoginAPI # send query again data=$(GetFTLData "search/${domain}?N=${max_results}&partial=${partial}") GenerateOutput "${data}" + + # Delete session LogoutAPI } From 5cb9f4faaafd2fdb78c3e56b454ea65ea8687e05 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 19 Jun 2024 22:28:14 +0200 Subject: [PATCH 235/413] Modify pihole -f to use TOML config items Signed-off-by: DL6ER --- advanced/Scripts/piholeLogFlush.sh | 44 +++++++++++++++++------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 4d97fec5..892645af 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -11,27 +11,29 @@ colfile="/opt/pihole/COL_TABLE" source ${colfile} +readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" +utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" +source "${utilsfile}" + # In case we're running at the same time as a system logrotate, use a # separate logrotate state file to prevent stepping on each other's # toes. STATEFILE="/var/lib/logrotate/pihole" # Determine database location -# Obtain DBFILE=... setting from pihole-FTL.db -# Constructed to return nothing when -# a) the setting is not present in the config file, or -# b) the setting is commented out (e.g. "#DBFILE=...") -FTLconf="/etc/pihole/pihole-FTL.conf" -if [ -e "$FTLconf" ]; then - DBFILE="$(sed -n -e 's/^\s*DBFILE\s*=\s*//p' ${FTLconf})" -fi -# Test for empty string. Use standard path in this case. +DBFILE=$(getFTLConfigValue "files.database") if [ -z "$DBFILE" ]; then DBFILE="/etc/pihole/pihole-FTL.db" fi +# Determine log file location +LOGFILE=$(getFTLConfigValue "files.log.dnsmasq") +if [ -z "$LOGFILE" ]; then + LOGFILE="/var/log/pihole.log" +fi + if [[ "$*" != *"quiet"* ]]; then - echo -ne " ${INFO} Flushing /var/log/pihole/pihole.log ..." + echo -ne " ${INFO} Flushing "${LOGFILE}" ..." fi if [[ "$*" == *"once"* ]]; then # Nightly logrotation @@ -44,9 +46,9 @@ if [[ "$*" == *"once"* ]]; then # Note that moving the file is not an option, as # dnsmasq would happily continue writing into the # moved file (it will have the same file handler) - cp -p /var/log/pihole/pihole.log /var/log/pihole/pihole.log.1 - echo " " > /var/log/pihole/pihole.log - chmod 640 /var/log/pihole/pihole.log + cp -p "${LOGFILE}" "${LOGFILE}.1" + echo " " > "${LOGFILE}" + chmod 640 "${LOGFILE}" fi else # Manual flushing @@ -56,17 +58,21 @@ else /usr/sbin/logrotate --force --state "${STATEFILE}" /etc/pihole/logrotate else # Flush both pihole.log and pihole.log.1 (if existing) - echo " " > /var/log/pihole/pihole.log - if [ -f /var/log/pihole/pihole.log.1 ]; then - echo " " > /var/log/pihole/pihole.log.1 - chmod 640 /var/log/pihole/pihole.log.1 + echo " " > "${LOGFILE}" + if [ -f "${LOGFILE}.1" ]; then + echo " " > "${LOGFILE}.1" + chmod 640 "${LOGFILE}.1" fi fi + + # Stop FTL to make sure it doesn't write to the database while we're deleting data + service pihole-FTL stop + # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history) deleted=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1") - # Restart pihole-FTL to force reloading history - sudo pihole restartdns + # Restart FTL + service pihole-FTL restart fi if [[ "$*" != *"quiet"* ]]; then From 8f24e8aa5f67e5386cd36f87bb6f9ab9d6d34e9a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 19 Jun 2024 22:41:42 +0200 Subject: [PATCH 236/413] Modify pihole -t to use TOML config items Signed-off-by: DL6ER --- pihole | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pihole b/pihole index 7c84771c..f7963d73 100755 --- a/pihole +++ b/pihole @@ -391,19 +391,20 @@ exit 0 tailFunc() { # Warn user if Pi-hole's logging is disabled - local logging_enabled=$(grep -c "^log-queries" /etc/dnsmasq.d/01-pihole.conf) - if [[ "${logging_enabled}" == "0" ]]; then - # No "log-queries" lines are found. - # Commented out lines (such as "#log-queries") are ignored + local logging_enabled=$(getFTLConfigValue dns.queryLogging) + if [[ "${logging_enabled}" != "true" ]]; then echo " ${CROSS} Warning: Query logging is disabled" fi echo -e " ${INFO} Press Ctrl-C to exit" + # Get logfile path + readonly LOGFILE=$(getFTLConfigValue files.log.dnsmasq) + # Strip date from each line # Color blocklist/blacklist/wildcard entries as red # Color A/AAAA/DHCP strings as white # Color everything else as gray - tail -f /var/log/pihole/pihole.log | grep --line-buffered "${1}" | sed -E \ + tail -f $LOGFILE | grep --line-buffered "${1}" | sed -E \ -e "s,($(date +'%b %d ')| dnsmasq\[[0-9]*\]),,g" \ -e "s,(.*(blacklisted |gravity blocked ).*),${COL_RED}&${COL_NC}," \ -e "s,.*(query\\[A|DHCP).*,${COL_NC}&${COL_NC}," \ From 92b15cf7441462fceefeb0cd936845b0a5be984a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 19 Jun 2024 22:49:11 +0200 Subject: [PATCH 237/413] Modify pihole arpflush to stop FTL while performing the action and use the new TOML config values Signed-off-by: DL6ER --- advanced/Scripts/piholeARPTable.sh | 36 +++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/advanced/Scripts/piholeARPTable.sh b/advanced/Scripts/piholeARPTable.sh index b92dd124..c04c5b33 100755 --- a/advanced/Scripts/piholeARPTable.sh +++ b/advanced/Scripts/piholeARPTable.sh @@ -15,27 +15,29 @@ if [[ -f ${coltable} ]]; then source ${coltable} fi +readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" +utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" +source "${utilsfile}" + # Determine database location -# Obtain DBFILE=... setting from pihole-FTL.db -# Constructed to return nothing when -# a) the setting is not present in the config file, or -# b) the setting is commented out (e.g. "#DBFILE=...") -FTLconf="/etc/pihole/pihole-FTL.conf" -if [ -e "$FTLconf" ]; then - DBFILE="$(sed -n -e 's/^\s*DBFILE\s*=\s*//p' ${FTLconf})" -fi -# Test for empty string. Use standard path in this case. +DBFILE=$(getFTLConfigValue "files.database") if [ -z "$DBFILE" ]; then DBFILE="/etc/pihole/pihole-FTL.db" fi - flushARP(){ local output if [[ "${args[1]}" != "quiet" ]]; then echo -ne " ${INFO} Flushing network table ..." fi + # Stop FTL to prevent database access + if ! output=$(pihole-FTL service stop 2>&1); then + echo -e "${OVER} ${CROSS} Failed to stop FTL" + echo " Output: ${output}" + return 1 + fi + # Truncate network_addresses table in pihole-FTL.db # This needs to be done before we can truncate the network table due to # foreign key constraints @@ -54,6 +56,20 @@ flushARP(){ return 1 fi + # Flush ARP cache of the host + if ! output=$(ip -s -s neigh flush all 2>&1); then + echo -e "${OVER} ${CROSS} Failed to flush ARP cache" + echo " Output: ${output}" + return 1 + fi + + # Start FTL again + if ! output=$(pihole-FTL service restart 2>&1); then + echo -e "${OVER} ${CROSS} Failed to restart FTL" + echo " Output: ${output}" + return 1 + fi + if [[ "${args[1]}" != "quiet" ]]; then echo -e "${OVER} ${TICK} Flushed network table" fi From bfc18f8329ad6eca2d4c2e3ea9e641d86844a453 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 19 Jun 2024 23:04:39 +0200 Subject: [PATCH 238/413] Rewrite list functions to use the API Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 18 +- advanced/Scripts/list.sh | 358 +++++++++++++++------------------------ pihole | 20 +-- 3 files changed, 164 insertions(+), 232 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 4162eff1..21447105 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -75,12 +75,16 @@ TestAPIAvailability() { } LoginAPI() { + if [ -z "${API_URL}" ]; then + TestAPIAvailability + fi + # Try to read the CLI password (if enabled and readable by the current user) if [ -r /etc/pihole/cli_pw ]; then password=$(cat /etc/pihole/cli_pw) # Try to authenticate using the CLI password - LoginAPI + Authentication fi # If this did not work, ask the user for the password @@ -91,7 +95,7 @@ LoginAPI() { secretRead; printf '\n' # Try to authenticate again - LoginAPI + Authentication done } @@ -144,6 +148,16 @@ GetFTLData() { fi } +PostFTLData() { + local data response status + # send the data to the API + response=$(curl -skS -w "%{http_code}" -X POST "${API_URL}$1" --data-raw "$2" -H "Accept: application/json" -H "sid: ${SID}" ) + # status are the last 3 characters + status=$(printf %s "${response#"${response%???}"}") + # data is everything from response without the last 3 characters + printf %s "${response%???}" +} + secretRead() { # POSIX compliant function to read user-input and diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 76558e58..3bd4af75 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -5,261 +5,187 @@ # (c) 2017 Pi-hole, LLC (https://pi-hole.net) # Network-wide ad blocking via your own hardware. # -# Whitelist and blacklist domains +# allowlist and denylist domains # # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. -# Globals -piholeDir="/etc/pihole" -GRAVITYDB="${piholeDir}/gravity.db" -# Source pihole-FTL from install script -pihole_FTL="${piholeDir}/pihole-FTL.conf" -if [[ -f "${pihole_FTL}" ]]; then - source "${pihole_FTL}" +readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" +readonly utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" +source "${utilsfile}" + +readonly apifile="${PI_HOLE_SCRIPT_DIR}/api.sh" +source "${apifile}" + +# Determine database location +DBFILE=$(getFTLConfigValue "files.database") +if [ -z "$DBFILE" ]; then + DBFILE="/etc/pihole/pihole-FTL.db" fi -# Set this only after sourcing pihole-FTL.conf as the gravity database path may -# have changed -gravityDBfile="${GRAVITYDB}" +# Determine gravity database location +GRAVITYDB=$(getFTLConfigValue "files.gravity") +if [ -z "$GRAVITYDB" ]; then + GRAVITYDB="/etc/pihole/gravity.db" +fi -noReloadRequested=false addmode=true verbose=true wildcard=false -web=false domList=() typeId="" comment="" -declare -i domaincount -domaincount=0 -reload=false colfile="/opt/pihole/COL_TABLE" source ${colfile} -# IDs are hard-wired to domain interpretation in the gravity database scheme -# Clients (including FTL) will read them through the corresponding views -readonly whitelist="0" -readonly blacklist="1" -readonly regex_whitelist="2" -readonly regex_blacklist="3" - -GetListnameFromTypeId() { - if [[ "$1" == "${whitelist}" ]]; then - echo "whitelist" - elif [[ "$1" == "${blacklist}" ]]; then - echo "blacklist" - elif [[ "$1" == "${regex_whitelist}" ]]; then - echo "regex whitelist" - elif [[ "$1" == "${regex_blacklist}" ]]; then - echo "regex blacklist" - fi -} - -GetListParamFromTypeId() { - if [[ "${typeId}" == "${whitelist}" ]]; then - echo "w" - elif [[ "${typeId}" == "${blacklist}" ]]; then - echo "b" - elif [[ "${typeId}" == "${regex_whitelist}" && "${wildcard}" == true ]]; then - echo "-white-wild" - elif [[ "${typeId}" == "${regex_whitelist}" ]]; then - echo "-white-regex" - elif [[ "${typeId}" == "${regex_blacklist}" && "${wildcard}" == true ]]; then - echo "-wild" - elif [[ "${typeId}" == "${regex_blacklist}" ]]; then - echo "-regex" - fi -} - helpFunc() { - local listname param - - listname="$(GetListnameFromTypeId "${typeId}")" - param="$(GetListParamFromTypeId)" - - echo "Usage: pihole -${param} [options] -Example: 'pihole -${param} site.com', or 'pihole -${param} site1.com site2.com' -${listname^} one or more domains + echo "Usage: pihole ${abbrv} [options] +Example: 'pihole ${abbrv} site.com', or 'pihole ${abbrv} site1.com site2.com' +${typeId^} one or more ${kindId} domains Options: - -d, --delmode Remove domain(s) from the ${listname} - -nr, --noreload Update ${listname} without reloading the DNS server + -d, --delmode Remove domain(s) -q, --quiet Make output less verbose -h, --help Show this help dialog - -l, --list Display all your ${listname}listed domains + -l, --list Display domains --nuke Removes all entries in a list --comment \"text\" Add a comment to the domain. If adding multiple domains the same comment will be used for all" exit 0 } -ValidateDomain() { - # Convert to lowercase - domain="${1,,}" - local str validDomain - - # Check validity of domain (don't check for regex entries) - if [[ ( "${typeId}" == "${regex_blacklist}" || "${typeId}" == "${regex_whitelist}" ) && "${wildcard}" == false ]]; then - validDomain="${domain}" - else - # Check max length - if [[ "${#domain}" -le 253 ]]; then - validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check - validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label - # set error string - str="is not a valid argument or domain name!" - else - validDomain= - str="is too long!" - - fi +CreateDomainList() { + # Format domain into regex filter if requested + local dom=${1} + if [[ "${wildcard}" == true ]]; then + dom="(\\.|^)${dom//\./\\.}$" fi - - if [[ -n "${validDomain}" ]]; then - domList=("${domList[@]}" "${validDomain}") - else - echo -e " ${CROSS} ${domain} ${str}" - fi - - domaincount=$((domaincount+1)) -} - -ProcessDomainList() { - for dom in "${domList[@]}"; do - # Format domain into regex filter if requested - if [[ "${wildcard}" == true ]]; then - dom="(\\.|^)${dom//\./\\.}$" - fi - - # Logic: If addmode then add to desired list and remove from the other; - # if delmode then remove from desired list but do not add to the other - if ${addmode}; then - AddDomain "${dom}" - else - RemoveDomain "${dom}" - fi - done + domList=("${domList[@]}" "${dom}") } AddDomain() { - local domain num requestedListname existingTypeId existingListname - domain="$1" + local json num - # Is the domain in the list we want to add it to? - num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")" - requestedListname="$(GetListnameFromTypeId "${typeId}")" + # Authenticate with the API + LoginAPI - if [[ "${num}" -ne 0 ]]; then - existingTypeId="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")" - if [[ "${existingTypeId}" == "${typeId}" ]]; then - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} already exists in ${requestedListname}, no need to add!" + # Prepare request to POST /api/domains/{type}/{kind} + # Build JSON object of the following form + # { + # "domain": [ ], + # "comment": + # } + # where is an array of domain strings and is a string + # We use jq to build the JSON object + json=$(jq --null-input --compact-output --arg domains "${domList[*]}" --arg comment "${comment}" '{domain: $domains | split(" "), comment: $comment}') + + # Send the request + data=$(PostFTLData "domains/${typeId}/${kindId}" "${json}") + + # Display domain(s) added + # (they are listed in .processed.success, use jq) + num=$(echo "${data}" | jq '.processed.success | length') + if [[ "${num}" -gt 0 ]] && [[ "${verbose}" == true ]]; then + echo -e " ${TICK} Added ${num} domain(s):" + for i in $(seq 0 $((num-1))); do + echo -e " - ${COL_BLUE}$(echo "${data}" | jq --raw-output ".processed.success[$i].item")${COL_NC}" + done + fi + # Display failed domain(s) + # (they are listed in .processed.errors, use jq) + num=$(echo "${data}" | jq '.processed.errors | length') + if [[ "${num}" -gt 0 ]] && [[ "${verbose}" == true ]]; then + echo -e " ${CROSS} Failed to add ${num} domain(s):" + for i in $(seq 0 $((num-1))); do + echo -e " - ${COL_BLUE}$(echo "${data}" | jq --raw-output ".processed.errors[$i].item")${COL_NC}" + error=$(echo "${data}" | jq --raw-output ".processed.errors[$i].error") + if [[ "${error}" == "UNIQUE constraint failed: domainlist.domain, domainlist.type" ]]; then + error="Domain already in the specified list" fi - else - existingListname="$(GetListnameFromTypeId "${existingTypeId}")" - pihole-FTL sqlite3 -ni "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';" - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} already exists in ${existingListname}, it has been moved to ${requestedListname}!" - fi - fi - return + echo -e " ${error}" + done fi - # Domain not found in the table, add it! - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding ${domain} to the ${requestedListname}..." - fi - reload=true - # Insert only the domain here. The enabled and date_added fields will be filled - # with their default values (enabled = true, date_added = current timestamp) - if [[ -z "${comment}" ]]; then - pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});" - else - # also add comment when variable has been set through the "--comment" option - pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');" - fi + # Log out + LogoutAPI } RemoveDomain() { - local domain num requestedListname - domain="$1" + local json num - # Is the domain in the list we want to remove it from? - num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")" + # Authenticate with the API + LoginAPI - requestedListname="$(GetListnameFromTypeId "${typeId}")" + # Prepare request to POST /api/domains:batchDelete + # Build JSON object of the following form + # [{ + # "item": , + # "type": "${typeId}", + # "kind": "${kindId}", + # }] + # where is the domain string and ${typeId} and ${kindId} are the type and kind IDs + # We use jq to build the JSON object) + json=$(jq --null-input --compact-output --arg domains "${domList[*]}" --arg typeId "${typeId}" --arg kindId "${kindId}" '[ $domains | split(" ")[] as $item | {item: $item, type: $typeId, kind: $kindId} ]') - if [[ "${num}" -eq 0 ]]; then - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${domain} does not exist in ${requestedListname}, no need to remove!" - fi - return + # Send the request + data=$(PostFTLData "domains:batchDelete" "${json}") + + # If there is an .error object in the returned data, display it + local error + error=$(jq --compact-output <<< "${data}" '.error') + if [[ $error != "null" && $error != "" ]]; then + echo -e " ${CROSS} Failed to remove domain(s):" + echo -e " $(jq <<< "${data}" '.error')" + elif [[ "${verbose}" == true ]]; then + echo -e " ${TICK} Removed ${#domList[@]} domain(s):" + # Loop through the domains and display them + for dom in "${domList[@]}"; do + echo -e " - ${COL_BLUE}${dom}${COL_NC}" + done fi - # Domain found in the table, remove it! - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Removing ${domain} from the ${requestedListname}..." - fi - reload=true - # Remove it from the current list - pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};" + # Log out + LogoutAPI } Displaylist() { - local count num_pipes domain enabled status nicedate requestedListname + local data - requestedListname="$(GetListnameFromTypeId "${typeId}")" - data="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)" - - if [[ -z $data ]]; then - echo -e "Not showing empty list" - else - echo -e "Displaying ${requestedListname}:" - count=1 - while IFS= read -r line - do - # Count number of pipes seen in this line - # This is necessary because we can only detect the pipe separating the fields - # from the end backwards as the domain (which is the first field) may contain - # pipe symbols as they are perfectly valid regex filter control characters - num_pipes="$(grep -c "^" <<< "$(grep -o "|" <<< "${line}")")" - - # Extract domain and enabled status based on the obtained number of pipe characters - domain="$(cut -d'|' -f"-$((num_pipes-1))" <<< "${line}")" - enabled="$(cut -d'|' -f"$((num_pipes))" <<< "${line}")" - datemod="$(cut -d'|' -f"$((num_pipes+1))" <<< "${line}")" - - # Translate boolean status into human readable string - if [[ "${enabled}" -eq 1 ]]; then - status="enabled" - else - status="disabled" - fi - - # Get nice representation of numerical date stored in database - nicedate=$(date --rfc-2822 -d "@${datemod}") - - echo " ${count}: ${domain} (${status}, last modified ${nicedate})" - count=$((count+1)) - done <<< "${data}" + # if either typeId or kindId is empty, we cannot display the list + if [[ -z "${typeId}" ]] || [[ -z "${kindId}" ]]; then + echo " ${CROSS} Unable to display list. Please specify a list type and kind." + exit 1 fi - exit 0; -} -NukeList() { - count=$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};") - listname="$(GetListnameFromTypeId "${typeId}")" - if [ "$count" -gt 0 ];then - pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};" - echo " ${TICK} Removed ${count} domain(s) from the ${listname}" + # Authenticate with the API + LoginAPI + + # Send the request + data=$(GetFTLData "domains/${typeId}/${kindId}") + + # Display the list + num=$(echo "${data}" | jq '.domains | length') + if [[ "${num}" -gt 0 ]]; then + echo -e " ${TICK} Found ${num} domain(s) in the ${kindId} ${typeId}list:" + for i in $(seq 0 $((num-1))); do + echo -e " - ${COL_BLUE}$(echo "${data}" | jq --compact-output ".domains[$i].domain")${COL_NC}" + echo -e " Comment: $(echo "${data}" | jq --compact-output ".domains[$i].comment")" + echo -e " Groups: $(echo "${data}" | jq --compact-output ".domains[$i].groups")" + echo -e " Added: $(date -d @"$(echo "${data}" | jq --compact-output ".domains[$i].date_added")")" + echo -e " Last modified: $(date -d @"$(echo "${data}" | jq --compact-output ".domains[$i].date_modified")")" + done else - echo " ${INFO} ${listname} already empty. Nothing to do!" + echo -e " ${INFO} No domains found in the ${kindId} ${typeId}list" fi - exit 0; + + # Log out + LogoutAPI + + # Return early without adding/deleting domains + exit 0 } GetComment() { @@ -272,38 +198,30 @@ GetComment() { while (( "$#" )); do case "${1}" in - "-w" | "whitelist" ) typeId=0;; - "-b" | "blacklist" ) typeId=1;; - "--white-regex" | "white-regex" ) typeId=2;; - "--white-wild" | "white-wild" ) typeId=2; wildcard=true;; - "--wild" | "wildcard" ) typeId=3; wildcard=true;; - "--regex" | "regex" ) typeId=3;; - "-nr"| "--noreload" ) noReloadRequested=true;; + "-a" | "allowlist" ) kindId="exact"; typeId="allow"; abbrv="-a";; + "-b" | "denylist" ) kindId="exact"; typeId="deny"; abbrv="-b";; + "--allow-regex" | "allow-regex" ) kindId="regex"; typeId="allow"; abbrv="--allow-regex";; + "--allow-wild" | "allow-wild" ) kindId="regex"; typeId="allow"; wildcard=true; abbrv="--allow-wild";; + "--regex" | "regex" ) kindId="regex"; typeId="deny"; abbrv="--regex";; + "--wild" | "wildcard" ) kindId="regex"; typeId="deny"; wildcard=true; abbrv="--wild";; "-d" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; "-h" | "--help" ) helpFunc;; "-l" | "--list" ) Displaylist;; - "--nuke" ) NukeList;; - "--web" ) web=true;; "--comment" ) GetComment "${2}"; shift;; - * ) ValidateDomain "${1}";; + * ) CreateDomainList "${1}";; esac shift done shift -if [[ ${domaincount} == 0 ]]; then +if [[ ${#domList[@]} == 0 ]]; then helpFunc fi -ProcessDomainList - -# Used on web interface -if $web; then - echo "DONE" -fi - -if [[ ${reload} == true && ${noReloadRequested} == false ]]; then - pihole restartdns reload-lists +if ${addmode}; then + AddDomain +else + RemoveDomain fi diff --git a/pihole b/pihole index f7963d73..ce46fd0f 100755 --- a/pihole +++ b/pihole @@ -537,12 +537,12 @@ case "${1}" in "tricorder" ) tricorderFunc;; # we need to add all arguments that require sudo power to not trigger the * argument - "-w" | "whitelist" ) ;; - "-b" | "blacklist" ) ;; - "--wild" | "wildcard" ) ;; - "--regex" | "regex" ) ;; - "--white-regex" | "white-regex" ) ;; - "--white-wild" | "white-wild" ) ;; + "-a" | "allowlist" ) need_root=0;; + "-b" | "blocklist" | "denylist" ) need_root=0;; + "--wild" | "wildcard" ) need_root=0;; + "--regex" | "regex" ) need_root=0;; + "--allow-regex" | "allow-regex" ) need_root=0;; + "--allow-wild" | "allow-wild" ) need_root=0;; "-f" | "flush" ) ;; "-up" | "updatePihole" ) ;; "-r" | "reconfigure" ) ;; @@ -592,12 +592,12 @@ fi # Handle redirecting to specific functions based on arguments case "${1}" in - "-w" | "whitelist" ) listFunc "$@";; - "-b" | "blacklist" ) listFunc "$@";; + "-a" | "allowlist" ) listFunc "$@";; + "-b" | "blocklist" | "denylist" ) listFunc "$@";; "--wild" | "wildcard" ) listFunc "$@";; "--regex" | "regex" ) listFunc "$@";; - "--white-regex" | "white-regex" ) listFunc "$@";; - "--white-wild" | "white-wild" ) listFunc "$@";; + "--allow-regex" | "allow-regex" ) listFunc "$@";; + "--allow-wild" | "allow-wild" ) listFunc "$@";; "-d" | "debug" ) debugFunc "$@";; "-f" | "flush" ) flushFunc "$@";; "-up" | "updatePihole" ) updatePiholeFunc "$@";; From 25f384a923c3f0feccedc3ee4033cc482f2683c8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 20 Jun 2024 19:41:47 +0200 Subject: [PATCH 239/413] Do not use CLI password when no password is set at all Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 21447105..6f31a6d2 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -51,6 +51,12 @@ TestAPIAvailability() { API_PORT="" else # API is available at this URL combination + + if [ "${availabilityResonse}" = 200 ]; then + # API is available without authentication + needAuth=false + fi + break fi @@ -75,10 +81,16 @@ TestAPIAvailability() { } LoginAPI() { + # If the API URL is not set, test the availability if [ -z "${API_URL}" ]; then TestAPIAvailability fi + # Exit early if authentication is not needed + if [ "${needAuth}" = false ]; then + return + fi + # Try to read the CLI password (if enabled and readable by the current user) if [ -r /etc/pihole/cli_pw ]; then password=$(cat /etc/pihole/cli_pw) @@ -87,6 +99,8 @@ LoginAPI() { Authentication fi + + # If this did not work, ask the user for the password while [ "${validSession}" = false ] || [ -z "${validSession}" ] ; do echo "Authentication failed. Please enter your Pi-hole password" From 5dfcd02c40b9babd4f824805d53d4d919116a944 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 22 Jun 2024 10:31:03 +0200 Subject: [PATCH 240/413] Improve pihole -f Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 8 +-- advanced/Scripts/piholeLogFlush.sh | 79 +++++++++++++++++++++++------- 2 files changed, 64 insertions(+), 23 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 6f31a6d2..ea057adc 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -21,7 +21,7 @@ TestAPIAvailability() { # as we are running locally, we can get the port value from FTL directly - local chaos_api_list availabilityResonse + local chaos_api_list availabilityResponse # Query the API URLs from FTL using CHAOS TXT local.api.ftl # The result is a space-separated enumeration of full URLs @@ -43,16 +43,16 @@ TestAPIAvailability() { API_URL="${API_URL#\"}" # Test if the API is available at this URL - availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}auth") + availabilityResponse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}auth") # Test if http status code was 200 (OK) or 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 401 ]; then + if [ ! "${availabilityResponse}" = 200 ] && [ ! "${availabilityResponse}" = 401 ]; then # API is not available at this port/protocol combination API_PORT="" else # API is available at this URL combination - if [ "${availabilityResonse}" = 200 ]; then + if [ "${availabilityResponse}" = 200 ]; then # API is available without authentication needAuth=false fi diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 892645af..34d96318 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -29,16 +29,21 @@ fi # Determine log file location LOGFILE=$(getFTLConfigValue "files.log.dnsmasq") if [ -z "$LOGFILE" ]; then - LOGFILE="/var/log/pihole.log" + LOGFILE="/var/log/pihole/pihole.log" +fi +FTLFILE=$(getFTLConfigValue "files.log.ftl") +if [ -z "$FTLFILE" ]; then + FTLFILE="/var/log/pihole/FTL.log" fi -if [[ "$*" != *"quiet"* ]]; then - echo -ne " ${INFO} Flushing "${LOGFILE}" ..." -fi if [[ "$*" == *"once"* ]]; then # Nightly logrotation if command -v /usr/sbin/logrotate >/dev/null; then # Logrotate once + + if [[ "$*" != *"quiet"* ]]; then + echo -ne " ${INFO} Running logrotate ..." + fi /usr/sbin/logrotate --force --state "${STATEFILE}" /etc/pihole/logrotate else # Copy pihole.log over to pihole.log.1 @@ -46,23 +51,60 @@ if [[ "$*" == *"once"* ]]; then # Note that moving the file is not an option, as # dnsmasq would happily continue writing into the # moved file (it will have the same file handler) + if [[ "$*" != *"quiet"* ]]; then + echo -ne " ${INFO} Rotating ${LOGFILE} ..." + fi cp -p "${LOGFILE}" "${LOGFILE}.1" echo " " > "${LOGFILE}" chmod 640 "${LOGFILE}" + if [[ "$*" != *"quiet"* ]]; then + echo -e "${OVER} ${TICK} Rotated ${LOGFILE} ..." + fi + # Copy FTL.log over to FTL.log.1 + # and empty out FTL.log + if [[ "$*" != *"quiet"* ]]; then + echo -ne " ${INFO} Rotating ${FTLFILE} ..." + fi + cp -p "${FTLFILE}" "${FTLFILE}.1" + echo " " > "${FTLFILE}" + chmod 640 "${FTLFILE}" + if [[ "$*" != *"quiet"* ]]; then + echo -e "${OVER} ${TICK} Rotated ${FTLFILE} ..." + fi fi else # Manual flushing - if command -v /usr/sbin/logrotate >/dev/null; then - # Logrotate twice to move all data out of sight of FTL - /usr/sbin/logrotate --force --state "${STATEFILE}" /etc/pihole/logrotate; sleep 3 - /usr/sbin/logrotate --force --state "${STATEFILE}" /etc/pihole/logrotate - else - # Flush both pihole.log and pihole.log.1 (if existing) - echo " " > "${LOGFILE}" - if [ -f "${LOGFILE}.1" ]; then - echo " " > "${LOGFILE}.1" - chmod 640 "${LOGFILE}.1" - fi + + # Flush both pihole.log and pihole.log.1 (if existing) + if [[ "$*" != *"quiet"* ]]; then + echo -ne " ${INFO} Flushing ${LOGFILE} ..." + fi + echo " " > "${LOGFILE}" + chmod 640 "${LOGFILE}" + if [ -f "${LOGFILE}.1" ]; then + echo " " > "${LOGFILE}.1" + chmod 640 "${LOGFILE}.1" + fi + if [[ "$*" != *"quiet"* ]]; then + echo -e "${OVER} ${TICK} Flushed ${LOGFILE} ..." + fi + + # Flush both FTL.log and FTL.log.1 (if existing) + if [[ "$*" != *"quiet"* ]]; then + echo -ne " ${INFO} Flushing ${FTLFILE} ..." + fi + echo " " > "${FTLFILE}" + chmod 640 "${FTLFILE}" + if [ -f "${FTLFILE}.1" ]; then + echo " " > "${FTLFILE}.1" + chmod 640 "${FTLFILE}.1" + fi + if [[ "$*" != *"quiet"* ]]; then + echo -e "${OVER} ${TICK} Flushed ${FTLFILE} ..." + fi + + if [[ "$*" != *"quiet"* ]]; then + echo -ne " ${INFO} Flushing database, DNS resolution temporarily unavailable ..." fi # Stop FTL to make sure it doesn't write to the database while we're deleting data @@ -73,9 +115,8 @@ else # Restart FTL service pihole-FTL restart + if [[ "$*" != *"quiet"* ]]; then + echo -e "${OVER} ${TICK} Deleted ${deleted} queries from long-term query database" + fi fi -if [[ "$*" != *"quiet"* ]]; then - echo -e "${OVER} ${TICK} Flushed /var/log/pihole/pihole.log" - echo -e " ${TICK} Deleted ${deleted} queries from database" -fi From 7e91b9ab47f710311ed9cc2d509c5ea3a4c5405e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 22 Jun 2024 20:35:06 +0200 Subject: [PATCH 241/413] Update help text Signed-off-by: DL6ER --- pihole | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pihole b/pihole index ce46fd0f..d9a22c63 100755 --- a/pihole +++ b/pihole @@ -472,17 +472,17 @@ unsupportedFunc(){ helpFunc() { echo "Usage: pihole [options] -Example: 'pihole -w -h' +Example: 'pihole -a -h' Add '-h' after specific commands for more information on usage -Whitelist/Blacklist Options: - -w, whitelist Whitelist domain(s) - -b, blacklist Blacklist domain(s) +Domain Options: + -a, allowlist Allowlist domain(s) + -b, denylist Denylist domain(s) --regex, regex Regex blacklist domains(s) - --white-regex Regex whitelist domains(s) + --allow-regex Regex allowlist domains(s) --wild, wildcard Wildcard blacklist domain(s) - --white-wild Wildcard whitelist domain(s) - Add '-h' for more info on whitelist/blacklist usage + --allow-wild Wildcard allowlist domain(s) + Add '-h' for more info on allowlist/denylist usage Debugging Options: -d, debug Start a debugging session From 424e825bd9a28a416766067dc22821ae81bd139f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 22 Jun 2024 20:41:19 +0200 Subject: [PATCH 242/413] Do not auto-sudo in the pihole command Signed-off-by: DL6ER --- pihole | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/pihole b/pihole index d9a22c63..cae0cfb2 100755 --- a/pihole +++ b/pihole @@ -552,8 +552,8 @@ case "${1}" in "disable" ) ;; "-d" | "debug" ) ;; "restartdns" ) ;; - "-g" | "updateGravity" ) need_root=0;; - "reloaddns" ) need_root=0;; + "-g" | "updateGravity" ) ;; + "reloaddns" ) ;; "setpassword" ) ;; "checkout" ) ;; "updatechecker" ) ;; @@ -562,32 +562,18 @@ case "${1}" in * ) helpFunc;; esac -# Must be root to use this tool for most functions -if [[ ! $EUID -eq 0 && need_root -eq 1 ]];then - if [[ -x "$(command -v sudo)" ]]; then - exec sudo bash "$0" "$@" - exit $? - else - echo -e " ${CROSS} sudo is needed to run pihole commands. Please run this script as root or install sudo." - exit 1 - fi -fi - # In the case of alpine running in a container, the USER variable appears to be blank # which prevents the next trap from working correctly. Set it by running whoami if [[ -z ${USER} ]]; then USER=$(whoami) fi -# Can also be user pihole for other functions -if [[ ${USER} != "pihole" && need_root -eq 0 ]];then - if [[ -x "$(command -v sudo)" ]]; then - exec sudo -u pihole bash "$0" "$@" - exit $? - else - echo -e " ${CROSS} sudo is needed to run pihole commands. Please run this script as root or install sudo." - exit 1 - fi +# Check if the current user is neither root nor pihole and if the command +# requires root. If so, exit with an error message. +if [[ $EUID -ne 0 && ${USER} != "pihole" && need_root -eq 1 ]];then + echo -e " ${CROSS} The Pi-hole command requires root privileges, try:" + echo -e " ${COL_GREEN}sudo pihole ${@}${COL_NC}" + exit 1 fi # Handle redirecting to specific functions based on arguments From b835fa06a6360004c9fa75e56225fba06b7374f0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 22 Jun 2024 20:48:07 +0200 Subject: [PATCH 243/413] Further black -> deny renaming Signed-off-by: DL6ER --- pihole | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pihole b/pihole index cae0cfb2..00cbd893 100755 --- a/pihole +++ b/pihole @@ -401,12 +401,12 @@ tailFunc() { readonly LOGFILE=$(getFTLConfigValue files.log.dnsmasq) # Strip date from each line - # Color blocklist/blacklist/wildcard entries as red + # Color blocklist/denylist/wildcard entries as red # Color A/AAAA/DHCP strings as white # Color everything else as gray tail -f $LOGFILE | grep --line-buffered "${1}" | sed -E \ -e "s,($(date +'%b %d ')| dnsmasq\[[0-9]*\]),,g" \ - -e "s,(.*(blacklisted |gravity blocked ).*),${COL_RED}&${COL_NC}," \ + -e "s,(.*(denied |gravity blocked ).*),${COL_RED}&${COL_NC}," \ -e "s,.*(query\\[A|DHCP).*,${COL_NC}&${COL_NC}," \ -e "s,.*,${COL_GRAY}&${COL_NC}," exit 0 @@ -478,9 +478,9 @@ Add '-h' after specific commands for more information on usage Domain Options: -a, allowlist Allowlist domain(s) -b, denylist Denylist domain(s) - --regex, regex Regex blacklist domains(s) + --regex, regex Regex denylist domains(s) --allow-regex Regex allowlist domains(s) - --wild, wildcard Wildcard blacklist domain(s) + --wild, wildcard Wildcard denylist domain(s) --allow-wild Wildcard allowlist domain(s) Add '-h' for more info on allowlist/denylist usage From fe8e63853cf69e077eae5dedbe523f19ec0a5b0b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 22 Jun 2024 20:49:44 +0200 Subject: [PATCH 244/413] Use concatenate of arguments instead of using the array directly Signed-off-by: DL6ER --- pihole | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pihole b/pihole index 00cbd893..1313d487 100755 --- a/pihole +++ b/pihole @@ -572,7 +572,7 @@ fi # requires root. If so, exit with an error message. if [[ $EUID -ne 0 && ${USER} != "pihole" && need_root -eq 1 ]];then echo -e " ${CROSS} The Pi-hole command requires root privileges, try:" - echo -e " ${COL_GREEN}sudo pihole ${@}${COL_NC}" + echo -e " ${COL_GREEN}sudo pihole $*${COL_NC}" exit 1 fi From ccdbfd41309346ebb955e8910f2c130b92cd60d8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 25 Jun 2024 13:56:49 +0200 Subject: [PATCH 245/413] Use natural langauge for list manipulations, like pihole allow example.com or pihole deny other.net. Also remove using pihole deny not bad.org Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 9 ++++--- advanced/bash-completion/pihole | 6 ++--- manpages/pihole.8 | 42 +++++++++++++++------------------ pihole | 25 ++++++++++---------- 4 files changed, 39 insertions(+), 43 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 3bd4af75..144317ce 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -47,11 +47,10 @@ Example: 'pihole ${abbrv} site.com', or 'pihole ${abbrv} site1.com site2.com' ${typeId^} one or more ${kindId} domains Options: - -d, --delmode Remove domain(s) + not -d, --delmode Remove domain(s) -q, --quiet Make output less verbose -h, --help Show this help dialog -l, --list Display domains - --nuke Removes all entries in a list --comment \"text\" Add a comment to the domain. If adding multiple domains the same comment will be used for all" exit 0 @@ -198,13 +197,13 @@ GetComment() { while (( "$#" )); do case "${1}" in - "-a" | "allowlist" ) kindId="exact"; typeId="allow"; abbrv="-a";; - "-b" | "denylist" ) kindId="exact"; typeId="deny"; abbrv="-b";; + "allow" | "allowlist" ) kindId="exact"; typeId="allow"; abbrv="allow";; + "deny" | "denylist" ) kindId="exact"; typeId="deny"; abbrv="deny";; "--allow-regex" | "allow-regex" ) kindId="regex"; typeId="allow"; abbrv="--allow-regex";; "--allow-wild" | "allow-wild" ) kindId="regex"; typeId="allow"; wildcard=true; abbrv="--allow-wild";; "--regex" | "regex" ) kindId="regex"; typeId="deny"; abbrv="--regex";; "--wild" | "wildcard" ) kindId="regex"; typeId="deny"; wildcard=true; abbrv="--wild";; - "-d" | "--delmode" ) addmode=false;; + "-d" | "not" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; "-h" | "--help" ) helpFunc;; "-l" | "--list" ) Displaylist;; diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 89e02d2f..064193b4 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -7,11 +7,11 @@ _pihole() { case "${prev}" in "pihole") - opts="blacklist checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" + opts="allow allow-regex allow-wild deny checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard arpflush" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) ;; - "whitelist"|"blacklist"|"wildcard"|"regex") - opts_lists="\--delmode \--noreload \--quiet \--list \--nuke" + "allow"|"deny"|"wildcard"|"regex"|"allow-regx"|"allow-wild") + opts_lists="\not \--delmode \--quiet \--list \--help" COMPREPLY=( $(compgen -W "${opts_lists}" -- ${cur}) ) ;; "checkout") diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 55bbe6cb..1bf917db 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -52,47 +52,43 @@ pihole restartdns\fR [options] Available commands and options: .br -\fB-w, whitelist\fR [options] [ ] +\fBallow, allowlist\fR [options] [ ] .br - Adds or removes specified domain or domains to the Whitelist + Adds or removes specified domain or domains to the Allowlist .br -\fB-b, blacklist\fR [options] [ ] +\fBdeny, denylist\fR [options] [ ] .br - Adds or removes specified domain or domains to the blacklist + Adds or removes specified domain or domains to the denylist .br \fB--regex, regex\fR [options] [ ] .br - Add or removes specified regex filter to the regex blacklist + Add or removes specified regex filter to the regex denylist .br -\fB--white-regex\fR [options] [ ] +\fB--allow-regex\fR [options] [ ] .br - Add or removes specified regex filter to the regex whitelist + Add or removes specified regex filter to the regex allowlist .br \fB--wild, wildcard\fR [options] [ ] .br - Add or removes specified domain to the wildcard blacklist + Add or removes specified domain to the wildcard denylist .br -\fB--white-wild\fR [options] [ ] +\fB--allow-wild\fR [options] [ ] .br - Add or removes specified domain to the wildcard whitelist + Add or removes specified domain to the wildcard allowlist .br - (Whitelist/Blacklist manipulation options): + (Allow-/denylist manipulation options): .br - -d, --delmode Remove domain(s) from the list + not, -d, --delmode Remove domain(s) from the list .br - -nr, --noreload Update list without refreshing dnsmasq + -q, --quiet Make output less verbose .br - -q, --quiet Make output less verbose -.br - -l, --list Display all your listed domains -.br - --nuke Removes all entries in a list + -l, --list Display all your listed domains .br \fB-d, debug\fR [-a] @@ -279,17 +275,17 @@ Available commands and options: Some usage examples .br -Whitelist/blacklist manipulation +Allow-/denylist manipulation .br -\fBpihole -w iloveads.example.com\fR +\fBpihole allow iloveads.example.com\fR .br - Adds "iloveads.example.com" to whitelist + Allow "iloveads.example.com" .br -\fBpihole -b -d noads.example.com\fR +\fBpihole deny not noads.example.com\fR .br - Removes "noads.example.com" from blacklist + Removes "noads.example.com" from denylist .br \fBpihole --wild example.com\fR diff --git a/pihole b/pihole index 1313d487..5a3c847d 100755 --- a/pihole +++ b/pihole @@ -472,17 +472,17 @@ unsupportedFunc(){ helpFunc() { echo "Usage: pihole [options] -Example: 'pihole -a -h' +Example: 'pihole allow -h' Add '-h' after specific commands for more information on usage Domain Options: - -a, allowlist Allowlist domain(s) - -b, denylist Denylist domain(s) - --regex, regex Regex denylist domains(s) - --allow-regex Regex allowlist domains(s) - --wild, wildcard Wildcard denylist domain(s) - --allow-wild Wildcard allowlist domain(s) - Add '-h' for more info on allowlist/denylist usage + allow, allowlist Allow domain(s) + deny, denylist Deny domain(s) + --regex, regex Regex deny domains(s) + --allow-regex Regex allow domains(s) + --wild, wildcard Wildcard deny domain(s) + --allow-wild Wildcard allow domain(s) + Add '-h' for more info on allow/deny usage Debugging Options: -d, debug Start a debugging session @@ -537,8 +537,8 @@ case "${1}" in "tricorder" ) tricorderFunc;; # we need to add all arguments that require sudo power to not trigger the * argument - "-a" | "allowlist" ) need_root=0;; - "-b" | "blocklist" | "denylist" ) need_root=0;; + "allow" | "allowlist" ) need_root=0;; + "deny" | "denylist" ) need_root=0;; "--wild" | "wildcard" ) need_root=0;; "--regex" | "regex" ) need_root=0;; "--allow-regex" | "allow-regex" ) need_root=0;; @@ -578,8 +578,8 @@ fi # Handle redirecting to specific functions based on arguments case "${1}" in - "-a" | "allowlist" ) listFunc "$@";; - "-b" | "blocklist" | "denylist" ) listFunc "$@";; + "allow" | "allowlist" ) listFunc "$@";; + "deny" | "denylist" ) listFunc "$@";; "--wild" | "wildcard" ) listFunc "$@";; "--regex" | "regex" ) listFunc "$@";; "--allow-regex" | "allow-regex" ) listFunc "$@";; @@ -600,4 +600,5 @@ case "${1}" in "updatechecker" ) shift; updateCheckFunc "$@";; "arpflush" ) arpFunc "$@";; "-t" | "tail" ) tailFunc "$2";; + * ) helpFunc;; esac From c2ed30480deabaf5d53db6bcc617807ef52f6034 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 25 Jun 2024 14:10:35 +0200 Subject: [PATCH 246/413] Show when requested domains were not found on the list Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 10 +++++++--- advanced/Scripts/list.sh | 19 ++++++++++--------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index ea057adc..5843c585 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -166,10 +166,14 @@ PostFTLData() { local data response status # send the data to the API response=$(curl -skS -w "%{http_code}" -X POST "${API_URL}$1" --data-raw "$2" -H "Accept: application/json" -H "sid: ${SID}" ) - # status are the last 3 characters - status=$(printf %s "${response#"${response%???}"}") # data is everything from response without the last 3 characters - printf %s "${response%???}" + if [ "${3}" = "status" ]; then + # Keep the status code appended if requested + printf %s "${response}" + else + # Strip the status code + printf %s "${response%???}" + fi } secretRead() { diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 144317ce..d6a30325 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -66,7 +66,7 @@ CreateDomainList() { } AddDomain() { - local json num + local json num data # Authenticate with the API LoginAPI @@ -113,7 +113,7 @@ AddDomain() { } RemoveDomain() { - local json num + local json num data status # Authenticate with the API LoginAPI @@ -130,7 +130,10 @@ RemoveDomain() { json=$(jq --null-input --compact-output --arg domains "${domList[*]}" --arg typeId "${typeId}" --arg kindId "${kindId}" '[ $domains | split(" ")[] as $item | {item: $item, type: $typeId, kind: $kindId} ]') # Send the request - data=$(PostFTLData "domains:batchDelete" "${json}") + data=$(PostFTLData "domains:batchDelete" "${json}" "status") + # Separate the status from the data + status=$(printf %s "${data#"${data%???}"}") + data=$(printf %s "${data%???}") # If there is an .error object in the returned data, display it local error @@ -138,12 +141,10 @@ RemoveDomain() { if [[ $error != "null" && $error != "" ]]; then echo -e " ${CROSS} Failed to remove domain(s):" echo -e " $(jq <<< "${data}" '.error')" - elif [[ "${verbose}" == true ]]; then - echo -e " ${TICK} Removed ${#domList[@]} domain(s):" - # Loop through the domains and display them - for dom in "${domList[@]}"; do - echo -e " - ${COL_BLUE}${dom}${COL_NC}" - done + elif [[ "${verbose}" == true && "${status}" == "204" ]]; then + echo -e " ${TICK} Domain(s) removed from the ${kindId} ${typeId}list" + elif [[ "${verbose}" == true && "${status}" == "404" ]]; then + echo -e " ${TICK} Requested domain(s) not found on ${kindId} ${typeId}list" fi # Log out From 21fb5dabe19f97c53a76c9b0b81034352df5ce82 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 29 Jun 2024 09:32:13 +0200 Subject: [PATCH 247/413] Use "remove" and "delete" instead of "not" to trigger listed domain removals Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index d6a30325..8c930f04 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -47,7 +47,7 @@ Example: 'pihole ${abbrv} site.com', or 'pihole ${abbrv} site1.com site2.com' ${typeId^} one or more ${kindId} domains Options: - not -d, --delmode Remove domain(s) + remove, delete, -d Remove domain(s) -q, --quiet Make output less verbose -h, --help Show this help dialog -l, --list Display domains @@ -204,7 +204,7 @@ while (( "$#" )); do "--allow-wild" | "allow-wild" ) kindId="regex"; typeId="allow"; wildcard=true; abbrv="--allow-wild";; "--regex" | "regex" ) kindId="regex"; typeId="deny"; abbrv="--regex";; "--wild" | "wildcard" ) kindId="regex"; typeId="deny"; wildcard=true; abbrv="--wild";; - "-d" | "not" | "--delmode" ) addmode=false;; + "-d" | "remove" | "delete" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; "-h" | "--help" ) helpFunc;; "-l" | "--list" ) Displaylist;; From aa704a2e0e5644e8f188a55f31bc514293275ba5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 1 Jul 2024 21:07:09 +0200 Subject: [PATCH 248/413] Add pytest-clarity to test environment to improve error log output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 557d6497..f68261de 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -3,4 +3,4 @@ pytest == 8.2.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 tox == 4.15.1 - +pytest-clarity == 1.0.1 From 6b1d0e09d1786c27c273ad1d172d664c7ade6fe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 2 Jul 2024 08:23:48 +0200 Subject: [PATCH 249/413] Set column width to 120 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/tox.centos_9.ini | 2 ++ test/tox.debian_10.ini | 2 ++ test/tox.debian_11.ini | 2 ++ test/tox.debian_12.ini | 2 ++ test/tox.fedora_39.ini | 2 ++ test/tox.fedora_40.ini | 2 ++ test/tox.ubuntu_20.ini | 2 ++ test/tox.ubuntu_22.ini | 2 ++ test/tox.ubuntu_23.ini | 2 ++ test/tox.ubuntu_24.ini | 2 ++ 10 files changed, 20 insertions(+) diff --git a/test/tox.centos_9.ini b/test/tox.centos_9.ini index 4c51aefa..81dd0bd2 100644 --- a/test/tox.centos_9.ini +++ b/test/tox.centos_9.ini @@ -4,5 +4,7 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt +setenv = + COLUMNS=120 commands = docker buildx build --load --progress plain -f _centos_9.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py diff --git a/test/tox.debian_10.ini b/test/tox.debian_10.ini index f107300f..9995a852 100644 --- a/test/tox.debian_10.ini +++ b/test/tox.debian_10.ini @@ -4,5 +4,7 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt +setenv = + COLUMNS=120 commands = docker buildx build --load --progress plain -f _debian_10.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.debian_11.ini b/test/tox.debian_11.ini index c38a15fb..a8909d46 100644 --- a/test/tox.debian_11.ini +++ b/test/tox.debian_11.ini @@ -4,5 +4,7 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt +setenv = + COLUMNS=120 commands = docker buildx build --load --progress plain -f _debian_11.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.debian_12.ini b/test/tox.debian_12.ini index ee70e8bd..707e8710 100644 --- a/test/tox.debian_12.ini +++ b/test/tox.debian_12.ini @@ -4,5 +4,7 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt +setenv = + COLUMNS=120 commands = docker buildx build --load --progress plain -f _debian_12.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.fedora_39.ini b/test/tox.fedora_39.ini index 5c8557c9..aaa6b30e 100644 --- a/test/tox.fedora_39.ini +++ b/test/tox.fedora_39.ini @@ -4,5 +4,7 @@ envlist = py3 [testenv] allowlist_externals = docker deps = -rrequirements.txt +setenv = + COLUMNS=120 commands = docker buildx build --load --progress plain -f _fedora_39.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py diff --git a/test/tox.fedora_40.ini b/test/tox.fedora_40.ini index 149630d7..462c5ff1 100644 --- a/test/tox.fedora_40.ini +++ b/test/tox.fedora_40.ini @@ -4,5 +4,7 @@ envlist = py3 [testenv] allowlist_externals = docker deps = -rrequirements.txt +setenv = + COLUMNS=120 commands = docker buildx build --load --progress plain -f _fedora_40.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py diff --git a/test/tox.ubuntu_20.ini b/test/tox.ubuntu_20.ini index 49a6153e..bcfb1d2a 100644 --- a/test/tox.ubuntu_20.ini +++ b/test/tox.ubuntu_20.ini @@ -4,5 +4,7 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt +setenv = + COLUMNS=120 commands = docker buildx build --load --progress plain -f _ubuntu_20.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.ubuntu_22.ini b/test/tox.ubuntu_22.ini index 8014d6d6..c8e71abb 100644 --- a/test/tox.ubuntu_22.ini +++ b/test/tox.ubuntu_22.ini @@ -4,5 +4,7 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt +setenv = + COLUMNS=120 commands = docker buildx build --load --progress plain -f _ubuntu_22.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.ubuntu_23.ini b/test/tox.ubuntu_23.ini index 767ed9ef..f0a32a68 100644 --- a/test/tox.ubuntu_23.ini +++ b/test/tox.ubuntu_23.ini @@ -4,5 +4,7 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt +setenv = + COLUMNS=120 commands = docker buildx build --load --progress plain -f _ubuntu_23.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.ubuntu_24.ini b/test/tox.ubuntu_24.ini index dbd278d6..5b7e77a9 100644 --- a/test/tox.ubuntu_24.ini +++ b/test/tox.ubuntu_24.ini @@ -4,5 +4,7 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt +setenv = + COLUMNS=120 commands = docker buildx build --load --progress plain -f _ubuntu_24.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py From ddc289ca3e66e7739a63d1e91a7d42606157eb23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 2 Jul 2024 20:45:34 +0200 Subject: [PATCH 250/413] Fix Dockerfile syntax ENV legacy warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/_centos_9.Dockerfile | 10 +++++----- test/_debian_10.Dockerfile | 10 +++++----- test/_debian_11.Dockerfile | 10 +++++----- test/_debian_12.Dockerfile | 10 +++++----- test/_fedora_39.Dockerfile | 10 +++++----- test/_fedora_40.Dockerfile | 10 +++++----- test/_ubuntu_20.Dockerfile | 11 +++++------ test/_ubuntu_22.Dockerfile | 10 +++++----- test/_ubuntu_23.Dockerfile | 10 +++++----- test/_ubuntu_24.Dockerfile | 10 +++++----- 10 files changed, 50 insertions(+), 51 deletions(-) diff --git a/test/_centos_9.Dockerfile b/test/_centos_9.Dockerfile index 6ccd18b6..2e3d055a 100644 --- a/test/_centos_9.Dockerfile +++ b/test/_centos_9.Dockerfile @@ -1,18 +1,18 @@ FROM quay.io/centos/centos:stream9 RUN yum install -y --allowerasing curl git initscripts -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR RUN true && \ chmod +x $SCRIPTDIR/* -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net +ENV SKIP_INSTALL=true +ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_debian_10.Dockerfile b/test/_debian_10.Dockerfile index 3b177cc8..dc813ac2 100644 --- a/test/_debian_10.Dockerfile +++ b/test/_debian_10.Dockerfile @@ -1,17 +1,17 @@ FROM buildpack-deps:buster-scm -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR RUN true && \ chmod +x $SCRIPTDIR/* -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net +ENV SKIP_INSTALL=true +ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_debian_11.Dockerfile b/test/_debian_11.Dockerfile index 58c67e0f..cb7d27cc 100644 --- a/test/_debian_11.Dockerfile +++ b/test/_debian_11.Dockerfile @@ -1,17 +1,17 @@ FROM buildpack-deps:bullseye-scm -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR RUN true && \ chmod +x $SCRIPTDIR/* -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net +ENV SKIP_INSTALL=true +ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_debian_12.Dockerfile b/test/_debian_12.Dockerfile index a762fee0..50d709b1 100644 --- a/test/_debian_12.Dockerfile +++ b/test/_debian_12.Dockerfile @@ -1,17 +1,17 @@ FROM buildpack-deps:bookworm-scm -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR RUN true && \ chmod +x $SCRIPTDIR/* -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net +ENV SKIP_INSTALL=true +ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_fedora_39.Dockerfile b/test/_fedora_39.Dockerfile index 1727a3aa..1d3dbc63 100644 --- a/test/_fedora_39.Dockerfile +++ b/test/_fedora_39.Dockerfile @@ -1,18 +1,18 @@ FROM fedora:39 RUN dnf install -y git initscripts -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR RUN true && \ chmod +x $SCRIPTDIR/* -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net +ENV SKIP_INSTALL=true +ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_fedora_40.Dockerfile b/test/_fedora_40.Dockerfile index 6d00072f..e4879c92 100644 --- a/test/_fedora_40.Dockerfile +++ b/test/_fedora_40.Dockerfile @@ -1,18 +1,18 @@ FROM fedora:40 RUN dnf install -y git initscripts -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR RUN true && \ chmod +x $SCRIPTDIR/* -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net +ENV SKIP_INSTALL=true +ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_ubuntu_20.Dockerfile b/test/_ubuntu_20.Dockerfile index c63f883a..64d4f415 100644 --- a/test/_ubuntu_20.Dockerfile +++ b/test/_ubuntu_20.Dockerfile @@ -1,18 +1,17 @@ FROM buildpack-deps:focal-scm -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR -ENV DEBIAN_FRONTEND=noninteractive +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR RUN true && \ chmod +x $SCRIPTDIR/* -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net +ENV SKIP_INSTALL=true +ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_ubuntu_22.Dockerfile b/test/_ubuntu_22.Dockerfile index d44518b4..34faa361 100644 --- a/test/_ubuntu_22.Dockerfile +++ b/test/_ubuntu_22.Dockerfile @@ -1,18 +1,18 @@ FROM buildpack-deps:jammy-scm -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR ENV DEBIAN_FRONTEND=noninteractive RUN true && \ chmod +x $SCRIPTDIR/* -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net +ENV SKIP_INSTALL=true +ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_ubuntu_23.Dockerfile b/test/_ubuntu_23.Dockerfile index f9b3910b..ea0ad245 100644 --- a/test/_ubuntu_23.Dockerfile +++ b/test/_ubuntu_23.Dockerfile @@ -1,18 +1,18 @@ FROM buildpack-deps:lunar-scm -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR ENV DEBIAN_FRONTEND=noninteractive RUN true && \ chmod +x $SCRIPTDIR/* -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net +ENV SKIP_INSTALL=true +ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_ubuntu_24.Dockerfile b/test/_ubuntu_24.Dockerfile index 2b048361..4d581cd3 100644 --- a/test/_ubuntu_24.Dockerfile +++ b/test/_ubuntu_24.Dockerfile @@ -1,18 +1,18 @@ FROM buildpack-deps:24.04-scm -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR ENV DEBIAN_FRONTEND=noninteractive RUN true && \ chmod +x $SCRIPTDIR/* -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net +ENV SKIP_INSTALL=true +ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ From fafd47ba75a95b6c663af811f0407b818ee7dcb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 2 Jul 2024 20:52:44 +0200 Subject: [PATCH 251/413] Please editorconfig-checker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/stale.yml | 1 - .../Scripts/database_migration/gravity-db.sh | 284 +++++++++--------- .../database_migration/gravity/1_to_2.sql | 6 +- .../database_migration/gravity/2_to_3.sql | 24 +- .../database_migration/gravity/3_to_4.sql | 20 +- .../database_migration/gravity/4_to_5.sql | 16 +- .../database_migration/gravity/5_to_6.sql | 1 - .../database_migration/gravity/7_to_8.sql | 12 +- advanced/Templates/gravity.db.sql | 100 +++--- advanced/Templates/logrotate | 48 +-- advanced/bash-completion/pihole | 94 +++--- manpages/pihole.8 | 2 +- 12 files changed, 303 insertions(+), 305 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 2218ea02..2a7831af 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -45,4 +45,3 @@ jobs: run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index e99f1df2..8f84e2b0 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -13,150 +13,150 @@ readonly scriptPath="/etc/.pihole/advanced/Scripts/database_migration/gravity" upgrade_gravityDB(){ - local database piholeDir auditFile version - database="${1}" - piholeDir="${2}" - auditFile="${piholeDir}/auditlog.list" + local database piholeDir auditFile version + database="${1}" + piholeDir="${2}" + auditFile="${piholeDir}/auditlog.list" - # Exit early if the database does not exist (e.g. in CI tests) - if [[ ! -f "${database}" ]]; then - return - fi + # Exit early if the database does not exist (e.g. in CI tests) + if [[ ! -f "${database}" ]]; then + return + fi - # Get database version - version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" + # Get database version + version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" - if [[ "$version" == "1" ]]; then - # This migration script upgrades the gravity.db file by - # adding the domain_audit table - echo -e " ${INFO} Upgrading gravity database from version 1 to 2" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/1_to_2.sql" - version=2 + if [[ "$version" == "1" ]]; then + # This migration script upgrades the gravity.db file by + # adding the domain_audit table + echo -e " ${INFO} Upgrading gravity database from version 1 to 2" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/1_to_2.sql" + version=2 - # Store audit domains in database table - if [ -e "${auditFile}" ]; then - echo -e " ${INFO} Migrating content of ${auditFile} into new database" - # database_table_from_file is defined in gravity.sh - database_table_from_file "domain_audit" "${auditFile}" - fi - fi - if [[ "$version" == "2" ]]; then - # This migration script upgrades the gravity.db file by - # renaming the regex table to regex_blacklist, and - # creating a new regex_whitelist table + corresponding linking table and views - echo -e " ${INFO} Upgrading gravity database from version 2 to 3" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/2_to_3.sql" - version=3 - fi - if [[ "$version" == "3" ]]; then - # This migration script unifies the formally separated domain - # lists into a single table with a UNIQUE domain constraint - echo -e " ${INFO} Upgrading gravity database from version 3 to 4" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/3_to_4.sql" - version=4 - fi - if [[ "$version" == "4" ]]; then - # This migration script upgrades the gravity and list views - # implementing necessary changes for per-client blocking - echo -e " ${INFO} Upgrading gravity database from version 4 to 5" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/4_to_5.sql" - version=5 - fi - if [[ "$version" == "5" ]]; then - # This migration script upgrades the adlist view - # to return an ID used in gravity.sh - echo -e " ${INFO} Upgrading gravity database from version 5 to 6" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/5_to_6.sql" - version=6 - fi - if [[ "$version" == "6" ]]; then - # This migration script adds a special group with ID 0 - # which is automatically associated to all clients not - # having their own group assignments - echo -e " ${INFO} Upgrading gravity database from version 6 to 7" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/6_to_7.sql" - version=7 - fi - if [[ "$version" == "7" ]]; then - # This migration script recreated the group table - # to ensure uniqueness on the group name - # We also add date_added and date_modified columns - echo -e " ${INFO} Upgrading gravity database from version 7 to 8" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/7_to_8.sql" - version=8 - fi - if [[ "$version" == "8" ]]; then - # This migration fixes some issues that were introduced - # in the previous migration script. - echo -e " ${INFO} Upgrading gravity database from version 8 to 9" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/8_to_9.sql" - version=9 - fi - if [[ "$version" == "9" ]]; then - # This migration drops unused tables and creates triggers to remove - # obsolete groups assignments when the linked items are deleted - echo -e " ${INFO} Upgrading gravity database from version 9 to 10" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/9_to_10.sql" - version=10 - fi - if [[ "$version" == "10" ]]; then - # This adds timestamp and an optional comment field to the client table - # These fields are only temporary and will be replaces by the columns - # defined in gravity.db.sql during gravity swapping. We add them here - # to keep the copying process generic (needs the same columns in both the - # source and the destination databases). - echo -e " ${INFO} Upgrading gravity database from version 10 to 11" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/10_to_11.sql" - version=11 - fi - if [[ "$version" == "11" ]]; then - # Rename group 0 from "Unassociated" to "Default" - echo -e " ${INFO} Upgrading gravity database from version 11 to 12" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/11_to_12.sql" - version=12 - fi - if [[ "$version" == "12" ]]; then - # Add column date_updated to adlist table - echo -e " ${INFO} Upgrading gravity database from version 12 to 13" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/12_to_13.sql" - version=13 - fi - if [[ "$version" == "13" ]]; then - # Add columns number and status to adlist table - echo -e " ${INFO} Upgrading gravity database from version 13 to 14" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/13_to_14.sql" - version=14 - fi - if [[ "$version" == "14" ]]; then - # Changes the vw_adlist created in 5_to_6 - echo -e " ${INFO} Upgrading gravity database from version 14 to 15" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/14_to_15.sql" - version=15 - fi - if [[ "$version" == "15" ]]; then - # Add column abp_entries to adlist table - echo -e " ${INFO} Upgrading gravity database from version 15 to 16" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/15_to_16.sql" - version=16 - fi - if [[ "$version" == "16" ]]; then - # Add antigravity table - # Add column type to adlist table (to support adlist types) - echo -e " ${INFO} Upgrading gravity database from version 16 to 17" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/16_to_17.sql" - version=17 - fi - if [[ "$version" == "17" ]]; then - # Add adlist.id to vw_gravity and vw_antigravity - echo -e " ${INFO} Upgrading gravity database from version 17 to 18" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/17_to_18.sql" - version=18 - fi - if [[ "$version" == "18" ]]; then - # Modify DELETE triggers to delete BEFORE instead of AFTER to prevent - # foreign key constraint violations - echo -e " ${INFO} Upgrading gravity database from version 18 to 19" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/18_to_19.sql" - version=19 - fi + # Store audit domains in database table + if [ -e "${auditFile}" ]; then + echo -e " ${INFO} Migrating content of ${auditFile} into new database" + # database_table_from_file is defined in gravity.sh + database_table_from_file "domain_audit" "${auditFile}" + fi + fi + if [[ "$version" == "2" ]]; then + # This migration script upgrades the gravity.db file by + # renaming the regex table to regex_blacklist, and + # creating a new regex_whitelist table + corresponding linking table and views + echo -e " ${INFO} Upgrading gravity database from version 2 to 3" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/2_to_3.sql" + version=3 + fi + if [[ "$version" == "3" ]]; then + # This migration script unifies the formally separated domain + # lists into a single table with a UNIQUE domain constraint + echo -e " ${INFO} Upgrading gravity database from version 3 to 4" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/3_to_4.sql" + version=4 + fi + if [[ "$version" == "4" ]]; then + # This migration script upgrades the gravity and list views + # implementing necessary changes for per-client blocking + echo -e " ${INFO} Upgrading gravity database from version 4 to 5" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/4_to_5.sql" + version=5 + fi + if [[ "$version" == "5" ]]; then + # This migration script upgrades the adlist view + # to return an ID used in gravity.sh + echo -e " ${INFO} Upgrading gravity database from version 5 to 6" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/5_to_6.sql" + version=6 + fi + if [[ "$version" == "6" ]]; then + # This migration script adds a special group with ID 0 + # which is automatically associated to all clients not + # having their own group assignments + echo -e " ${INFO} Upgrading gravity database from version 6 to 7" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/6_to_7.sql" + version=7 + fi + if [[ "$version" == "7" ]]; then + # This migration script recreated the group table + # to ensure uniqueness on the group name + # We also add date_added and date_modified columns + echo -e " ${INFO} Upgrading gravity database from version 7 to 8" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/7_to_8.sql" + version=8 + fi + if [[ "$version" == "8" ]]; then + # This migration fixes some issues that were introduced + # in the previous migration script. + echo -e " ${INFO} Upgrading gravity database from version 8 to 9" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/8_to_9.sql" + version=9 + fi + if [[ "$version" == "9" ]]; then + # This migration drops unused tables and creates triggers to remove + # obsolete groups assignments when the linked items are deleted + echo -e " ${INFO} Upgrading gravity database from version 9 to 10" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/9_to_10.sql" + version=10 + fi + if [[ "$version" == "10" ]]; then + # This adds timestamp and an optional comment field to the client table + # These fields are only temporary and will be replaces by the columns + # defined in gravity.db.sql during gravity swapping. We add them here + # to keep the copying process generic (needs the same columns in both the + # source and the destination databases). + echo -e " ${INFO} Upgrading gravity database from version 10 to 11" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/10_to_11.sql" + version=11 + fi + if [[ "$version" == "11" ]]; then + # Rename group 0 from "Unassociated" to "Default" + echo -e " ${INFO} Upgrading gravity database from version 11 to 12" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/11_to_12.sql" + version=12 + fi + if [[ "$version" == "12" ]]; then + # Add column date_updated to adlist table + echo -e " ${INFO} Upgrading gravity database from version 12 to 13" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/12_to_13.sql" + version=13 + fi + if [[ "$version" == "13" ]]; then + # Add columns number and status to adlist table + echo -e " ${INFO} Upgrading gravity database from version 13 to 14" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/13_to_14.sql" + version=14 + fi + if [[ "$version" == "14" ]]; then + # Changes the vw_adlist created in 5_to_6 + echo -e " ${INFO} Upgrading gravity database from version 14 to 15" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/14_to_15.sql" + version=15 + fi + if [[ "$version" == "15" ]]; then + # Add column abp_entries to adlist table + echo -e " ${INFO} Upgrading gravity database from version 15 to 16" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/15_to_16.sql" + version=16 + fi + if [[ "$version" == "16" ]]; then + # Add antigravity table + # Add column type to adlist table (to support adlist types) + echo -e " ${INFO} Upgrading gravity database from version 16 to 17" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/16_to_17.sql" + version=17 + fi + if [[ "$version" == "17" ]]; then + # Add adlist.id to vw_gravity and vw_antigravity + echo -e " ${INFO} Upgrading gravity database from version 17 to 18" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/17_to_18.sql" + version=18 + fi + if [[ "$version" == "18" ]]; then + # Modify DELETE triggers to delete BEFORE instead of AFTER to prevent + # foreign key constraint violations + echo -e " ${INFO} Upgrading gravity database from version 18 to 19" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/18_to_19.sql" + version=19 + fi } diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql index 6d57a6fe..ef445cc6 100644 --- a/advanced/Scripts/database_migration/gravity/1_to_2.sql +++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql @@ -4,9 +4,9 @@ BEGIN TRANSACTION; CREATE TABLE domain_audit ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) ); UPDATE info SET value = 2 WHERE property = 'version'; diff --git a/advanced/Scripts/database_migration/gravity/2_to_3.sql b/advanced/Scripts/database_migration/gravity/2_to_3.sql index fd7c24d2..9ade340a 100644 --- a/advanced/Scripts/database_migration/gravity/2_to_3.sql +++ b/advanced/Scripts/database_migration/gravity/2_to_3.sql @@ -8,9 +8,9 @@ ALTER TABLE regex RENAME TO regex_blacklist; CREATE TABLE regex_blacklist_by_group ( - regex_blacklist_id INTEGER NOT NULL REFERENCES regex_blacklist (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (regex_blacklist_id, group_id) + regex_blacklist_id INTEGER NOT NULL REFERENCES regex_blacklist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (regex_blacklist_id, group_id) ); INSERT INTO regex_blacklist_by_group SELECT * FROM regex_by_group; @@ -32,19 +32,19 @@ CREATE TRIGGER tr_regex_blacklist_update AFTER UPDATE ON regex_blacklist CREATE TABLE regex_whitelist ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT ); CREATE TABLE regex_whitelist_by_group ( - regex_whitelist_id INTEGER NOT NULL REFERENCES regex_whitelist (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (regex_whitelist_id, group_id) + regex_whitelist_id INTEGER NOT NULL REFERENCES regex_whitelist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (regex_whitelist_id, group_id) ); CREATE VIEW vw_regex_whitelist AS SELECT DISTINCT domain diff --git a/advanced/Scripts/database_migration/gravity/3_to_4.sql b/advanced/Scripts/database_migration/gravity/3_to_4.sql index 05231f72..2b3d0a79 100644 --- a/advanced/Scripts/database_migration/gravity/3_to_4.sql +++ b/advanced/Scripts/database_migration/gravity/3_to_4.sql @@ -6,13 +6,13 @@ BEGIN TRANSACTION; CREATE TABLE domainlist ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - type INTEGER NOT NULL DEFAULT 0, - domain TEXT UNIQUE NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT + id INTEGER PRIMARY KEY AUTOINCREMENT, + type INTEGER NOT NULL DEFAULT 0, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT ); ALTER TABLE whitelist ADD COLUMN type INTEGER; @@ -41,9 +41,9 @@ DROP TABLE regex_whitelist_by_group; DROP TABLE regex_blacklist_by_group; CREATE TABLE domainlist_by_group ( - domainlist_id INTEGER NOT NULL REFERENCES domainlist (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (domainlist_id, group_id) + domainlist_id INTEGER NOT NULL REFERENCES domainlist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (domainlist_id, group_id) ); DROP TRIGGER tr_whitelist_update; diff --git a/advanced/Scripts/database_migration/gravity/4_to_5.sql b/advanced/Scripts/database_migration/gravity/4_to_5.sql index 4ae9f980..1436c69d 100644 --- a/advanced/Scripts/database_migration/gravity/4_to_5.sql +++ b/advanced/Scripts/database_migration/gravity/4_to_5.sql @@ -7,9 +7,9 @@ BEGIN TRANSACTION; DROP TABLE gravity; CREATE TABLE gravity ( - domain TEXT NOT NULL, - adlist_id INTEGER NOT NULL REFERENCES adlist (id), - PRIMARY KEY(domain, adlist_id) + domain TEXT NOT NULL, + adlist_id INTEGER NOT NULL REFERENCES adlist (id), + PRIMARY KEY(domain, adlist_id) ); DROP VIEW vw_gravity; @@ -22,15 +22,15 @@ CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id CREATE TABLE client ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - ip TEXT NOL NULL UNIQUE + id INTEGER PRIMARY KEY AUTOINCREMENT, + ip TEXT NOL NULL UNIQUE ); CREATE TABLE client_by_group ( - client_id INTEGER NOT NULL REFERENCES client (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (client_id, group_id) + client_id INTEGER NOT NULL REFERENCES client (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (client_id, group_id) ); UPDATE info SET value = 5 WHERE property = 'version'; diff --git a/advanced/Scripts/database_migration/gravity/5_to_6.sql b/advanced/Scripts/database_migration/gravity/5_to_6.sql index d2bb3145..a058156a 100644 --- a/advanced/Scripts/database_migration/gravity/5_to_6.sql +++ b/advanced/Scripts/database_migration/gravity/5_to_6.sql @@ -15,4 +15,3 @@ CREATE VIEW vw_adlist AS SELECT DISTINCT address, adlist.id AS id UPDATE info SET value = 6 WHERE property = 'version'; COMMIT; - diff --git a/advanced/Scripts/database_migration/gravity/7_to_8.sql b/advanced/Scripts/database_migration/gravity/7_to_8.sql index ccf0c148..c6a5b35b 100644 --- a/advanced/Scripts/database_migration/gravity/7_to_8.sql +++ b/advanced/Scripts/database_migration/gravity/7_to_8.sql @@ -8,12 +8,12 @@ ALTER TABLE "group" RENAME TO "group__"; CREATE TABLE "group" ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - enabled BOOLEAN NOT NULL DEFAULT 1, - name TEXT UNIQUE NOT NULL, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - description TEXT + id INTEGER PRIMARY KEY AUTOINCREMENT, + enabled BOOLEAN NOT NULL DEFAULT 1, + name TEXT UNIQUE NOT NULL, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + description TEXT ); CREATE TRIGGER tr_group_update AFTER UPDATE ON "group" diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 42060443..9782a044 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -3,99 +3,99 @@ BEGIN TRANSACTION; CREATE TABLE "group" ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - enabled BOOLEAN NOT NULL DEFAULT 1, - name TEXT UNIQUE NOT NULL, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - description TEXT + id INTEGER PRIMARY KEY AUTOINCREMENT, + enabled BOOLEAN NOT NULL DEFAULT 1, + name TEXT UNIQUE NOT NULL, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + description TEXT ); INSERT INTO "group" (id,enabled,name,description) VALUES (0,1,'Default','The default group'); CREATE TABLE domainlist ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - type INTEGER NOT NULL DEFAULT 0, - domain TEXT NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT, - UNIQUE(domain, type) + id INTEGER PRIMARY KEY AUTOINCREMENT, + type INTEGER NOT NULL DEFAULT 0, + domain TEXT NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT, + UNIQUE(domain, type) ); CREATE TABLE adlist ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - address TEXT NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT, - date_updated INTEGER, - number INTEGER NOT NULL DEFAULT 0, - invalid_domains INTEGER NOT NULL DEFAULT 0, - status INTEGER NOT NULL DEFAULT 0, - abp_entries INTEGER NOT NULL DEFAULT 0, - type INTEGER NOT NULL DEFAULT 0, - UNIQUE(address, type) + id INTEGER PRIMARY KEY AUTOINCREMENT, + address TEXT NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT, + date_updated INTEGER, + number INTEGER NOT NULL DEFAULT 0, + invalid_domains INTEGER NOT NULL DEFAULT 0, + status INTEGER NOT NULL DEFAULT 0, + abp_entries INTEGER NOT NULL DEFAULT 0, + type INTEGER NOT NULL DEFAULT 0, + UNIQUE(address, type) ); CREATE TABLE adlist_by_group ( - adlist_id INTEGER NOT NULL REFERENCES adlist (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (adlist_id, group_id) + adlist_id INTEGER NOT NULL REFERENCES adlist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (adlist_id, group_id) ); CREATE TABLE gravity ( - domain TEXT NOT NULL, - adlist_id INTEGER NOT NULL REFERENCES adlist (id) + domain TEXT NOT NULL, + adlist_id INTEGER NOT NULL REFERENCES adlist (id) ); CREATE TABLE antigravity ( - domain TEXT NOT NULL, - adlist_id INTEGER NOT NULL REFERENCES adlist (id) + domain TEXT NOT NULL, + adlist_id INTEGER NOT NULL REFERENCES adlist (id) ); CREATE TABLE info ( - property TEXT PRIMARY KEY, - value TEXT NOT NULL + property TEXT PRIMARY KEY, + value TEXT NOT NULL ); INSERT INTO "info" VALUES('version','18'); CREATE TABLE domain_audit ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) ); CREATE TABLE domainlist_by_group ( - domainlist_id INTEGER NOT NULL REFERENCES domainlist (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (domainlist_id, group_id) + domainlist_id INTEGER NOT NULL REFERENCES domainlist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (domainlist_id, group_id) ); CREATE TABLE client ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - ip TEXT NOT NULL UNIQUE, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT + id INTEGER PRIMARY KEY AUTOINCREMENT, + ip TEXT NOT NULL UNIQUE, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT ); CREATE TABLE client_by_group ( - client_id INTEGER NOT NULL REFERENCES client (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (client_id, group_id) + client_id INTEGER NOT NULL REFERENCES client (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (client_id, group_id) ); CREATE TRIGGER tr_adlist_update AFTER UPDATE OF address,enabled,comment ON adlist diff --git a/advanced/Templates/logrotate b/advanced/Templates/logrotate index 5f609e0f..9e52776b 100644 --- a/advanced/Templates/logrotate +++ b/advanced/Templates/logrotate @@ -1,32 +1,32 @@ /var/log/pihole/pihole.log { - # su # - daily - copytruncate - rotate 5 - compress - delaycompress - notifempty - nomail + # su # + daily + copytruncate + rotate 5 + compress + delaycompress + notifempty + nomail } /var/log/pihole/FTL.log { - # su # - weekly - copytruncate - rotate 3 - compress - delaycompress - notifempty - nomail + # su # + weekly + copytruncate + rotate 3 + compress + delaycompress + notifempty + nomail } /var/log/pihole/webserver.log { - # su # - weekly - copytruncate - rotate 3 - compress - delaycompress - notifempty - nomail + # su # + weekly + copytruncate + rotate 3 + compress + delaycompress + notifempty + nomail } diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 89e02d2f..c2b58c6e 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -1,51 +1,51 @@ _pihole() { - local cur prev opts opts_checkout opts_debug opts_logging opts_query opts_update opts_version - COMPREPLY=() - cur="${COMP_WORDS[COMP_CWORD]}" - prev="${COMP_WORDS[COMP_CWORD-1]}" - prev2="${COMP_WORDS[COMP_CWORD-2]}" + local cur prev opts opts_checkout opts_debug opts_logging opts_query opts_update opts_version + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + prev2="${COMP_WORDS[COMP_CWORD-2]}" - case "${prev}" in - "pihole") - opts="blacklist checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" - COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) - ;; - "whitelist"|"blacklist"|"wildcard"|"regex") - opts_lists="\--delmode \--noreload \--quiet \--list \--nuke" - COMPREPLY=( $(compgen -W "${opts_lists}" -- ${cur}) ) - ;; - "checkout") - opts_checkout="core ftl web master dev" - COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) - ;; - "debug") - opts_debug="-a" - COMPREPLY=( $(compgen -W "${opts_debug}" -- ${cur}) ) - ;; - "logging") - opts_logging="on off 'off noflush'" - COMPREPLY=( $(compgen -W "${opts_logging}" -- ${cur}) ) - ;; - "query") - opts_query="--partial --all" - COMPREPLY=( $(compgen -W "${opts_query}" -- ${cur}) ) - ;; - "updatePihole"|"-up") - opts_update="--check-only" - COMPREPLY=( $(compgen -W "${opts_update}" -- ${cur}) ) - ;; - "core"|"admin"|"ftl") - if [[ "$prev2" == "checkout" ]]; then - opts_checkout="master dev" - COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) - else - return 1 - fi - ;; - *) - return 1 - ;; - esac - return 0 + case "${prev}" in + "pihole") + opts="blacklist checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + ;; + "whitelist"|"blacklist"|"wildcard"|"regex") + opts_lists="\--delmode \--noreload \--quiet \--list \--nuke" + COMPREPLY=( $(compgen -W "${opts_lists}" -- ${cur}) ) + ;; + "checkout") + opts_checkout="core ftl web master dev" + COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) + ;; + "debug") + opts_debug="-a" + COMPREPLY=( $(compgen -W "${opts_debug}" -- ${cur}) ) + ;; + "logging") + opts_logging="on off 'off noflush'" + COMPREPLY=( $(compgen -W "${opts_logging}" -- ${cur}) ) + ;; + "query") + opts_query="--partial --all" + COMPREPLY=( $(compgen -W "${opts_query}" -- ${cur}) ) + ;; + "updatePihole"|"-up") + opts_update="--check-only" + COMPREPLY=( $(compgen -W "${opts_update}" -- ${cur}) ) + ;; + "core"|"admin"|"ftl") + if [[ "$prev2" == "checkout" ]]; then + opts_checkout="master dev" + COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) + else + return 1 + fi + ;; + *) + return 1 + ;; + esac + return 0 } complete -F _pihole pihole diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 55bbe6cb..bdc4b744 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -170,7 +170,7 @@ Available commands and options: Specify whether the Pi-hole log should be used .br - (Logging options): + (Logging options): .br on Enable the Pi-hole log at /var/log/pihole/pihole.log .br From 2b33a0494b61eb4cbaaa26e1376914a526b7cd91 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Jul 2024 10:12:06 +0000 Subject: [PATCH 252/413] Bump tox from 4.15.1 to 4.16.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.15.1 to 4.16.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.15.1...4.16.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 557d6497..6d230749 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.2.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.15.1 +tox == 4.16.0 From e8802173f5eacad1671997e833bce82f3bff69ee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Jul 2024 20:09:24 +0000 Subject: [PATCH 253/413] Bump tox from 4.15.1 to 4.16.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.15.1 to 4.16.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.15.1...4.16.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index f68261de..73b9ad6e 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.2.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.15.1 +tox == 4.16.0 pytest-clarity == 1.0.1 From 6ffb20f09e6a24152e3fe10c2a03b07c8d401d09 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 30 Apr 2024 15:47:57 +0200 Subject: [PATCH 254/413] Add protocol validation when downloading blocklist from URL Signed-off-by: DL6ER --- gravity.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gravity.sh b/gravity.sh index 522d92f5..d1664394 100755 --- a/gravity.sh +++ b/gravity.sh @@ -622,6 +622,12 @@ gravity_DownloadBlocklistFromUrl() { fi fi + # Check for allowed protocols + if [[ $url != "http"* && $url != "https"* && $url != "file"* && $url != "ftp"* && $url != "ftps"* && $url != "sftp"* ]]; then + echo -e "${OVER} ${CROSS} ${str} Invalid protocol specified, ignoring list" + download=false + fi + if [[ "${download}" == true ]]; then # shellcheck disable=SC2086 httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2>/dev/null) From 5cb740ef01ef29433c20c973a2d2d84942ecfd86 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Wed, 10 Jul 2024 14:29:40 +0900 Subject: [PATCH 255/413] chore: update basic-install.sh minor fix Signed-off-by: Ikko Eltociear Ashimine --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a134c0d2..ef891ec4 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -529,7 +529,7 @@ find_IPv4_information() { local route local IPv4bare - # Find IP used to route to outside world by checking the the route to Google's public DNS server + # Find IP used to route to outside world by checking the route to Google's public DNS server route=$(ip route get 8.8.8.8) # Get just the interface IPv4 address From bdbec058e994020bc732fe284343674e94da971f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 10:14:54 +0000 Subject: [PATCH 256/413] Bump actions/setup-python from 5.1.0 to 5.1.1 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.1.0 to 5.1.1. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.1.0...v5.1.1) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1da17f0a..9ce20a01 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,7 +74,7 @@ jobs: uses: actions/checkout@v4.1.7 - name: Set up Python 3.10 - uses: actions/setup-python@v5.1.0 + uses: actions/setup-python@v5.1.1 with: python-version: "3.10" From e90df12053fff0a6766c7f92ffe6cfcdf14b9b21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 10:15:25 +0000 Subject: [PATCH 257/413] Bump actions/setup-python from 5.1.0 to 5.1.1 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.1.0 to 5.1.1. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.1.0...v5.1.1) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ec2b5728..6ec42e9c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -67,7 +67,7 @@ jobs: uses: actions/checkout@v4.1.7 - name: Set up Python 3.10 - uses: actions/setup-python@v5.1.0 + uses: actions/setup-python@v5.1.1 with: python-version: "3.10" From 7ca4b59b34f4f23199c7a9c2f751df39f9bae858 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 16 Jul 2024 10:27:07 +0200 Subject: [PATCH 258/413] Do not test API availability outside of LoginAPI Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index c76e890e..3340bdd2 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -131,9 +131,6 @@ Main() { # https://github.com/pi-hole/FTL/pull/1715 # no need to do it here - # Test if the authentication endpoint is available - TestAPIAvailability - # Authenticate with FTL LoginAPI From 897e23089c10a6ac8c7c51dbfe5c897c7051c22c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 16 Jul 2024 10:55:28 +0200 Subject: [PATCH 259/413] Implement pihole enable/disable using the API Signed-off-by: DL6ER --- advanced/Scripts/pihole-reenable.sh | 23 ------ pihole | 123 +++++++++++++--------------- 2 files changed, 57 insertions(+), 89 deletions(-) delete mode 100755 advanced/Scripts/pihole-reenable.sh diff --git a/advanced/Scripts/pihole-reenable.sh b/advanced/Scripts/pihole-reenable.sh deleted file mode 100755 index 93ec3b95..00000000 --- a/advanced/Scripts/pihole-reenable.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Pi-hole: A black hole for Internet advertisements -# (c) 2020 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. -# -# -# The pihole disable command has the option to set a specified time before -# blocking is automatically re-enabled. -# -# Present script is responsible for the sleep & re-enable part of the job and -# is automatically terminated if it is still running when pihole is enabled by -# other means. -# -# This ensures that pihole ends up in the correct state after a sequence of -# commands suchs as: `pihole disable 30s; pihole enable; pihole disable` - -readonly PI_HOLE_BIN_DIR="/usr/local/bin" - -sleep "${1}" -"${PI_HOLE_BIN_DIR}"/pihole enable diff --git a/pihole b/pihole index 5a3c847d..08ff5b76 100755 --- a/pihole +++ b/pihole @@ -19,9 +19,13 @@ PI_HOLE_BIN_DIR="/usr/local/bin" readonly colfile="${PI_HOLE_SCRIPT_DIR}/COL_TABLE" source "${colfile}" -utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" +readonly utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" source "${utilsfile}" +# Source api functions +readonly apifile="${PI_HOLE_SCRIPT_DIR}/api.sh" +source "${apifile}" + versionsfile="/etc/pihole/versions" if [ -f "${versionsfile}" ]; then # Only source versionsfile if the file exits @@ -205,73 +209,60 @@ restartDNS() { piholeEnable() { if [[ "${2}" == "-h" ]] || [[ "${2}" == "--help" ]]; then - echo "Usage: pihole disable [time] -Example: 'pihole disable', or 'pihole disable 5m' -Disable Pi-hole subsystems + echo "Usage: pihole enable/disable [time] +Example: 'pihole enable', or 'pihole disable 5m' +En- or disable Pi-hole subsystems Time: - #s Disable Pi-hole functionality for # second(s) - #m Disable Pi-hole functionality for # minute(s)" + #s En-/disable Pi-hole functionality for # second(s) + #m En-/disable Pi-hole functionality for # minute(s)" exit 0 - elif [[ "${1}" == "0" ]]; then - # Disable Pi-hole - if ! getFTLConfigValue dns.blocking.active; then - echo -e " ${INFO} Blocking already disabled, nothing to do" - exit 0 - fi - if [[ $# -gt 1 ]]; then - local error=false - if [[ "${2}" == *"s" ]]; then - tt=${2%"s"} - if [[ "${tt}" =~ ^-?[0-9]+$ ]];then - local str="Disabling blocking for ${tt} seconds" - echo -e " ${INFO} ${str}..." - local str="Blocking will be re-enabled in ${tt} seconds" - nohup "${PI_HOLE_SCRIPT_DIR}"/pihole-reenable.sh ${tt} /dev/null & - else - local error=true - fi - elif [[ "${2}" == *"m" ]]; then - tt=${2%"m"} - if [[ "${tt}" =~ ^-?[0-9]+$ ]];then - local str="Disabling blocking for ${tt} minutes" - echo -e " ${INFO} ${str}..." - local str="Blocking will be re-enabled in ${tt} minutes" - tt=$((${tt}*60)) - nohup "${PI_HOLE_SCRIPT_DIR}"/pihole-reenable.sh ${tt} /dev/null & - else - local error=true - fi - elif [[ -n "${2}" ]]; then - local error=true - else - echo -e " ${INFO} Disabling blocking" - fi - - if [[ ${error} == true ]];then - echo -e " ${COL_LIGHT_RED}Unknown format for delayed reactivation of the blocking!${COL_NC}" - echo -e " Try 'pihole disable --help' for more information." - exit 1 - fi - - local str="Pi-hole Disabled" - setFTLConfigValue dns.blocking.active false - fi - else - # Enable Pi-hole - killall -q pihole-reenable - if getFTLConfigValue dns.blocking.active; then - echo -e " ${INFO} Blocking already enabled, nothing to do" - exit 0 - fi - echo -e " ${INFO} Enabling blocking" - local str="Pi-hole Enabled" - - setFTLConfigValue dns.blocking.active true fi - restartDNS reload-lists + # Get timer + local tt="null" + if [[ $# -gt 1 ]]; then + local error=false + if [[ "${2}" == *"s" ]]; then + tt=${2%"s"} + if [[ ! "${tt}" =~ ^-?[0-9]+$ ]];then + local error=true + fi + elif [[ "${2}" == *"m" ]]; then + tt=${2%"m"} + if [[ "${tt}" =~ ^-?[0-9]+$ ]];then + tt=$((${tt}*60)) + else + local error=true + fi + elif [[ -n "${2}" ]]; then + local error=true + fi + + if [[ ${error} == true ]];then + echo -e " ${COL_LIGHT_RED}Unknown format for blocking timer!${COL_NC}" + echo -e " Try 'pihole disable --help' for more information." + exit 1 + fi + fi + + # Authenticate with the API + LoginAPI + + # Send the request + data=$(PostFTLData "dns/blocking" "{ \"blocking\": ${1}, \"timer\": ${tt} }") + + # Check the response + local extra=" forever" + local timer="$(echo "${data}"| jq --raw-output '.timer' )" + if [[ "${timer}" != "null" ]]; then + extra=" for ${timer}s" + fi + local str="Pi-hole $(echo "${data}" | jq --raw-output '.blocking')${extra}" + + # Logout from the API + LogoutAPI echo -e "${OVER} ${TICK} ${str}" } @@ -548,8 +539,8 @@ case "${1}" in "-r" | "reconfigure" ) ;; "-l" | "logging" ) ;; "uninstall" ) ;; - "enable" ) ;; - "disable" ) ;; + "enable" ) need_root=0;; + "disable" ) need_root=0;; "-d" | "debug" ) ;; "restartdns" ) ;; "-g" | "updateGravity" ) ;; @@ -591,8 +582,8 @@ case "${1}" in "-g" | "updateGravity" ) updateGravityFunc "$@";; "-l" | "logging" ) piholeLogging "$@";; "uninstall" ) uninstallFunc;; - "enable" ) piholeEnable 1;; - "disable" ) piholeEnable 0 "$2";; + "enable" ) piholeEnable true "$2";; + "disable" ) piholeEnable false "$2";; "restartdns" ) restartDNS "$2";; "reloaddns" ) restartDNS "reload";; "setpassword" ) SetWebPassword "$@";; From 3cc194594252966cd4adce76263f2b7c8c3da713 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Jul 2024 10:08:11 +0000 Subject: [PATCH 260/413] Bump pytest from 8.2.2 to 8.3.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.2.2 to 8.3.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.2.2...8.3.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 6d230749..b373944f 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.2.2 +pytest == 8.3.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 tox == 4.16.0 From 0980f9acf62f08d4e46f3c2d993b8cee88e57cd4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Jul 2024 10:38:22 +0000 Subject: [PATCH 261/413] Bump pytest from 8.2.2 to 8.3.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.2.2 to 8.3.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.2.2...8.3.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 73b9ad6e..7c066192 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.2.2 +pytest == 8.3.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 tox == 4.16.0 From 2c32d485bd8d1c5892ae635e3f35065fde004e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 27 Jul 2024 21:55:18 +0200 Subject: [PATCH 262/413] Remove obsolet getFTLPIDFile() We can get the path of the PID file via getFTLConfigValue files.pid MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/utils.sh | 19 ------------- advanced/Templates/pihole-FTL-poststop.sh | 4 +-- advanced/Templates/pihole-FTL-prestart.sh | 4 +-- advanced/Templates/pihole-FTL.service | 4 +-- pihole | 4 +-- test/test_any_utils.py | 34 +---------------------- 6 files changed, 9 insertions(+), 60 deletions(-) diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 2fe419e8..67301394 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -81,25 +81,6 @@ removeKey() { sed -i "/^${key}/d" "${file}" } -####################### -# returns path of FTL's PID file -####################### -getFTLPIDFile() { - local FTLCONFFILE="/etc/pihole/pihole-FTL.conf" - local DEFAULT_PID_FILE="/run/pihole-FTL.pid" - local FTL_PID_FILE - - if [ -s "${FTLCONFFILE}" ]; then - # if PIDFILE is not set in pihole-FTL.conf, use the default path - FTL_PID_FILE="$({ grep '^PIDFILE=' "${FTLCONFFILE}" || echo "${DEFAULT_PID_FILE}"; } | cut -d'=' -f2-)" - else - # if there is no pihole-FTL.conf, use the default path - FTL_PID_FILE="${DEFAULT_PID_FILE}" - fi - - echo "${FTL_PID_FILE}" -} - ####################### # returns FTL's PID based on the content of the pihole-FTL.pid file # diff --git a/advanced/Templates/pihole-FTL-poststop.sh b/advanced/Templates/pihole-FTL-poststop.sh index ac3898d2..b5ddbc97 100755 --- a/advanced/Templates/pihole-FTL-poststop.sh +++ b/advanced/Templates/pihole-FTL-poststop.sh @@ -1,13 +1,13 @@ #!/usr/bin/env sh -# Source utils.sh for getFTLPIDFile() +# Source utils.sh for getFTLConfigValue() PI_HOLE_SCRIPT_DIR='/opt/pihole' utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" # shellcheck disable=SC1090 . "${utilsfile}" # Get file paths -FTL_PID_FILE="$(getFTLPIDFile)" +FTL_PID_FILE="$(getFTLConfigValue files.pid)" # Cleanup rm -f /run/pihole/FTL.sock /dev/shm/FTL-* "${FTL_PID_FILE}" diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index c6817828..d807b81c 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -1,13 +1,13 @@ #!/usr/bin/env sh -# Source utils.sh for getFTLPIDFile() +# Source utils.sh for getFTLConfigValue() PI_HOLE_SCRIPT_DIR='/opt/pihole' utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" # shellcheck disable=SC1090 . "${utilsfile}" # Get file paths -FTL_PID_FILE="$(getFTLPIDFile)" +FTL_PID_FILE="$(getFTLConfigValue files.pid)" # Ensure that permissions are set so that pihole-FTL can edit all necessary files # shellcheck disable=SC2174 diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service index 009401fc..151d4f90 100644 --- a/advanced/Templates/pihole-FTL.service +++ b/advanced/Templates/pihole-FTL.service @@ -9,7 +9,7 @@ # Description: Enable service provided by pihole-FTL daemon ### END INIT INFO -# Source utils.sh for getFTLPIDFile(), getFTLPID() +# Source utils.sh for getFTLConfigValue(), getFTLPID() PI_HOLE_SCRIPT_DIR="/opt/pihole" utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" # shellcheck disable=SC1090 @@ -98,7 +98,7 @@ status() { trap 'cleanup; exit 1' INT HUP TERM ABRT # Get FTL's PID file path -FTL_PID_FILE="$(getFTLPIDFile)" +FTL_PID_FILE="$(getFTLConfigValue files.pid)" # Get FTL's current PID FTL_PID="$(getFTLPID "${FTL_PID_FILE}")" diff --git a/pihole b/pihole index 08ff5b76..4bb7d5e5 100755 --- a/pihole +++ b/pihole @@ -152,7 +152,7 @@ restartDNS() { svcOption="${1:-restart}" # get the current path to the pihole-FTL.pid - FTL_PID_FILE="$(getFTLPIDFile)" + FTL_PID_FILE="$(getFTLConfigValue files.pid)" # Determine if we should reload or restart if [[ "${svcOption}" =~ "reload-lists" ]]; then @@ -337,7 +337,7 @@ statusFunc() { # Determine if there is pihole-FTL service is listening local pid port ftl_pid_file block_status - ftl_pid_file="$(getFTLPIDFile)" + ftl_pid_file="$(getFTLConfigValue files.pid)" pid="$(getFTLPID ${ftl_pid_file})" diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 9eee6885..59745c48 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -82,18 +82,6 @@ def test_key_removal_works(host): assert expected_stdout == output.stdout -def test_getFTLPIDFile_default(host): - """Confirms getFTLPIDFile returns the default PID file path""" - output = host.run( - """ - source /opt/pihole/utils.sh - getFTLPIDFile - """ - ) - expected_stdout = "/run/pihole-FTL.pid\n" - assert expected_stdout == output.stdout - - def test_getFTLPID_default(host): """Confirms getFTLPID returns the default value if FTL is not running""" output = host.run( @@ -106,27 +94,7 @@ def test_getFTLPID_default(host): assert expected_stdout == output.stdout -def test_getFTLPIDFile_and_getFTLPID_custom(host): - """Confirms getFTLPIDFile returns a custom PID file path""" - host.run( - """ - tmpfile=$(mktemp) - echo "PIDFILE=${tmpfile}" > /etc/pihole/pihole-FTL.conf - echo "1234" > ${tmpfile} - """ - ) - output = host.run( - """ - source /opt/pihole/utils.sh - FTL_PID_FILE=$(getFTLPIDFile) - getFTLPID "${FTL_PID_FILE}" - """ - ) - expected_stdout = "1234\n" - assert expected_stdout == output.stdout - - -def test_getFTLConfigValue_getFTLConfigValue(host): +def test_setFTLConfigValue_getFTLConfigValue(host): """ Confirms getFTLConfigValue works (also assumes setFTLConfigValue works) Requires FTL to be installed, so we do that first From db1431a1ae9bb796dc1a2e381f0a565c385d7c54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 27 Jul 2024 22:25:45 +0200 Subject: [PATCH 263/413] Remove obsolet files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .stickler.yml | 10 ---------- .yamllint.conf | 3 --- advanced/Templates/pihole-FTL.conf | 2 -- 3 files changed, 15 deletions(-) delete mode 100644 .stickler.yml delete mode 100644 .yamllint.conf delete mode 100644 advanced/Templates/pihole-FTL.conf diff --git a/.stickler.yml b/.stickler.yml deleted file mode 100644 index 5fdbbf1e..00000000 --- a/.stickler.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -linters: - shellcheck: - shell: bash - phpcs: - flake8: - max-line-length: 120 - yamllint: - config: ./.yamllint.conf - remarklint: diff --git a/.yamllint.conf b/.yamllint.conf deleted file mode 100644 index d1b0953b..00000000 --- a/.yamllint.conf +++ /dev/null @@ -1,3 +0,0 @@ -rules: - line-length: disable - document-start: disable diff --git a/advanced/Templates/pihole-FTL.conf b/advanced/Templates/pihole-FTL.conf deleted file mode 100644 index 269fcf9d..00000000 --- a/advanced/Templates/pihole-FTL.conf +++ /dev/null @@ -1,2 +0,0 @@ -#; Pi-hole FTL config file -#; Comments should start with #; to avoid issues with PHP and bash reading this file From 1ed5994fa575c95201df0259282b563195077e88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 28 Jul 2024 13:01:41 +0200 Subject: [PATCH 264/413] Remove temporal log file symlink code. It was introduced 2 years ago with v5.11 and always thought to be temporarily only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 12 ---------- automated install/basic-install.sh | 28 ----------------------- 2 files changed, 40 deletions(-) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index d807b81c..c604d5d4 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -32,15 +32,3 @@ find /etc/pihole -type d -exec chmod 0755 {} \; [ -f /var/log/pihole/FTL.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/FTL.log [ -f /var/log/pihole/pihole.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/pihole.log [ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases - - -# Backward compatibility for user-scripts that still expect log files in /var/log instead of /var/log/pihole -# Should be removed with Pi-hole v6.0 -if [ ! -f /var/log/pihole.log ]; then - ln -sf /var/log/pihole/pihole.log /var/log/pihole.log - chown -h pihole:pihole /var/log/pihole.log -fi -if [ ! -f /var/log/pihole-FTL.log ]; then - ln -sf /var/log/pihole/FTL.log /var/log/pihole-FTL.log - chown -h pihole:pihole /var/log/pihole-FTL.log -fi diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ef891ec4..ab3a2290 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2323,34 +2323,6 @@ main() { # the service before enabling causes installer to exit enable_service pihole-FTL - # If this is an update from a previous Pi-hole installation - # we need to move any existing `pihole*` logs from `/var/log` to `/var/log/pihole` - # if /var/log/pihole.log is not a symlink (set during FTL startup) move the files - # can be removed with Pi-hole v6.0 - # To be sure FTL is not running when we move the files we explicitly stop it here - - stop_service pihole-FTL &>/dev/null - - if [ ! -d /var/log/pihole/ ]; then - mkdir -m 0755 /var/log/pihole/ - fi - - # Special handling for pihole-FTL.log -> pihole/FTL.log - if [ -f /var/log/pihole-FTL.log ] && [ ! -L /var/log/pihole-FTL.log ]; then - # /var/log/pihole-FTL.log -> /var/log/pihole/FTL.log - # /var/log/pihole-FTL.log.1 -> /var/log/pihole/FTL.log.1 - # /var/log/pihole-FTL.log.2.gz -> /var/log/pihole/FTL.log.2.gz - # /var/log/pihole-FTL.log.3.gz -> /var/log/pihole/FTL.log.3.gz - # /var/log/pihole-FTL.log.4.gz -> /var/log/pihole/FTL.log.4.gz - # /var/log/pihole-FTL.log.5.gz -> /var/log/pihole/FTL.log.5.gz - for f in /var/log/pihole-FTL.log*; do mv "$f" "$(sed "s/pihole-/pihole\//" <<<"$f")"; done - fi - - # Remaining log files - if [ -f /var/log/pihole.log ] && [ ! -L /var/log/pihole.log ]; then - mv /var/log/pihole*.* /var/log/pihole/ 2>/dev/null - fi - restart_service pihole-FTL # Download and compile the aggregated block list From 58a41641ab8e499ab5a27109e04c8a1e174b3860 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 10 Aug 2024 10:19:39 +0000 Subject: [PATCH 265/413] Bump tox from 4.16.0 to 4.17.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.16.0 to 4.17.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.16.0...4.17.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index b373944f..623e2ec5 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.3.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.16.0 +tox == 4.17.1 From 21a9410242aa337fb4213576287647cf5fd095a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 10 Aug 2024 10:19:42 +0000 Subject: [PATCH 266/413] Bump pyyaml from 6.0.1 to 6.0.2 in /test Bumps [pyyaml](https://github.com/yaml/pyyaml) from 6.0.1 to 6.0.2. - [Release notes](https://github.com/yaml/pyyaml/releases) - [Changelog](https://github.com/yaml/pyyaml/blob/main/CHANGES) - [Commits](https://github.com/yaml/pyyaml/compare/6.0.1...6.0.2) --- updated-dependencies: - dependency-name: pyyaml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index b373944f..e09fcea3 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,4 +1,4 @@ -pyyaml == 6.0.1 +pyyaml == 6.0.2 pytest == 8.3.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 From c42890ad5481edfd7da349b0473b22d08af44cf4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 10 Aug 2024 10:56:14 +0000 Subject: [PATCH 267/413] Bump pyyaml from 6.0.1 to 6.0.2 in /test Bumps [pyyaml](https://github.com/yaml/pyyaml) from 6.0.1 to 6.0.2. - [Release notes](https://github.com/yaml/pyyaml/releases) - [Changelog](https://github.com/yaml/pyyaml/blob/main/CHANGES) - [Commits](https://github.com/yaml/pyyaml/compare/6.0.1...6.0.2) --- updated-dependencies: - dependency-name: pyyaml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 7c066192..84887ee8 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,4 +1,4 @@ -pyyaml == 6.0.1 +pyyaml == 6.0.2 pytest == 8.3.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 From 4f52c6afc056f3fe430dec91850cf79af29067bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 10 Aug 2024 12:01:35 +0000 Subject: [PATCH 268/413] Bump tox from 4.16.0 to 4.17.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.16.0 to 4.17.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.16.0...4.17.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 84887ee8..a313086b 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.2 pytest == 8.3.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.16.0 +tox == 4.17.1 pytest-clarity == 1.0.1 From 27fd80c01e9430a60917bf09668e8801bfc873fb Mon Sep 17 00:00:00 2001 From: Allen <64094914+allendema@users.noreply.github.com> Date: Thu, 15 Aug 2024 02:31:33 +0200 Subject: [PATCH 269/413] [fix] [v6] typo in bash-completion allow-regex option also removes a space from double whitespace Signed-off-by: Allen Dema <64094914+allendema@users.noreply.github.com> Signed-off-by: Allen <64094914+allendema@users.noreply.github.com> --- advanced/bash-completion/pihole | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 45536d69..112e64ab 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -1,5 +1,5 @@ _pihole() { - local cur prev opts opts_checkout opts_debug opts_logging opts_query opts_update opts_version + local cur prev opts opts_checkout opts_debug opts_logging opts_query opts_update opts_version COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" @@ -10,7 +10,7 @@ _pihole() { opts="allow allow-regex allow-wild deny checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard arpflush" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) ;; - "allow"|"deny"|"wildcard"|"regex"|"allow-regx"|"allow-wild") + "allow"|"deny"|"wildcard"|"regex"|"allow-regex"|"allow-wild") opts_lists="\not \--delmode \--quiet \--list \--help" COMPREPLY=( $(compgen -W "${opts_lists}" -- ${cur}) ) ;; From 5024a98a4fdff701c118e7308d4b288f85b00701 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 17 Aug 2024 10:06:35 +0000 Subject: [PATCH 270/413] Bump tox from 4.17.1 to 4.18.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.17.1 to 4.18.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.17.1...4.18.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 492cdf79..fe83d743 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.2 pytest == 8.3.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.17.1 +tox == 4.18.0 From f6681f75fbb155d7d661a56fb12903e3841b269f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 17 Aug 2024 10:39:02 +0000 Subject: [PATCH 271/413] Bump tox from 4.17.1 to 4.18.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.17.1 to 4.18.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.17.1...4.18.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index a313086b..4bc7c372 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.2 pytest == 8.3.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.17.1 +tox == 4.18.0 pytest-clarity == 1.0.1 From fc156f521ccce13449cfefeffe28f21762daa53e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 10 Aug 2024 13:15:34 +0200 Subject: [PATCH 272/413] Fix setting query logging and privacy level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ab3a2290..5b517ab9 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -86,9 +86,9 @@ adlistFile="/etc/pihole/adlists.list" IPV4_ADDRESS=${IPV4_ADDRESS} IPV6_ADDRESS=${IPV6_ADDRESS} # Give settings their default values. These may be changed by prompts later in the script. -QUERY_LOGGING=true +QUERY_LOGGING= WEBPORT=8080 -PRIVACY_LEVEL=0 +PRIVACY_LEVEL= # Where old configs go to if a v6 migration is performed V6_CONF_MIGRATION_DIR="/etc/pihole/migration_backup_v6" @@ -2300,6 +2300,15 @@ main() { pihole -a -p "${pw}" fi + # write privacy level and logging to pihole.toml + # set on fresh installations by setPrivacyLevel() and setLogging( + if [ -n "${QUERY_LOGGING}" ]; then + pihole-FTL --config dns.queryLogging "${QUERY_LOGGING}" + fi + if [ -n "${PRIVACY_LEVEL}" ]; then + pihole-FTL --config misc.privacylevel "${PRIVACY_LEVEL}" + fi + # Migrate existing install to v6.0 migrate_dnsmasq_configs From 8e4fc27168a424a0a628cb27ed5967958ee09c0b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 24 Aug 2024 10:06:48 +0200 Subject: [PATCH 273/413] Add pihole api [endpoint] callback suitable for local API requests Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 63 ++++++++++++++++++++++++++++++++++------- pihole | 2 ++ 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 5843c585..4776222b 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -146,19 +146,22 @@ GetFTLData() { response=$(curl -skS -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" ) # status are the last 3 characters - status=$(printf %s "${response#"${response%???}"}") + status="${response#"${response%???}"}" # data is everything from response without the last 3 characters - data=$(printf %s "${response%???}") + data="${response%???}" - if [ "${status}" = 200 ]; then - # response OK - printf %s "${data}" - elif [ "${status}" = 000 ]; then - # connection lost - echo "000" - elif [ "${status}" = 401 ]; then - # unauthorized - echo "401" + if [ "${2}" = "raw" ]; then + # return the raw response + echo "${response}" + else + # return only the data + if [ "${status}" = 200 ]; then + # response OK + echo "${data}" + else + # connection lost + echo "${status}" + fi fi } @@ -226,3 +229,41 @@ secretRead() { # restore original terminal settings stty "${stty_orig}" } + +apiFunc() { + local data response status status_col + + # Authenticate with the API + LoginAPI + + echo "Requesting: ${COL_PURPLE}GET ${COL_CYAN}${API_URL}${COL_YELLOW}$1${COL_NC}" + echo "" + + # Get the data from the API + response=$(GetFTLData "$1" raw) + + # status are the last 3 characters + status="${response#"${response%???}"}" + # data is everything from response without the last 3 characters + data="${response%???}" + + # Output the status (200 -> green, else red) + if [ "${status}" = 200 ]; then + status_col="${COL_GREEN}" + else + status_col="${COL_RED}" + fi + echo "Status: ${status_col}${status}${COL_NC}" + + # Output the data. Format it with jq if available and data is actually JSON. + # Otherwise just print it + echo "Data:" + if command -v jq >/dev/null && echo "${data}" | jq . >/dev/null 2>&1; then + echo "${data}" | jq . + else + echo "${data}" + fi + + # Delete the session + LogoutAPI +} diff --git a/pihole b/pihole index 4bb7d5e5..8a9aa297 100755 --- a/pihole +++ b/pihole @@ -550,6 +550,7 @@ case "${1}" in "updatechecker" ) ;; "arpflush" ) ;; "-t" | "tail" ) ;; + "api" ) need_root=0;; * ) helpFunc;; esac @@ -591,5 +592,6 @@ case "${1}" in "updatechecker" ) shift; updateCheckFunc "$@";; "arpflush" ) arpFunc "$@";; "-t" | "tail" ) tailFunc "$2";; + "api" ) apiFunc "$2";; * ) helpFunc;; esac From b304562c8ebbb9ecf3a678bd06be2ffe72e90161 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 24 Aug 2024 10:37:58 +0200 Subject: [PATCH 274/413] Add session details to API command output Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 4776222b..43c91d69 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -88,6 +88,9 @@ LoginAPI() { # Exit early if authentication is not needed if [ "${needAuth}" = false ]; then + if [ "${1}" = "verbose" ]; then + echo "API Authentication: Not needed" + fi return fi @@ -95,8 +98,15 @@ LoginAPI() { if [ -r /etc/pihole/cli_pw ]; then password=$(cat /etc/pihole/cli_pw) + if [ "${1}" = "verbose" ]; then + echo "API Authentication: Trying to use CLI password" + fi + # Try to authenticate using the CLI password - Authentication + Authentication "${1}" + + elif [ "${1}" = "verbose" ]; then + echo "API Authentication: CLI password not available" fi @@ -109,7 +119,7 @@ LoginAPI() { secretRead; printf '\n' # Try to authenticate again - Authentication + Authentication "${1}" done } @@ -124,6 +134,14 @@ Authentication() { # obtain validity and session ID from session response validSession=$(echo "${sessionResponse}"| jq .session.valid 2>/dev/null) SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null) + + if [ "${1}" = "verbose" ]; then + if [ "${validSession}" = true ]; then + echo "API Authentication: ${COL_GREEN}Success${COL_NC}" + else + echo "API Authentication: ${COL_RED}Failed${COL_NC}" + fi + fi } LogoutAPI() { @@ -134,10 +152,12 @@ LogoutAPI() { deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in - "401") printf "%b" "Logout attempt without a valid session. Unauthorized!\n";; - esac; + "401") echo "Logout attempt without a valid session. Unauthorized!";; + "204") if [ "${1}" = "verbose" ]; then echo "API Logout: ${COL_GREEN}Success${COL_NC} (session deleted)"; fi;; + esac; + elif [ "${1}" = "verbose" ]; then + echo "API Logout: ${COL_GREEN}Success${COL_NC} (no valid session)" fi - } GetFTLData() { @@ -234,7 +254,8 @@ apiFunc() { local data response status status_col # Authenticate with the API - LoginAPI + LoginAPI verbose + echo "" echo "Requesting: ${COL_PURPLE}GET ${COL_CYAN}${API_URL}${COL_YELLOW}$1${COL_NC}" echo "" @@ -265,5 +286,5 @@ apiFunc() { fi # Delete the session - LogoutAPI + LogoutAPI verbose } From 72afa1073da05a0e7ddb22c0acdb0b63190b8284 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 25 Aug 2024 21:47:07 +0100 Subject: [PATCH 275/413] Add missing creation of table antigravity in migration script 16 to 17 Signed-off-by: Adam Warner --- advanced/Scripts/database_migration/gravity/16_to_17.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/advanced/Scripts/database_migration/gravity/16_to_17.sql b/advanced/Scripts/database_migration/gravity/16_to_17.sql index 23532e3a..ed2ec7c5 100644 --- a/advanced/Scripts/database_migration/gravity/16_to_17.sql +++ b/advanced/Scripts/database_migration/gravity/16_to_17.sql @@ -8,6 +8,12 @@ ALTER TABLE adlist ADD COLUMN type INTEGER NOT NULL DEFAULT 0; UPDATE adlist SET type = 0; +CREATE TABLE antigravity +( + domain TEXT NOT NULL, + adlist_id INTEGER NOT NULL REFERENCES adlist (id) +); + CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id FROM antigravity LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id From a302d7b5d741f2ff27de6ed8be2149bca3c6ba07 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 22 Aug 2024 21:22:52 +0200 Subject: [PATCH 276/413] Make the help text of "pihole checkout [what] [branch]" more colorful Signed-off-by: DL6ER --- advanced/Scripts/piholeCheckout.sh | 11 ++++++----- automated install/basic-install.sh | 7 ++++--- pihole | 24 +++++++++++++----------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index 41fd8606..c7780cd0 100755 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -77,7 +77,7 @@ checkout() { if [[ "${1}" == "dev" ]] ; then # Shortcut to check out development branches - echo -e " ${INFO} Shortcut \"dev\" detected - checking out development / devel branches..." + echo -e " ${INFO} Shortcut \"${COL_YELLOW}dev${COL_NC}\" detected - checking out development / devel branches..." echo "" echo -e " ${INFO} Pi-hole Core" fetch_checkout_pull_branch "${PI_HOLE_FILES_DIR}" "development" || { echo " ${CROSS} Unable to pull Core development branch"; exit 1; } @@ -92,7 +92,7 @@ checkout() { chmod 644 /etc/pihole/ftlbranch elif [[ "${1}" == "master" ]] ; then # Shortcut to check out master branches - echo -e " ${INFO} Shortcut \"master\" detected - checking out master branches..." + echo -e " ${INFO} Shortcut \"${COL_YELLOW}master${COL_NC}\" detected - checking out master branches..." echo -e " ${INFO} Pi-hole core" fetch_checkout_pull_branch "${PI_HOLE_FILES_DIR}" "master" || { echo " ${CROSS} Unable to pull Core master branch"; exit 1; } echo -e " ${INFO} Web interface" @@ -123,7 +123,7 @@ checkout() { echo "" # Have the user choose the branch they want if ! (for e in "${corebranches[@]}"; do [[ "$e" == "${2}" ]] && exit 0; done); then - echo -e " ${INFO} Requested branch \"${2}\" is not available" + echo -e " ${INFO} Requested branch \"${COL_CYAN}${2}${COL_NC}\" is not available" echo -e " ${INFO} Available branches for Core are:" for e in "${corebranches[@]}"; do echo " - $e"; done exit 1 @@ -150,7 +150,7 @@ checkout() { echo "" # Have the user choose the branch they want if ! (for e in "${webbranches[@]}"; do [[ "$e" == "${2}" ]] && exit 0; done); then - echo -e " ${INFO} Requested branch \"${2}\" is not available" + echo -e " ${INFO} Requested branch \"${COL_CYAN}${2}${COL_NC}\" is not available" echo -e " ${INFO} Available branches for Web Admin are:" for e in "${webbranches[@]}"; do echo " - $e"; done exit 1 @@ -164,6 +164,7 @@ checkout() { path="${2}/${binary}" oldbranch="$(pihole-FTL -b)" + echo -e " ${INFO} Checking for branch ${COL_CYAN}${2}${COL_NC} on https://ftl.pi-hole.net" check_download_exists "$path" local ret=$? if [ $ret -eq 0 ]; then @@ -193,7 +194,7 @@ checkout() { fi else - echo -e " ${INFO} Requested option \"${1}\" is not available" + echo -e " ${CROSS} Requested option \"${1}\" is not available" exit 1 fi diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 5b517ab9..1b11f2f2 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -16,9 +16,10 @@ # # curl -sSL https://install.pi-hole.net | bash -# -e option instructs bash to immediately exit if any command [1] has a non-zero exit status -# We do not want users to end up with a partially working install, so we exit the script -# instead of continuing the installation with something broken +# -e option instructs bash to immediately exit if any command [1] has a non-zero +# exit status We do not want users to end up with a partially working install, +# so we exit the script instead of continuing the installation with something +# broken set -e # Append common folders to the PATH to ensure that all basic commands are available. diff --git a/pihole b/pihole index 4bb7d5e5..93545cb7 100755 --- a/pihole +++ b/pihole @@ -408,19 +408,21 @@ piholeCheckoutFunc() { unsupportedFunc else if [[ "$2" == "-h" ]] || [[ "$2" == "--help" ]]; then - echo "Usage: pihole checkout [repo] [branch] - Example: 'pihole checkout master' or 'pihole checkout core dev' - Switch Pi-hole subsystems to a different GitHub branch + echo "Switch Pi-hole subsystems to a different GitHub branch + Usage: ${COL_GREEN}pihole checkout${COL_NC} ${COL_YELLOW}shortcut${COL_NC} + or ${COL_GREEN}pihole checkout${COL_NC} ${COL_PURPLE}repo${COL_NC} ${COL_CYAN}branch${COL_NC} + Example: ${COL_GREEN}pihole checkout${COL_NC} ${COL_YELLOW}master${COL_NC} + or ${COL_GREEN}pihole checkout${COL_NC} ${COL_PURPLE}ftl ${COL_CYAN}development${COL_NC} - Repositories: - core [branch] Change the branch of Pi-hole's core subsystem - web [branch] Change the branch of Web Interface subsystem - ftl [branch] Change the branch of Pi-hole's FTL subsystem + Shortcuts: + ${COL_YELLOW}master${COL_NC} Update all subsystems to the latest stable release + ${COL_YELLOW}dev${COL_NC} Update all subsystems to the latest development release + + Individual components: + ${COL_PURPLE}core${COL_NC} ${COL_CYAN}branch${COL_NC} Change the branch of Pi-hole's core subsystem + ${COL_PURPLE}web${COL_NC} ${COL_CYAN}branch${COL_NC} Change the branch of the web interface subsystem + ${COL_PURPLE}ftl${COL_NC} ${COL_CYAN}branch${COL_NC} Change the branch of Pi-hole's FTL subsystem" - Branches: - master Update subsystems to the latest stable release - dev Update subsystems to the latest development release - branchname Update subsystems to the specified branchname" exit 0 fi From 9e9c985245308a2ee36a69bbbabb0b2bdf21441f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 26 Aug 2024 06:11:42 +0200 Subject: [PATCH 277/413] FTL checkout: Check for availability of branches before trying to download from the webserver. Also, fix check_download_exists() possibly killing the script on non-availability of requested branches Signed-off-by: DL6ER --- advanced/Scripts/piholeCheckout.sh | 44 +++++++++++++++++++++--------- automated install/basic-install.sh | 17 +++++------- pihole | 1 + 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index c7780cd0..65e6ebb8 100755 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -161,34 +161,52 @@ checkout() { elif [[ "${1}" == "ftl" ]] ; then local path local oldbranch + local existing=false path="${2}/${binary}" oldbranch="$(pihole-FTL -b)" - echo -e " ${INFO} Checking for branch ${COL_CYAN}${2}${COL_NC} on https://ftl.pi-hole.net" - check_download_exists "$path" - local ret=$? - if [ $ret -eq 0 ]; then - echo " ${TICK} Branch ${2} exists" + # Check if requested branch is available + echo -e " ${INFO} Checking for availability of branch ${COL_CYAN}${2}${COL_NC} on GitHub" + ftlbranches=( $(git ls-remote https://github.com/pi-hole/ftl | grep "refs/heads" | cut -d'/' -f3- -) ) + # If returned array is empty -> connectivity issue + if [[ ${#ftlbranches[@]} -eq 0 ]]; then + echo -e " ${CROSS} Unable to fetch branches from GitHub. Please check your Internet connection and try again later." + exit 1 + fi + + for e in "${ftlbranches[@]}"; do [[ "$e" == "${2}" ]] && existing=true; done + if [[ "${existing}" == false ]]; then + echo -e " ${CROSS} Requested branch is not available\n" + echo -e " ${INFO} Available branches are:" + for e in "${ftlbranches[@]}"; do echo " - $e"; done + exit 1 + fi + echo -e " ${TICK} Branch ${2} exists on GitHub" + + echo -e " ${INFO} Checking for ${COL_YELLOW}${binary}${COL_NC} binary on https://ftl.pi-hole.net" + + if check_download_exists "$path"; then + echo " ${TICK} Binary exists" echo "${2}" > /etc/pihole/ftlbranch chmod 644 /etc/pihole/ftlbranch - echo -e " ${INFO} Switching to branch: \"${2}\" from \"${oldbranch}\"" + echo -e " ${INFO} Switching to branch: ${COL_CYAN}${2}${COL_NC} from ${COL_CYAN}${oldbranch}${COL_NC}" FTLinstall "${binary}" restart_service pihole-FTL enable_service pihole-FTL # Update local and remote versions via updatechecker /opt/pihole/updatecheck.sh else - if [[ $ret -eq 1 ]]; then - echo " ${CROSS} Requested branch \"${2}\" is not available" - ftlbranches=( $(git ls-remote https://github.com/pi-hole/ftl | grep 'heads' | sed 's/refs\/heads\///;s/ //g' | awk '{print $2}') ) - echo -e " ${INFO} Available branches for FTL are:" - for e in "${ftlbranches[@]}"; do echo " - $e"; done + if [ $? -eq 1 ]; then + # Binary for requested branch is not available, may still be + # int he process of being built or CI build job failed + printf " %b Binary for requested branch is not available, please try again later.\\n" ${CROSS} + printf " If the issue persists, please contact Pi-hole Support and ask them to re-generate the binary.\\n" exit 1 - elif [[ $ret -eq 2 ]]; then + elif [ $? -eq 2 ]; then printf " %b Unable to download from ftl.pi-hole.net. Please check your Internet connection and try again later.\\n" "${CROSS}" exit 1 else - printf " %b Unknown error. Please contact Pi-hole Support\\n" "${CROSS}" + printf " %b Unknown checkout error. Please contact Pi-hole Support\\n" "${CROSS}" exit 1 fi fi diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 1b11f2f2..9927b374 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -16,10 +16,9 @@ # # curl -sSL https://install.pi-hole.net | bash -# -e option instructs bash to immediately exit if any command [1] has a non-zero -# exit status We do not want users to end up with a partially working install, -# so we exit the script instead of continuing the installation with something -# broken +# -e option instructs bash to immediately exit if any command [1] has a non-zero exit status +# We do not want users to end up with a partially working install, so we exit the script +# instead of continuing the installation with something broken set -e # Append common folders to the PATH to ensure that all basic commands are available. @@ -1694,7 +1693,7 @@ update_dialogs() { check_download_exists() { # Check if the download exists and we can reach the server - status=$(curl --head --silent "https://ftl.pi-hole.net/${1}" | head -n 1) + local status=$(curl --head --silent "https://ftl.pi-hole.net/${1}" | head -n 1) # Check the status code if grep -q "200" <<<"$status"; then @@ -2012,14 +2011,12 @@ FTLcheckUpdate() { # Check whether or not the binary for this FTL branch actually exists. If not, then there is no update! # shellcheck disable=SC1090 - check_download_exists "$path" - local ret=$? - if [ $ret -ne 0 ]; then - if [[ $ret -eq 1 ]]; then + if ! check_download_exists "$path"; then + if [ $? -eq 1 ]; then printf " %b Branch \"%s\" is not available.\\n" "${INFO}" "${ftlBranch}" printf " %b Use %bpihole checkout ftl [branchname]%b to switch to a valid branch.\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${COL_NC}" return 2 - elif [[ $ret -eq 2 ]]; then + elif [ $? -eq 2 ]; then printf " %b Unable to download from ftl.pi-hole.net. Please check your Internet connection and try again later.\\n" "${CROSS}" return 3 else diff --git a/pihole b/pihole index 93545cb7..f4951d27 100755 --- a/pihole +++ b/pihole @@ -411,6 +411,7 @@ piholeCheckoutFunc() { echo "Switch Pi-hole subsystems to a different GitHub branch Usage: ${COL_GREEN}pihole checkout${COL_NC} ${COL_YELLOW}shortcut${COL_NC} or ${COL_GREEN}pihole checkout${COL_NC} ${COL_PURPLE}repo${COL_NC} ${COL_CYAN}branch${COL_NC} + Example: ${COL_GREEN}pihole checkout${COL_NC} ${COL_YELLOW}master${COL_NC} or ${COL_GREEN}pihole checkout${COL_NC} ${COL_PURPLE}ftl ${COL_CYAN}development${COL_NC} From e8582f774099928741351fcc961f407977c663e6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 26 Aug 2024 19:44:42 +0200 Subject: [PATCH 278/413] Add pihole -h and autocompletion entry Signed-off-by: DL6ER --- advanced/bash-completion/pihole | 2 +- pihole | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 112e64ab..674eeea4 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -7,7 +7,7 @@ _pihole() { case "${prev}" in "pihole") - opts="allow allow-regex allow-wild deny checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard arpflush" + opts="allow allow-regex allow-wild deny checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard arpflush api" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) ;; "allow"|"deny"|"wildcard"|"regex"|"allow-regex"|"allow-wild") diff --git a/pihole b/pihole index 8a9aa297..ed4dd67a 100755 --- a/pihole +++ b/pihole @@ -484,6 +484,7 @@ Debugging Options: -t, tail [arg] View the live output of the Pi-hole log. Add an optional argument to filter the log (regular expressions are supported) + api Query the Pi-hole API at Options: From cfbf3f61cda1b89d27db42e96b1a122018607fbd Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 26 Aug 2024 21:30:12 +0100 Subject: [PATCH 279/413] Update advanced/Scripts/database_migration/gravity/16_to_17.sql Co-authored-by: Dominik Signed-off-by: Adam Warner --- advanced/Scripts/database_migration/gravity/16_to_17.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/database_migration/gravity/16_to_17.sql b/advanced/Scripts/database_migration/gravity/16_to_17.sql index ed2ec7c5..47b631cc 100644 --- a/advanced/Scripts/database_migration/gravity/16_to_17.sql +++ b/advanced/Scripts/database_migration/gravity/16_to_17.sql @@ -8,7 +8,7 @@ ALTER TABLE adlist ADD COLUMN type INTEGER NOT NULL DEFAULT 0; UPDATE adlist SET type = 0; -CREATE TABLE antigravity +CREATE TABLE IF NOT EXISTS antigravity ( domain TEXT NOT NULL, adlist_id INTEGER NOT NULL REFERENCES adlist (id) From a21b4c5203f1ff0fba37a34b2716c86c71152d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 27 Aug 2024 21:24:50 +0200 Subject: [PATCH 280/413] Update existing logrotate files to inlcude webserver.log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 35 +++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9927b374..6b65313e 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1496,6 +1496,7 @@ create_pihole_user() { installLogrotate() { local str="Installing latest logrotate script" local target=/etc/pihole/logrotate + local logfileUpdate=false printf "\\n %b %s..." "${INFO}" "${str}" if [[ -f ${target} ]]; then @@ -1506,17 +1507,35 @@ installLogrotate() { sed -i 's/\/var\/log\/pihole-FTL.log/\/var\/log\/pihole\/FTL.log/g' ${target} printf "\\n\\t%b Old log file paths updated in existing logrotate file. \\n" "${INFO}" - return 3 + logfileUpdate=true fi - printf "\\n\\t%b Existing logrotate file found. No changes made.\\n" "${INFO}" - # Return value isn't that important, using 2 to indicate that it's not a fatal error but - # the function did not complete. - return 2 + # Account for added webserver.log in v6.0 + if ! grep -q "/var/log/pihole/webserver.log" ${target}; then + echo "/var/log/pihole/webserver.log { +# su # +weekly +copytruncate +rotate 3 +compress +delaycompress +notifempty +nomail +}" >> ${target} + + printf "\\n\\t%b webserver.log added to logrotate file. \\n" "${INFO}" + logfileUpdate=true + fi + if [[ "${logfileUpdate}" == false ]]; then + printf "\\n\\t%b Existing logrotate file found. No changes made.\\n" "${INFO}" + return + fi + else + # Copy the file over from the local repo + # Logrotate config file must be owned by root and not writable by group or other + install -o root -g root -D -m 644 -T "${PI_HOLE_LOCAL_REPO}"/advanced/Templates/logrotate ${target} fi - # Copy the file over from the local repo - # Logrotate config file must be owned by root and not writable by group or other - install -o root -g root -D -m 644 -T "${PI_HOLE_LOCAL_REPO}"/advanced/Templates/logrotate ${target} + # Different operating systems have different user / group # settings for logrotate that makes it impossible to create # a static logrotate file that will work with e.g. From 5e69078ed10beaf4067fb2b10749b71b833a4d48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 1 Sep 2024 19:36:29 +0200 Subject: [PATCH 281/413] Disable SELINUX on CentOS 9 test dockerfile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/_centos_9.Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/_centos_9.Dockerfile b/test/_centos_9.Dockerfile index 2e3d055a..7e3c5b3a 100644 --- a/test/_centos_9.Dockerfile +++ b/test/_centos_9.Dockerfile @@ -1,4 +1,6 @@ FROM quay.io/centos/centos:stream9 +# Disable SELinux +RUN echo "SELINUX=disabled" > /etc/selinux/config RUN yum install -y --allowerasing curl git initscripts ENV GITDIR=/etc/.pihole From 632d8af4b069ce4f965701bab96766e65b3850fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 1 Sep 2024 19:36:29 +0200 Subject: [PATCH 282/413] Disable SELINUX on CentOS 9 test dockerfile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/_centos_9.Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/_centos_9.Dockerfile b/test/_centos_9.Dockerfile index 6ccd18b6..35f7b3f8 100644 --- a/test/_centos_9.Dockerfile +++ b/test/_centos_9.Dockerfile @@ -1,4 +1,6 @@ FROM quay.io/centos/centos:stream9 +# Disable SELinux +RUN echo "SELINUX=disabled" > /etc/selinux/config RUN yum install -y --allowerasing curl git initscripts ENV GITDIR /etc/.pihole From b4444023a2f9620b3f7a2a7971f0a8d3fd5e54d7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Mar 2024 05:49:24 +0100 Subject: [PATCH 283/413] Wait after restarting FTL before trying to check version Signed-off-by: DL6ER --- advanced/Scripts/piholeCheckout.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index 65e6ebb8..0514b8eb 100755 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -193,6 +193,19 @@ checkout() { FTLinstall "${binary}" restart_service pihole-FTL enable_service pihole-FTL + str="Restarting FTL..." + echo -ne " ${INFO} ${str}" + # Wait until name resolution is working again after restarting FTL, + # so that the updatechecker can run successfully and does not fail + # trying to resolve github.com + until getent hosts github.com &> /dev/null; do + # Append one dot for each second waiting + str="${str}." + echo -ne " ${OVER} ${INFO} ${str}" + sleep 1 + done + echo -e " ${OVER} ${TICK} Restarted FTL service" + # Update local and remote versions via updatechecker /opt/pihole/updatecheck.sh else From bcef4f0c97637f7d17a94fa8130dda7785ba6264 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 2 Sep 2024 18:59:42 +0200 Subject: [PATCH 284/413] pihole status should return (= exit) early on error instead of continuing the script Signed-off-by: DL6ER --- pihole | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pihole b/pihole index f3e0a73a..98f478a3 100755 --- a/pihole +++ b/pihole @@ -346,7 +346,7 @@ statusFunc() { "web") echo "-1";; *) echo -e " ${CROSS} DNS service is NOT running";; esac - return 0 + exit 0 else # get the DNS port pihole-FTL is listening on port="$(getFTLConfigValue dns.port)" @@ -355,7 +355,7 @@ statusFunc() { "web") echo "-1";; *) echo -e " ${CROSS} DNS service is NOT listening";; esac - return 0 + exit 0 else if [[ "${1}" != "web" ]]; then echo -e " ${TICK} FTL is listening on port ${port}" @@ -377,7 +377,8 @@ statusFunc() { *) echo -e " ${CROSS} Pi-hole blocking is disabled";; esac fi -exit 0 + + exit 0 } tailFunc() { From f66707bd3e65a625e8fa7195dc5e47851c48febd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 20:30:11 +0000 Subject: [PATCH 285/413] Bump actions/setup-python from 5.1.1 to 5.2.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.1.1 to 5.2.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.1.1...v5.2.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9ce20a01..cbade0e5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,7 +74,7 @@ jobs: uses: actions/checkout@v4.1.7 - name: Set up Python 3.10 - uses: actions/setup-python@v5.1.1 + uses: actions/setup-python@v5.2.0 with: python-version: "3.10" From cd7226d5e98114dc7f506700cc7c4c7d38a43f93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 26 Jul 2024 13:19:11 +0200 Subject: [PATCH 286/413] Remove obsolet Debian 10 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 1 - test/_debian_10.Dockerfile | 17 ----------------- test/tox.debian_10.ini | 10 ---------- 3 files changed, 28 deletions(-) delete mode 100644 test/_debian_10.Dockerfile delete mode 100644 test/tox.debian_10.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cbade0e5..0eef25f0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,7 +56,6 @@ jobs: matrix: distro: [ - debian_10, debian_11, debian_12, ubuntu_20, diff --git a/test/_debian_10.Dockerfile b/test/_debian_10.Dockerfile deleted file mode 100644 index dc813ac2..00000000 --- a/test/_debian_10.Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM buildpack-deps:buster-scm - -ENV GITDIR=/etc/.pihole -ENV SCRIPTDIR=/opt/pihole - -RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole -ADD . $GITDIR -RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR - -RUN true && \ - chmod +x $SCRIPTDIR/* - -ENV SKIP_INSTALL=true -ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net - -#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/tox.debian_10.ini b/test/tox.debian_10.ini deleted file mode 100644 index 9995a852..00000000 --- a/test/tox.debian_10.ini +++ /dev/null @@ -1,10 +0,0 @@ -[tox] -envlist = py3 - -[testenv:py3] -allowlist_externals = docker -deps = -rrequirements.txt -setenv = - COLUMNS=120 -commands = docker buildx build --load --progress plain -f _debian_10.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py From 50dcd8d62fbcc304940717cd0c99924e68bad509 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 3 Sep 2024 22:31:13 +0200 Subject: [PATCH 287/413] Add fallback option for OS check without hard-coded nameserver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 33 ++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 6b65313e..873d6d7c 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -177,7 +177,7 @@ os_check() { detected_os=$(grep '^ID=' /etc/os-release | cut -d '=' -f2 | tr -d '"') detected_version=$(grep VERSION_ID /etc/os-release | cut -d '=' -f2 | tr -d '"') - # Test via IPv4 + # Test via IPv4 and hardcoded nameserver ns1.pi-hole.net cmdResult="$( dig -4 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1 echo $? @@ -198,7 +198,7 @@ os_check() { fi fi - # Try again via IPv6 + # Try again via IPv6 and hardcoded nameserver ns1.pi-hole.net if [ "$valid_response" = false ]; then unset valid_response @@ -223,6 +223,31 @@ os_check() { fi fi + # Try again without hardcoded nameserver + if [ "$valid_response" = false ]; then + unset valid_response + + cmdResult="$( + dig +short -t txt "${remote_os_domain}" 2>&1 + echo $? + )" + # Gets the return code of the previous command (last line) + digReturnCode="${cmdResult##*$'\n'}" + + if [ ! "${digReturnCode}" == "0" ]; then + valid_response=false + else + # Dig returned 0 (success), so get the actual response, and loop through it to determine if the detected variables above are valid + response="${cmdResult%%$'\n'*}" + # If the value of ${response} is a single 0, then this is the return code, not an actual response. + if [ "${response}" == 0 ]; then + valid_response=false + else + valid_response=true + fi + fi + fi + if [ "$valid_response" = true ]; then IFS=" " read -r -a supportedOS < <(echo "${response}" | tr -d '"') for distro_and_versions in "${supportedOS[@]}"; do @@ -259,8 +284,8 @@ os_check() { printf " %b %bRetrieval of supported OS list failed. %s. %b\\n" "${CROSS}" "${COL_LIGHT_RED}" "${errStr}" "${COL_NC}" printf " %bUnable to determine if the detected OS (%s %s) is supported%b\\n" "${COL_LIGHT_RED}" "${detected_os^}" "${detected_version}" "${COL_NC}" printf " Possible causes for this include:\\n" - printf " - Firewall blocking certain DNS lookups from Pi-hole device\\n" - printf " - ns1.pi-hole.net being blocked (required to obtain TXT record from versions.pi-hole.net containing supported operating systems)\\n" + printf " - Firewall blocking DNS lookups from Pi-hole device to ns1.pi-hole.net\\n" + printf " - DNS resolution issues of the host system\\n" printf " - Other internet connectivity issues\\n" else printf " %b %bUnsupported OS detected: %s %s%b\\n" "${CROSS}" "${COL_LIGHT_RED}" "${detected_os^}" "${detected_version}" "${COL_NC}" From a57d539098aefc390900162f3a5e891dd91eddc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 3 Sep 2024 23:02:15 +0200 Subject: [PATCH 288/413] Remove leftover DNS check via dig as it now only doubles what we already have with getent hosts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/gravity.sh b/gravity.sh index d1664394..6a8918db 100755 --- a/gravity.sh +++ b/gravity.sh @@ -308,18 +308,6 @@ gravity_CheckDNSResolutionAvailable() { exit 1 fi - # If the /etc/resolv.conf contains resolvers other than 127.0.0.1 then the local dnsmasq will not be queried and pi.hole is NXDOMAIN. - # This means that even though name resolution is working, the getent hosts check fails and the holddown timer keeps ticking and eventually fails - # So we check the output of the last command and if it failed, attempt to use dig +short as a fallback - if timeout 4 dig +short "${lookupDomain}" &>/dev/null; then - if [[ -n "${secs:-}" ]]; then - echo -e "${OVER} ${TICK} DNS resolution is now available\\n" - fi - return 0 - elif [[ -n "${secs:-}" ]]; then - echo -e "${OVER} ${CROSS} DNS resolution is not available" - exit 1 - fi # Determine error output message if pgrep pihole-FTL &>/dev/null; then From ba2d10c65eca2fcf205b9d82a2d1901df5f946e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 3 Sep 2024 23:03:59 +0200 Subject: [PATCH 289/413] DNS check in gravity should not check for FTL as it might not be the host's name server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/gravity.sh b/gravity.sh index 6a8918db..7ca695f5 100755 --- a/gravity.sh +++ b/gravity.sh @@ -309,14 +309,6 @@ gravity_CheckDNSResolutionAvailable() { fi - # Determine error output message - if pgrep pihole-FTL &>/dev/null; then - echo -e " ${CROSS} DNS resolution is currently unavailable" - else - echo -e " ${CROSS} DNS service is not running" - "${PIHOLE_COMMAND}" restartdns - fi - # Ensure DNS server is given time to be resolvable secs="120" echo -ne " ${INFO} Time until retry: ${secs}" From f80efa51aac22611ff5d0fa031c80cd7b1287e39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 3 Sep 2024 23:20:00 +0200 Subject: [PATCH 290/413] Wait until DNS resolution is available MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/gravity.sh b/gravity.sh index 7ca695f5..1ac0e282 100755 --- a/gravity.sh +++ b/gravity.sh @@ -298,29 +298,22 @@ gravity_CheckDNSResolutionAvailable() { # Determine if $lookupDomain is resolvable if timeout 4 getent hosts "${lookupDomain}" &>/dev/null; then - # Print confirmation of resolvability if it had previously failed - if [[ -n "${secs:-}" ]]; then - echo -e "${OVER} ${TICK} DNS resolution is now available\\n" - fi + echo -e "${OVER} ${TICK} DNS resolution is available\\n" return 0 - elif [[ -n "${secs:-}" ]]; then - echo -e "${OVER} ${CROSS} DNS resolution is not available" - exit 1 + else + echo -e " ${CROSS} DNS resolution is currently unavailable" fi - - # Ensure DNS server is given time to be resolvable - secs="120" - echo -ne " ${INFO} Time until retry: ${secs}" - until timeout 1 getent hosts "${lookupDomain}" &>/dev/null; do - [[ "${secs:-}" -eq 0 ]] && break - echo -ne "${OVER} ${INFO} Time until retry: ${secs}" - : $((secs--)) + echo -e " ${INFO} Waiting until DNS resolution is available..." + until getent hosts github.com &> /dev/null; do + # Append one dot for each second waiting + str="${str}." + echo -ne " ${OVER} ${INFO} ${str}" sleep 1 done - # Try again - gravity_CheckDNSResolutionAvailable + # If we reach this point, DNS resolution is available + echo -e "${OVER} ${TICK} DNS resolution is available" } # Retrieve blocklist URLs and parse domains from adlist.list From fc73cee156b16c7b8b62594ad10103d3496b5cfe Mon Sep 17 00:00:00 2001 From: SkyLined Date: Mon, 8 Jul 2024 10:01:34 +0200 Subject: [PATCH 291/413] Fix risk of popd without a pushd `pushd` was inside if, `popd` was outside; there was a risk of not doing a `pushd` and only doing a `popd`. Signed-off-by: SkyLined --- automated install/basic-install.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 6b65313e..752e9816 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -378,14 +378,13 @@ is_repo() { pushd "${directory}" &>/dev/null || return 1 # Use git to check if the directory is a repo # git -C is not used here to support git versions older than 1.8.4 - git status --short &>/dev/null || rc=$? - # If the command was not successful, + git status --short &> /dev/null || rc=$? + # Move back into the directory the user started in + popd &> /dev/null || return 1 else # Set a non-zero return code if directory does not exist rc=1 fi - # Move back into the directory the user started in - popd &>/dev/null || return 1 # Return the code; if one is not set, return 0 return "${rc:-0}" } From 333764a7c1e4d8f6eceb7b6ba98525f6e994bd03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 4 Sep 2024 21:56:09 +0200 Subject: [PATCH 292/413] Account for renaming of `devel` branch on `web` repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeCheckout.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index 0514b8eb..84c966df 100755 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -77,13 +77,13 @@ checkout() { if [[ "${1}" == "dev" ]] ; then # Shortcut to check out development branches - echo -e " ${INFO} Shortcut \"${COL_YELLOW}dev${COL_NC}\" detected - checking out development / devel branches..." + echo -e " ${INFO} Shortcut \"${COL_YELLOW}dev${COL_NC}\" detected - checking out development branches..." echo "" echo -e " ${INFO} Pi-hole Core" fetch_checkout_pull_branch "${PI_HOLE_FILES_DIR}" "development" || { echo " ${CROSS} Unable to pull Core development branch"; exit 1; } echo "" echo -e " ${INFO} Web interface" - fetch_checkout_pull_branch "${webInterfaceDir}" "devel" || { echo " ${CROSS} Unable to pull Web development branch"; exit 1; } + fetch_checkout_pull_branch "${webInterfaceDir}" "development" || { echo " ${CROSS} Unable to pull Web development branch"; exit 1; } #echo -e " ${TICK} Pi-hole Core" local path From cffb3117d4680e7b1bd600435c806f5bc2814f1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 4 Sep 2024 22:32:03 +0200 Subject: [PATCH 293/413] Remove lines containing ABP JavaScript rules from adlists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index 1ac0e282..e7cd47cf 100755 --- a/gravity.sh +++ b/gravity.sh @@ -699,15 +699,17 @@ gravity_ParseFileIntoDomains() { # 3) Remove lines starting with ! (ABP Comments) # 4) Remove lines starting with [ (ABP Header) # 5) Remove lines containing ABP extended CSS selectors ("##", "#!#", "#@#", "#?#") preceded by a letter - # 6) Remove comments (text starting with "#", include possible spaces before the hash sign) - # 7) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) - # 8) Remove empty lines + # 6) Remove lines containing ABP JavaScript rules ("#%#") preceded by a letter + # 7) Remove comments (text starting with "#", include possible spaces before the hash sign) + # 8) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) + # 9) Remove empty lines sed -i -r \ -e 's/\r$//' \ -e 's/\s*!.*//g' \ -e 's/\s*\[.*//g' \ -e '/[a-z]\#[$?@]{0,1}\#/d' \ + -e '/[a-z]\#%\#/d' \ -e 's/\s*#.*//g' \ -e 's/^.*\s+//g' \ -e '/^$/d' "${destination}" From e291a9f7e044c0bb317c7f886c72648e8d43a163 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Thu, 5 Sep 2024 13:42:10 +0200 Subject: [PATCH 294/413] It's Adguard not ABP Co-authored-by: Adam Warner Signed-off-by: yubiuser --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index e7cd47cf..7d1358f2 100755 --- a/gravity.sh +++ b/gravity.sh @@ -699,7 +699,7 @@ gravity_ParseFileIntoDomains() { # 3) Remove lines starting with ! (ABP Comments) # 4) Remove lines starting with [ (ABP Header) # 5) Remove lines containing ABP extended CSS selectors ("##", "#!#", "#@#", "#?#") preceded by a letter - # 6) Remove lines containing ABP JavaScript rules ("#%#") preceded by a letter + # 6) Remove lines containing Adguard JavaScript rules ("#%#") preceded by a letter # 7) Remove comments (text starting with "#", include possible spaces before the hash sign) # 8) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) # 9) Remove empty lines From 0ad2a79624a3f2365acabf9fe234e4325aa8e2fe Mon Sep 17 00:00:00 2001 From: yubiuser Date: Fri, 6 Sep 2024 21:20:36 +0200 Subject: [PATCH 295/413] Apply suggestions from code review Co-authored-by: Adam Warner Signed-off-by: yubiuser --- gravity.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index 7d1358f2..dce387f5 100755 --- a/gravity.sh +++ b/gravity.sh @@ -699,7 +699,6 @@ gravity_ParseFileIntoDomains() { # 3) Remove lines starting with ! (ABP Comments) # 4) Remove lines starting with [ (ABP Header) # 5) Remove lines containing ABP extended CSS selectors ("##", "#!#", "#@#", "#?#") preceded by a letter - # 6) Remove lines containing Adguard JavaScript rules ("#%#") preceded by a letter # 7) Remove comments (text starting with "#", include possible spaces before the hash sign) # 8) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) # 9) Remove empty lines @@ -708,8 +707,7 @@ gravity_ParseFileIntoDomains() { -e 's/\r$//' \ -e 's/\s*!.*//g' \ -e 's/\s*\[.*//g' \ - -e '/[a-z]\#[$?@]{0,1}\#/d' \ - -e '/[a-z]\#%\#/d' \ + -e '/[a-z]\#[$?@%]{0,3}\#/d' \ -e 's/\s*#.*//g' \ -e 's/^.*\s+//g' \ -e '/^$/d' "${destination}" From f02162b0212ed1bb9e6cdaf0ae0dd9ae11e0a456 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 6 Sep 2024 21:22:22 +0200 Subject: [PATCH 296/413] Update comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index dce387f5..4f345960 100755 --- a/gravity.sh +++ b/gravity.sh @@ -698,10 +698,10 @@ gravity_ParseFileIntoDomains() { # 2) Remove carriage returns # 3) Remove lines starting with ! (ABP Comments) # 4) Remove lines starting with [ (ABP Header) - # 5) Remove lines containing ABP extended CSS selectors ("##", "#!#", "#@#", "#?#") preceded by a letter - # 7) Remove comments (text starting with "#", include possible spaces before the hash sign) - # 8) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) - # 9) Remove empty lines + # 5) Remove lines containing ABP extended CSS selectors ("##", "#$#", "#@#", "#?#") and Adguard JavaScript (#%#) preceded by a letter + # 6) Remove comments (text starting with "#", include possible spaces before the hash sign) + # 7) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) + # 8) Remove empty lines sed -i -r \ -e 's/\r$//' \ From 129272a69553df0465d58fd354882c1a55bc3694 Mon Sep 17 00:00:00 2001 From: Jack'lul Date: Mon, 9 Sep 2024 18:37:17 +0200 Subject: [PATCH 297/413] Fix wrong message being displayed while waiting for DNS Signed-off-by: Jack'lul --- gravity.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 1ac0e282..7ab3243c 100755 --- a/gravity.sh +++ b/gravity.sh @@ -304,7 +304,8 @@ gravity_CheckDNSResolutionAvailable() { echo -e " ${CROSS} DNS resolution is currently unavailable" fi - echo -e " ${INFO} Waiting until DNS resolution is available..." + str="Waiting until DNS resolution is available..." + echo -ne " ${INFO} ${str}" until getent hosts github.com &> /dev/null; do # Append one dot for each second waiting str="${str}." From d60ad57cac6a37bfe8f2ddb047601730833f52a5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 29 Aug 2024 20:01:06 +0200 Subject: [PATCH 298/413] Add /etc/pihole/dnsmasq.conf to debug log (stripped-down version) Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 7c558127..0a15972c 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -76,6 +76,7 @@ PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml" +PIHOLE_DNSMASQ_CONF_FILE="${PIHOLE_DIRECTORY}/dnsmasq.conf" PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" # Read the value of an FTL config key. The value is printed to stdout. @@ -114,6 +115,7 @@ REQUIRED_FILES=("${PIHOLE_CRON_FILE}" "${PIHOLE_LOCAL_HOSTS_FILE}" "${PIHOLE_LOGROTATE_FILE}" "${PIHOLE_FTL_CONF_FILE}" +"${PIHOLE_DNSMASQ_CONF_FILE}" "${PIHOLE_COMMAND}" "${PIHOLE_COLTABLE_FILE}" "${FTL_PID}" From ce5ef79a3a7717c9f85e449658fc8d39c386fac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 11 Sep 2024 13:31:47 +0200 Subject: [PATCH 299/413] Reduce code duplications and add check via IPv6 without hard-coded nameserver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 92 ++++++++++++++++++------------ 1 file changed, 56 insertions(+), 36 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 873d6d7c..f9cfc94c 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -167,88 +167,108 @@ is_command() { command -v "${check_command}" >/dev/null 2>&1 } +os_check_dig(){ + local protocol="$1" + local domain="$2" + local nameserver="$3" + local response + + response="$(dig "${protocol}" +short -t txt "${domain}" "${nameserver}" 2>&1 + echo $? + )" + echo "${response}" +} + +os_check_dig_response(){ + # Checks the reply from the dig command to determine if it's a valid response + local digReply="$1" + local response + + # Dig returned 0 (success), so get the actual response, and loop through it to determine if the detected variables above are valid + response="${digReply%%$'\n'*}" + # If the value of ${response} is a single 0, then this is the return code, not an actual response. + if [ "${response}" == 0 ]; then + echo false + else + echo true + fi +} + os_check() { if [ "$PIHOLE_SKIP_OS_CHECK" != true ]; then # This function gets a list of supported OS versions from a TXT record at versions.pi-hole.net # and determines whether or not the script is running on one of those systems local remote_os_domain valid_os valid_version valid_response detected_os detected_version display_warning cmdResult digReturnCode response + local piholeNameserver="@ns1.pi-hole.net" remote_os_domain=${OS_CHECK_DOMAIN_NAME:-"versions.pi-hole.net"} detected_os=$(grep '^ID=' /etc/os-release | cut -d '=' -f2 | tr -d '"') detected_version=$(grep VERSION_ID /etc/os-release | cut -d '=' -f2 | tr -d '"') # Test via IPv4 and hardcoded nameserver ns1.pi-hole.net - cmdResult="$( - dig -4 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1 - echo $? - )" + cmdResult=$(os_check_dig 4 "${remote_os_domain}" "${piholeNameserver}") + # Gets the return code of the previous command (last line) digReturnCode="${cmdResult##*$'\n'}" if [ ! "${digReturnCode}" == "0" ]; then valid_response=false else - # Dig returned 0 (success), so get the actual response, and loop through it to determine if the detected variables above are valid - response="${cmdResult%%$'\n'*}" - # If the value of ${response} is a single 0, then this is the return code, not an actual response. - if [ "${response}" == 0 ]; then - valid_response=false - else - valid_response=true - fi + valid_response=$(os_check_dig_response cmdResult) fi # Try again via IPv6 and hardcoded nameserver ns1.pi-hole.net if [ "$valid_response" = false ]; then unset valid_response + unset cmdResult + unset digReturnCode - cmdResult="$( - dig -6 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1 - echo $? - )" + cmdResult=$(os_check_dig 6 "${remote_os_domain}" "${piholeNameserver}") # Gets the return code of the previous command (last line) digReturnCode="${cmdResult##*$'\n'}" if [ ! "${digReturnCode}" == "0" ]; then valid_response=false else - # Dig returned 0 (success), so get the actual response, and loop through it to determine if the detected variables above are valid - response="${cmdResult%%$'\n'*}" - # If the value of ${response} is a single 0, then this is the return code, not an actual response. - if [ "${response}" == 0 ]; then - valid_response=false - else - valid_response=true - fi + valid_response=$(os_check_dig_response cmdResult) fi fi # Try again without hardcoded nameserver if [ "$valid_response" = false ]; then unset valid_response + unset cmdResult + unset digReturnCode - cmdResult="$( - dig +short -t txt "${remote_os_domain}" 2>&1 - echo $? - )" + cmdResult=$(os_check_dig 4 "${remote_os_domain}") # Gets the return code of the previous command (last line) digReturnCode="${cmdResult##*$'\n'}" if [ ! "${digReturnCode}" == "0" ]; then valid_response=false else - # Dig returned 0 (success), so get the actual response, and loop through it to determine if the detected variables above are valid - response="${cmdResult%%$'\n'*}" - # If the value of ${response} is a single 0, then this is the return code, not an actual response. - if [ "${response}" == 0 ]; then - valid_response=false - else - valid_response=true - fi + valid_response=$(os_check_dig_response cmdResult) + fi + fi + + if [ "$valid_response" = false ]; then + unset valid_response + unset cmdResult + unset digReturnCode + + cmdResult=$(os_check_dig 6 "${remote_os_domain}") + # Gets the return code of the previous command (last line) + digReturnCode="${cmdResult##*$'\n'}" + + if [ ! "${digReturnCode}" == "0" ]; then + valid_response=false + else + valid_response=$(os_check_dig_response cmdResult) fi fi if [ "$valid_response" = true ]; then + response="${cmdResult%%$'\n'*}" IFS=" " read -r -a supportedOS < <(echo "${response}" | tr -d '"') for distro_and_versions in "${supportedOS[@]}"; do distro_part="${distro_and_versions%%=*}" From 62ef2d17772673713e483f12a7a291bec447a2bc Mon Sep 17 00:00:00 2001 From: yubiuser Date: Wed, 11 Sep 2024 18:44:17 +0200 Subject: [PATCH 300/413] Add forgetting dash Co-authored-by: RD WebDesign Signed-off-by: yubiuser --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f9cfc94c..18591773 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -173,7 +173,7 @@ os_check_dig(){ local nameserver="$3" local response - response="$(dig "${protocol}" +short -t txt "${domain}" "${nameserver}" 2>&1 + response="$(dig -"${protocol}" +short -t txt "${domain}" "${nameserver}" 2>&1 echo $? )" echo "${response}" From 251f3295f3b64a81c86ef1e18ce92f91f8658f5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 12 Sep 2024 21:32:36 +0200 Subject: [PATCH 301/413] Return early during v6 migration if migration dir exists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 752e9816..a123e8d9 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2146,8 +2146,8 @@ migrate_dnsmasq_configs() { # avoid conflicts with other services on this system # Exit early if this is already Pi-hole v6.0 - # We decide this on the presence of the file /etc/pihole/pihole.toml - if [[ -f /etc/pihole/pihole.toml ]]; then + # We decide this on the presence of the migration dir + if [[ -d "${V6_CONF_MIGRATION_DIR}" ]]; then return 0 fi From 68888979999657d9a0eb57bb319b1a415b6819b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 13 Sep 2024 21:15:25 +0200 Subject: [PATCH 302/413] Revert "Return early during v6 migration if migration dir exists" This reverts commit 251f3295f3b64a81c86ef1e18ce92f91f8658f5e. --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a123e8d9..752e9816 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2146,8 +2146,8 @@ migrate_dnsmasq_configs() { # avoid conflicts with other services on this system # Exit early if this is already Pi-hole v6.0 - # We decide this on the presence of the migration dir - if [[ -d "${V6_CONF_MIGRATION_DIR}" ]]; then + # We decide this on the presence of the file /etc/pihole/pihole.toml + if [[ -f /etc/pihole/pihole.toml ]]; then return 0 fi From 20d20e116c855b8d618caec1f94d8461f8157f36 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 14 Sep 2024 10:25:32 +0000 Subject: [PATCH 303/413] Bump tox from 4.18.0 to 4.18.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.18.0 to 4.18.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.18.0...4.18.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 4bc7c372..2c417afb 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.2 pytest == 8.3.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.18.0 +tox == 4.18.1 pytest-clarity == 1.0.1 From 05707c624d5e2b2e746ff9f32db8180b26ab3300 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 14 Sep 2024 10:36:07 +0000 Subject: [PATCH 304/413] Bump pytest from 8.3.2 to 8.3.3 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.3.2 to 8.3.3. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.3.2...8.3.3) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 2c417afb..4b9882b7 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.2 -pytest == 8.3.2 +pytest == 8.3.3 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 tox == 4.18.1 From 73301da68f0243717424cc3de7ffa6a0655168ce Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 1 Jan 2024 08:42:31 +0100 Subject: [PATCH 305/413] Optimize the database by running ANALYZE after gravity finished Signed-off-by: DL6ER --- gravity.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/gravity.sh b/gravity.sh index 7ab3243c..1a01946a 100755 --- a/gravity.sh +++ b/gravity.sh @@ -837,6 +837,24 @@ database_recovery() { echo "" } +gravity_optimize() { + # The ANALYZE command gathers statistics about tables and indices and stores + # the collected information in internal tables of the database where the + # query optimizer can access the information and use it to help make better + # query planning choices + local str="Optimizing database" + echo -ne " ${INFO} ${str}..." + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "PRAGMA analysis_limit=0; ANALYZE" 2>&1; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to optimize database ${gravityTEMPfile}\\n ${output}" + gravity_Cleanup "error" + else + echo -e "${OVER} ${TICK} ${str}" + fi +} + helpFunc() { echo "Usage: pihole -g Update domains from blocklists specified in adlists.list @@ -949,6 +967,9 @@ gravity_build_tree # numbers quickly from the tree instead of having to scan the whole database) gravity_ShowCount +# Optimize the database +gravity_optimize + # Migrate rest of the data from old to new database # IMPORTANT: Swapping the databases must be the last step before the cleanup if ! gravity_swap_databases; then From 111dfc63ff1d948de424ca31f1cb551dbe5310ac Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 17 Sep 2024 20:23:09 +0200 Subject: [PATCH 306/413] Add new option allowing timing the individual parts of gravity Signed-off-by: DL6ER --- gravity.sh | 65 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/gravity.sh b/gravity.sh index 1a01946a..05f3fb0c 100755 --- a/gravity.sh +++ b/gravity.sh @@ -428,7 +428,7 @@ gravity_DownloadBlocklists() { if [[ "${check_url}" =~ ${regex} ]]; then echo -e " ${CROSS} Invalid Target" else - gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" "${adlist_type}" "${domain}" + timeit gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" "${adlist_type}" "${domain}" fi echo "" done @@ -855,12 +855,58 @@ gravity_optimize() { fi } +# Function: timeit +# Description: Measures the execution time of a given command. +# +# Usage: +# timeit +# +# Parameters: +# - The command to be executed and timed. +# +# Returns: +# The exit status of the executed command. +# +# Output: +# If the 'timed' variable is set to true, prints the elapsed time in seconds +# with millisecond precision. +# +# Example: +# timeit ls -l +# +timeit(){ + local start_time end_time elapsed_time ret + + # Capture the start time + start_time=$(date +%s%3N) + + # Execute the command passed as arguments + "$@" + ret=$? + + if [[ "${timed:-}" != true ]]; then + return $ret + fi + + # Capture the end time + end_time=$(date +%s%3N) + + # Calculate the elapsed time + elapsed_time=$((end_time - start_time)) + + # Display the elapsed time + printf " --> took %d.%03d seconds\n" $((elapsed_time / 1000)) $((elapsed_time % 1000)) + + return $ret +} + helpFunc() { echo "Usage: pihole -g Update domains from blocklists specified in adlists.list Options: -f, --force Force the download of all specified blocklists + -t, --timeit Time the gravity update process -h, --help Show this help dialog" exit 0 } @@ -897,6 +943,7 @@ Available options: for var in "$@"; do case "${var}" in "-f" | "--force") forceDelete=true ;; + "-t" | "--timeit") timed=true ;; "-r" | "--repair") repairSelector "$3" ;; "-u" | "--upgrade") upgrade_gravityDB "${gravityDBfile}" "${piholeDir}" @@ -925,11 +972,11 @@ if [[ "${recreate_database:-}" == true ]]; then fi if [[ "${recover_database:-}" == true ]]; then - database_recovery "$4" + timeit database_recovery "$4" fi # Move possibly existing legacy files to the gravity database -if ! migrate_to_database; then +if ! timeit migrate_to_database; then echo -e " ${CROSS} Unable to migrate to database. Please contact support." exit 1 fi @@ -943,7 +990,7 @@ if [[ "${forceDelete:-}" == true ]]; then fi # Gravity downloads blocklists next -if ! gravity_CheckDNSResolutionAvailable; then +if ! timeit gravity_CheckDNSResolutionAvailable; then echo -e " ${CROSS} Can not complete gravity update, no DNS is available. Please contact support." exit 1 fi @@ -961,23 +1008,23 @@ chown pihole:pihole "${gravityTEMPfile}" chmod g+w "${piholeDir}" "${gravityTEMPfile}" # Build the tree -gravity_build_tree +timeit gravity_build_tree # Compute numbers to be displayed (do this after building the tree to get the # numbers quickly from the tree instead of having to scan the whole database) -gravity_ShowCount +timeit gravity_ShowCount # Optimize the database -gravity_optimize +timeit gravity_optimize # Migrate rest of the data from old to new database # IMPORTANT: Swapping the databases must be the last step before the cleanup -if ! gravity_swap_databases; then +if ! timeit gravity_swap_databases; then echo -e " ${CROSS} Unable to create database. Please contact support." exit 1 fi -gravity_Cleanup +timeit gravity_Cleanup echo "" echo " ${TICK} Done." From 0f4ad7734c51ca3cde692d0cef759c95c5db36a8 Mon Sep 17 00:00:00 2001 From: Dominik Date: Wed, 18 Sep 2024 16:13:35 -0400 Subject: [PATCH 307/413] Update gravity.sh to use color with -t Co-authored-by: RD WebDesign Signed-off-by: Dominik --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 05f3fb0c..2605baf1 100755 --- a/gravity.sh +++ b/gravity.sh @@ -895,7 +895,7 @@ timeit(){ elapsed_time=$((end_time - start_time)) # Display the elapsed time - printf " --> took %d.%03d seconds\n" $((elapsed_time / 1000)) $((elapsed_time % 1000)) + printf " %b--> took %d.%03d seconds%b\n" ${COL_BLUE} $((elapsed_time / 1000)) $((elapsed_time % 1000)) ${COL_NC} return $ret } From 885895e9943de5da79116aaf2839bd82c7117723 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 21 Sep 2024 10:10:37 +0000 Subject: [PATCH 308/413] Bump tox from 4.18.1 to 4.20.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.18.1 to 4.20.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.18.1...4.20.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 4b9882b7..8f96a644 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.2 pytest == 8.3.3 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.18.1 +tox == 4.20.0 pytest-clarity == 1.0.1 From cc476a49fef3291ff0849139a0c423ba488ef77d Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 24 Sep 2024 14:15:15 +0100 Subject: [PATCH 309/413] remove the restartdns functionality and promote the reloaddns functions Signed-off-by: Adam Warner --- advanced/bash-completion/pihole | 2 +- manpages/pihole.8 | 10 ++++------ pihole | 18 ++++++------------ 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 674eeea4..822ec288 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -7,7 +7,7 @@ _pihole() { case "${prev}" in "pihole") - opts="allow allow-regex allow-wild deny checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard arpflush api" + opts="allow allow-regex allow-wild deny checkout debug disable enable flush help logging query reconfigure regex reloaddns reloaddns-lists status tail uninstall updateGravity updatePihole version wildcard arpflush api" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) ;; "allow"|"deny"|"wildcard"|"regex"|"allow-regex"|"allow-wild") diff --git a/manpages/pihole.8 b/manpages/pihole.8 index d8b1a7fe..3da23ffe 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -236,16 +236,14 @@ Available commands and options: #m Disable Pi-hole functionality for # minute(s) .br -\fBrestartdns\fR [options] +\fBreloaddns\fR .br - Full restart Pi-hole subsystems. Without any options (see below) a full restart causes config file parsing and history re-reading + Update the lists and flush the cache without restarting the DNS server .br - (restart options): +\fBreloaddns-lists\fR .br - reload Updates the lists (incl. HOSTS files) and flushes DNS cache. Does not reparse config files -.br - reload-lists Updates the lists (excl. HOSTS files) WITHOUT flushing the DNS cache. Does not reparse config files + Update the lists WITHOUT flushing the cache or restarting the DNS server .br \fBcheckout\fR [repo] [branch] diff --git a/pihole b/pihole index 98f478a3..3b98d2a9 100755 --- a/pihole +++ b/pihole @@ -147,7 +147,7 @@ versionFunc() { exec "${PI_HOLE_SCRIPT_DIR}"/version.sh } -restartDNS() { +reloadDNS() { local svcOption svc str output status pid icon FTL_PID_FILE svcOption="${1:-restart}" @@ -184,11 +184,6 @@ restartDNS() { str="Flushing DNS cache" icon="${TICK}" fi - else - # A full restart has been requested - svc="service pihole-FTL restart" - str="Restarting DNS server" - icon="${TICK}" fi # Print output to Terminal, but not to Web Admin @@ -509,9 +504,8 @@ Options: enable Enable Pi-hole subsystems disable Disable Pi-hole subsystems Add '-h' for more info on disable usage - restartdns Full restart Pi-hole subsystems - Add 'reload' to update the lists and flush the cache without restarting the DNS server - Add 'reload-lists' to only update the lists WITHOUT flushing the cache or restarting the DNS server + reloaddns Update the lists and flush the cache without restarting the DNS server + reloaddns-lists Update the lists WITHOUT flushing the cache or restarting the DNS server checkout Switch Pi-hole subsystems to a different GitHub branch Add '-h' for more info on checkout usage arpflush Flush information stored in Pi-hole's network tables"; @@ -547,9 +541,9 @@ case "${1}" in "enable" ) need_root=0;; "disable" ) need_root=0;; "-d" | "debug" ) ;; - "restartdns" ) ;; "-g" | "updateGravity" ) ;; "reloaddns" ) ;; + "reloaddns-lists" ) ;; "setpassword" ) ;; "checkout" ) ;; "updatechecker" ) ;; @@ -590,8 +584,8 @@ case "${1}" in "uninstall" ) uninstallFunc;; "enable" ) piholeEnable true "$2";; "disable" ) piholeEnable false "$2";; - "restartdns" ) restartDNS "$2";; - "reloaddns" ) restartDNS "reload";; + "reloaddns" ) reloadDNS "reload";; + "reloaddns-lists" ) reloadDNS "reload-lists";; "setpassword" ) SetWebPassword "$@";; "checkout" ) piholeCheckoutFunc "$@";; "updatechecker" ) shift; updateCheckFunc "$@";; From 5f1e4680f7e07a2a721367d2c8daf21b25af5e4e Mon Sep 17 00:00:00 2001 From: Wouter Servaes Date: Tue, 10 Sep 2024 21:36:12 +0200 Subject: [PATCH 310/413] Grouped common dependencies of distros Signed-off-by: Wouter Servaes --- automated install/basic-install.sh | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 752e9816..b2803651 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -316,8 +316,10 @@ test_dpkg_lock() { # Compatibility package_manager_detect() { - # TODO - pull common packages for both distributions out into a common variable, then add - # the distro-specific ones below. + # pull common packages for both distributions out into a common variable + OS_CHECK_COMMON_DEPS=(grep) + PIHOLE_COMMON_DEPS=(curl psmisc sudo unzip jq); + INSTALLER_COMMON_DEPS=(git dialog ca-certificates) # First check to see if apt-get is installed. if is_command apt-get; then @@ -333,11 +335,11 @@ package_manager_detect() { # Update package cache update_package_cache || exit 1 # Packages required to perform the os_check and FTL binary detection - OS_CHECK_DEPS=(grep dnsutils binutils) + OS_CHECK_DEPS=(dnsutils binutils) # Packages required to run this install script - INSTALLER_DEPS=(git iproute2 dialog ca-certificates) + INSTALLER_DEPS=(iproute2) # Packages required to run Pi-hole - PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq lshw bash-completion) + PIHOLE_DEPS=(cron iputils-ping libcap2-bin dns-root-data libcap2 netcat-openbsd procps lshw bash-completion) # If apt-get is not found, check for rpm. elif is_command rpm; then @@ -352,9 +354,9 @@ package_manager_detect() { PKG_INSTALL=("${PKG_MANAGER}" install -y) # CentOS package manager returns 100 when there are packages to update so we need to || true to prevent the script from exiting. PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src|.riscv64)' | wc -l || true" - OS_CHECK_DEPS=(grep bind-utils) - INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates binutils) - PIHOLE_DEPS=(cronie curl findutils sudo unzip psmisc libcap nmap-ncat jq lshw bash-completion) + OS_CHECK_DEPS=(bind-utils) + INSTALLER_DEPS=(iproute newt procps-ng chkconfig binutils) + PIHOLE_DEPS=(cronie findutils libcap nmap-ncat lshw bash-completion) # If neither apt-get or yum/dnf package managers were found else @@ -2223,7 +2225,7 @@ main() { # Install packages necessary to perform os_check printf " %b Checking for / installing Required dependencies for OS Check...\\n" "${INFO}" - install_dependent_packages "${OS_CHECK_DEPS[@]}" + install_dependent_packages "${OS_CHECK_COMMON_DEPS[@]}" "${OS_CHECK_DEPS[@]}" # Check that the installed OS is officially supported - display warning if not os_check @@ -2240,7 +2242,7 @@ main() { # Install packages used by this installation script printf " %b Checking for / installing Required dependencies for this install script...\\n" "${INFO}" - install_dependent_packages "${INSTALLER_DEPS[@]}" + install_dependent_packages "${INSTALLER_COMMON_DEPS[@]}" "${INSTALLER_DEPS[@]}" # if it's running unattended, if [[ "${runUnattended}" == true ]]; then @@ -2281,7 +2283,7 @@ main() { clone_or_update_repos # Install the Core dependencies - local dep_install_list=("${PIHOLE_DEPS[@]}") + local dep_install_list=("${PIHOLE_COMMON_DEPS[@]}" "${PIHOLE_DEPS[@]}") # Install packages used by the actual software printf " %b Checking for / installing Required dependencies for Pi-hole software...\\n" "${INFO}" From e9fdfac569e7a4476456cd66514752b4bff64977 Mon Sep 17 00:00:00 2001 From: Wouter Servaes Date: Wed, 11 Sep 2024 18:10:09 +0200 Subject: [PATCH 311/413] Added common dependencies to uninstall script for removal Signed-off-by: Wouter Servaes --- automated install/uninstall.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 8cffae20..ac06da73 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -45,7 +45,7 @@ source "${setupVars}" package_manager_detect # Uninstall packages used by the Pi-hole -DEPS=("${INSTALLER_DEPS[@]}" "${PIHOLE_DEPS[@]}" "${OS_CHECK_DEPS[@]}") +DEPS=("${INSTALLER_COMMON_DEPS[@]}" "${PIHOLE_COMMON_DEPS[@]}" "${OS_CHECK_COMMON_DEPS[@]}" "${INSTALLER_DEPS[@]}" "${PIHOLE_DEPS[@]}" "${OS_CHECK_DEPS[@]}") # Compatibility if [ -x "$(command -v apt-get)" ]; then From 8c56572d0b2eb82bd4854fa688287bcdd6b5df81 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 28 Sep 2024 10:12:44 +0000 Subject: [PATCH 312/413] Bump actions/checkout from 4.1.7 to 4.2.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.7 to 4.2.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.7...v4.2.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 0a5b59b9..44c7ff5a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.7 + uses: actions/checkout@v4.2.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 2a7831af..d2f552a2 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.7 + uses: actions/checkout@v4.2.0 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index fe24c9b5..067d4f2a 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.7 + uses: actions/checkout@v4.2.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0eef25f0..34a12b74 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.7 + uses: actions/checkout@v4.2.0 - name: Check scripts in repository are executable run: | @@ -70,7 +70,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.7 + uses: actions/checkout@v4.2.0 - name: Set up Python 3.10 uses: actions/setup-python@v5.2.0 From bcc3a7e35e9369f32b60a9c8abcf7187169a5880 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 30 Sep 2024 21:04:55 +0200 Subject: [PATCH 313/413] Show version information after an web only update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/update.sh | 6 ++++++ automated install/basic-install.sh | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index 8a35ef2e..2ccad27c 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -221,6 +221,12 @@ main() { echo -e " ${INFO} Local version file information updated." fi + # if there was only a web update, show the new versions + # (on core and FTL updates, this is done as part of the installer run) + if [[ "${web_update}" == true && "${FTL_update}" == false && "${core_update}" == false ]]; then + "${PI_HOLE_BIN_DIR}"/pihole version + fi + echo "" exit 0 } diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9428b05a..b948b8a5 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2434,7 +2434,7 @@ main() { if [[ "${INSTALL_TYPE}" == "Update" ]]; then printf "\\n" - "${PI_HOLE_BIN_DIR}"/pihole version --current + "${PI_HOLE_BIN_DIR}"/pihole version fi } From ada7d2bd31d36e5266d0a28e3773bccb13aac844 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 30 Sep 2024 21:10:33 +0200 Subject: [PATCH 314/413] Fix removing old man page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9428b05a..ca81b6be 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1230,8 +1230,8 @@ install_manpage() { if [[ -f "/usr/local/share/man/man5/pihole-FTL.conf.5" ]]; then rm /usr/local/share/man/man5/pihole-FTL.conf.5 fi - if [[ -f "/usr/local/share/man/man5/pihole-FTL.8" ]]; then - rm /usr/local/share/man/man5/pihole-FTL.8 + if [[ -f "/usr/local/share/man/man8/pihole-FTL.8" ]]; then + rm /usr/local/share/man/man8/pihole-FTL.8 fi if mandb -q &>/dev/null; then From 74e52675f187e750bf200fdbd24f511714b2c590 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 2 Oct 2024 13:47:23 +0200 Subject: [PATCH 315/413] Use pihole.toml to decide if installer runs on an update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4598f88e..4852929c 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -77,6 +77,7 @@ PI_HOLE_FILES=(list piholeDebug piholeLogFlush setupLCD update version gravity u PI_HOLE_INSTALL_DIR="/opt/pihole" PI_HOLE_CONFIG_DIR="/etc/pihole" PI_HOLE_BIN_DIR="/usr/local/bin" +PI_HOLE_V6_CONFIG="${PI_HOLE_CONFIG_DIR}/pihole.toml" if [ -z "$useUpdateVars" ]; then useUpdateVars=false fi @@ -2194,7 +2195,7 @@ migrate_dnsmasq_configs() { # Exit early if this is already Pi-hole v6.0 # We decide this on the presence of the file /etc/pihole/pihole.toml - if [[ -f /etc/pihole/pihole.toml ]]; then + if [[ -f "${PI_HOLE_V6_CONFIG}" ]]; then return 0 fi @@ -2289,16 +2290,19 @@ main() { printf " %b Checking for / installing Required dependencies for this install script...\\n" "${INFO}" install_dependent_packages "${INSTALLER_COMMON_DEPS[@]}" "${INSTALLER_DEPS[@]}" - # if it's running unattended, - if [[ "${runUnattended}" == true ]]; then - printf " %b Performing unattended setup, no dialogs will be displayed\\n" "${INFO}" - # Use the setup variables - useUpdateVars=true - # also disable debconf-apt-progress dialogs - export DEBIAN_FRONTEND="noninteractive" - else - # If running attended, show the available options (repair/reconfigure) - update_dialogs + # in case of an update + if [[ -f "${PI_HOLE_V6_CONFIG}" ]]; then + # if it's running unattended, + if [[ "${runUnattended}" == true ]]; then + printf " %b Performing unattended setup, no dialogs will be displayed\\n" "${INFO}" + # Use the setup variables + useUpdateVars=true + # also disable debconf-apt-progress dialogs + export DEBIAN_FRONTEND="noninteractive" + else + # If running attended, show the available options (repair/reconfigure) + update_dialogs + fi fi if [[ "${useUpdateVars}" == false ]]; then From be00f3e7235d5513dce6517b9e9cd92a79bb3f42 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 2 Oct 2024 19:31:39 +0100 Subject: [PATCH 316/413] Rename `reloaddns-lists` to `reloadlists` per suggestion Co-authored-by: RD WebDesign Signed-off-by: Adam Warner --- advanced/bash-completion/pihole | 2 +- manpages/pihole.8 | 2 +- pihole | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 822ec288..4343cf92 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -7,7 +7,7 @@ _pihole() { case "${prev}" in "pihole") - opts="allow allow-regex allow-wild deny checkout debug disable enable flush help logging query reconfigure regex reloaddns reloaddns-lists status tail uninstall updateGravity updatePihole version wildcard arpflush api" + opts="allow allow-regex allow-wild deny checkout debug disable enable flush help logging query reconfigure regex reloaddns reloadlists status tail uninstall updateGravity updatePihole version wildcard arpflush api" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) ;; "allow"|"deny"|"wildcard"|"regex"|"allow-regex"|"allow-wild") diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 3da23ffe..5964c691 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -241,7 +241,7 @@ Available commands and options: Update the lists and flush the cache without restarting the DNS server .br -\fBreloaddns-lists\fR +\fBreloadlists\fR .br Update the lists WITHOUT flushing the cache or restarting the DNS server .br diff --git a/pihole b/pihole index 3b98d2a9..df8225ab 100755 --- a/pihole +++ b/pihole @@ -505,7 +505,7 @@ Options: disable Disable Pi-hole subsystems Add '-h' for more info on disable usage reloaddns Update the lists and flush the cache without restarting the DNS server - reloaddns-lists Update the lists WITHOUT flushing the cache or restarting the DNS server + reloadlists Update the lists WITHOUT flushing the cache or restarting the DNS server checkout Switch Pi-hole subsystems to a different GitHub branch Add '-h' for more info on checkout usage arpflush Flush information stored in Pi-hole's network tables"; @@ -543,7 +543,7 @@ case "${1}" in "-d" | "debug" ) ;; "-g" | "updateGravity" ) ;; "reloaddns" ) ;; - "reloaddns-lists" ) ;; + "reloadlists" ) ;; "setpassword" ) ;; "checkout" ) ;; "updatechecker" ) ;; @@ -585,7 +585,7 @@ case "${1}" in "enable" ) piholeEnable true "$2";; "disable" ) piholeEnable false "$2";; "reloaddns" ) reloadDNS "reload";; - "reloaddns-lists" ) reloadDNS "reload-lists";; + "reloadlists" ) reloadDNS "reload-lists";; "setpassword" ) SetWebPassword "$@";; "checkout" ) piholeCheckoutFunc "$@";; "updatechecker" ) shift; updateCheckFunc "$@";; From 84a659dfac0c610a0f24a4bea9357eac8b3d9f3a Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 2 Oct 2024 19:41:17 +0100 Subject: [PATCH 317/413] Update pihole Co-authored-by: yubiuser Signed-off-by: Adam Warner --- pihole | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pihole b/pihole index df8225ab..bbef74ed 100755 --- a/pihole +++ b/pihole @@ -149,7 +149,7 @@ versionFunc() { reloadDNS() { local svcOption svc str output status pid icon FTL_PID_FILE - svcOption="${1:-restart}" + svcOption="${1:-reload}" # get the current path to the pihole-FTL.pid FTL_PID_FILE="$(getFTLConfigValue files.pid)" From 6645136f6b309ff22586a976a66e3f6cdf46971d Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 2 Oct 2024 21:19:39 +0100 Subject: [PATCH 318/413] Align the indentation for the case statement in the bottom part of pihole script Signed-off-by: Adam Warner --- pihole | 102 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/pihole b/pihole index bbef74ed..93ebfa56 100755 --- a/pihole +++ b/pihole @@ -505,7 +505,7 @@ Options: disable Disable Pi-hole subsystems Add '-h' for more info on disable usage reloaddns Update the lists and flush the cache without restarting the DNS server - reloadlists Update the lists WITHOUT flushing the cache or restarting the DNS server + reloadlists Update the lists WITHOUT flushing the cache or restarting the DNS server checkout Switch Pi-hole subsystems to a different GitHub branch Add '-h' for more info on checkout usage arpflush Flush information stored in Pi-hole's network tables"; @@ -519,38 +519,38 @@ fi # functions that do not require sudo power need_root=1 case "${1}" in - "-h" | "help" | "--help" ) helpFunc;; - "-v" | "version" ) versionFunc;; - "-c" | "chronometer" ) chronometerFunc "$@";; - "-q" | "query" ) queryFunc "$@";; - "status" ) statusFunc "$2";; - "tricorder" ) tricorderFunc;; + "-h" | "help" | "--help" ) helpFunc;; + "-v" | "version" ) versionFunc;; + "-c" | "chronometer" ) chronometerFunc "$@";; + "-q" | "query" ) queryFunc "$@";; + "status" ) statusFunc "$2";; + "tricorder" ) tricorderFunc;; # we need to add all arguments that require sudo power to not trigger the * argument - "allow" | "allowlist" ) need_root=0;; - "deny" | "denylist" ) need_root=0;; - "--wild" | "wildcard" ) need_root=0;; - "--regex" | "regex" ) need_root=0;; + "allow" | "allowlist" ) need_root=0;; + "deny" | "denylist" ) need_root=0;; + "--wild" | "wildcard" ) need_root=0;; + "--regex" | "regex" ) need_root=0;; "--allow-regex" | "allow-regex" ) need_root=0;; "--allow-wild" | "allow-wild" ) need_root=0;; - "-f" | "flush" ) ;; - "-up" | "updatePihole" ) ;; - "-r" | "reconfigure" ) ;; - "-l" | "logging" ) ;; - "uninstall" ) ;; - "enable" ) need_root=0;; - "disable" ) need_root=0;; - "-d" | "debug" ) ;; - "-g" | "updateGravity" ) ;; - "reloaddns" ) ;; - "reloadlists" ) ;; - "setpassword" ) ;; - "checkout" ) ;; - "updatechecker" ) ;; - "arpflush" ) ;; - "-t" | "tail" ) ;; - "api" ) need_root=0;; - * ) helpFunc;; + "-f" | "flush" ) ;; + "-up" | "updatePihole" ) ;; + "-r" | "reconfigure" ) ;; + "-l" | "logging" ) ;; + "uninstall" ) ;; + "enable" ) need_root=0;; + "disable" ) need_root=0;; + "-d" | "debug" ) ;; + "-g" | "updateGravity" ) ;; + "reloaddns" ) ;; + "reloadlists" ) ;; + "setpassword" ) ;; + "checkout" ) ;; + "updatechecker" ) ;; + "arpflush" ) ;; + "-t" | "tail" ) ;; + "api" ) need_root=0;; + * ) helpFunc;; esac # In the case of alpine running in a container, the USER variable appears to be blank @@ -569,28 +569,28 @@ fi # Handle redirecting to specific functions based on arguments case "${1}" in - "allow" | "allowlist" ) listFunc "$@";; - "deny" | "denylist" ) listFunc "$@";; - "--wild" | "wildcard" ) listFunc "$@";; - "--regex" | "regex" ) listFunc "$@";; + "allow" | "allowlist" ) listFunc "$@";; + "deny" | "denylist" ) listFunc "$@";; + "--wild" | "wildcard" ) listFunc "$@";; + "--regex" | "regex" ) listFunc "$@";; "--allow-regex" | "allow-regex" ) listFunc "$@";; "--allow-wild" | "allow-wild" ) listFunc "$@";; - "-d" | "debug" ) debugFunc "$@";; - "-f" | "flush" ) flushFunc "$@";; - "-up" | "updatePihole" ) updatePiholeFunc "$@";; - "-r" | "reconfigure" ) reconfigurePiholeFunc;; - "-g" | "updateGravity" ) updateGravityFunc "$@";; - "-l" | "logging" ) piholeLogging "$@";; - "uninstall" ) uninstallFunc;; - "enable" ) piholeEnable true "$2";; - "disable" ) piholeEnable false "$2";; - "reloaddns" ) reloadDNS "reload";; - "reloadlists" ) reloadDNS "reload-lists";; - "setpassword" ) SetWebPassword "$@";; - "checkout" ) piholeCheckoutFunc "$@";; - "updatechecker" ) shift; updateCheckFunc "$@";; - "arpflush" ) arpFunc "$@";; - "-t" | "tail" ) tailFunc "$2";; - "api" ) apiFunc "$2";; - * ) helpFunc;; + "-d" | "debug" ) debugFunc "$@";; + "-f" | "flush" ) flushFunc "$@";; + "-up" | "updatePihole" ) updatePiholeFunc "$@";; + "-r" | "reconfigure" ) reconfigurePiholeFunc;; + "-g" | "updateGravity" ) updateGravityFunc "$@";; + "-l" | "logging" ) piholeLogging "$@";; + "uninstall" ) uninstallFunc;; + "enable" ) piholeEnable true "$2";; + "disable" ) piholeEnable false "$2";; + "reloaddns" ) reloadDNS "reload";; + "reloadlists" ) reloadDNS "reload-lists";; + "setpassword" ) SetWebPassword "$@";; + "checkout" ) piholeCheckoutFunc "$@";; + "updatechecker" ) shift; updateCheckFunc "$@";; + "arpflush" ) arpFunc "$@";; + "-t" | "tail" ) tailFunc "$2";; + "api" ) apiFunc "$2";; + * ) helpFunc;; esac From afdf45031f363ea9ed6fe7f295372b1bd8241965 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 2 Oct 2024 21:51:56 +0100 Subject: [PATCH 319/413] Missed a couple of references in the previous PR Signed-off-by: Adam Warner --- gravity.sh | 6 ------ manpages/pihole.8 | 4 +++- pihole | 1 - 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/gravity.sh b/gravity.sh index 3c89c1c0..6a4fb88e 100755 --- a/gravity.sh +++ b/gravity.sh @@ -777,12 +777,6 @@ gravity_Cleanup() { echo -e "${OVER} ${TICK} ${str}" - # # Only restart DNS service if offline - # if ! pgrep pihole-FTL &> /dev/null; then - # "${PIHOLE_COMMAND}" restartdns - # dnsWasOffline=true - # fi - # Print Pi-hole status if an error occurred if [[ -n "${error}" ]]; then "${PIHOLE_COMMAND}" status diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 5964c691..825a1fd1 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -39,7 +39,9 @@ pihole -g\fR .br pihole status .br -pihole restartdns\fR [options] +pihole reloaddns\fR +.br +pihole reloadlists\fR .br \fBpihole\fR (\fBenable\fR|\fBdisable\fR [time]) .br diff --git a/pihole b/pihole index 93ebfa56..5657637b 100755 --- a/pihole +++ b/pihole @@ -293,7 +293,6 @@ Options: Try 'pihole logging --help' for more information." exit 1 fi - restartDNS echo -e "${OVER} ${TICK} ${str}" } From d4cf5d95a68f82d74c97f367a1c257dd6c548308 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 5 Oct 2024 10:39:29 +0000 Subject: [PATCH 320/413] Bump tox from 4.20.0 to 4.21.2 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.20.0 to 4.21.2. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.20.0...4.21.2) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 8f96a644..903a6acc 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.2 pytest == 8.3.3 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.20.0 +tox == 4.21.2 pytest-clarity == 1.0.1 From aedb2f0a9331fea96d9e0659c6fdfbdc03a201a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 12 Oct 2024 10:02:08 +0000 Subject: [PATCH 321/413] Bump actions/checkout from 4.2.0 to 4.2.1 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.2.0 to 4.2.1. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.2.0...v4.2.1) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 44c7ff5a..21f6c075 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.2.0 + uses: actions/checkout@v4.2.1 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index d2f552a2..8faba669 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.2.0 + uses: actions/checkout@v4.2.1 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 067d4f2a..066a6a1d 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.2.0 + uses: actions/checkout@v4.2.1 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 34a12b74..b4b67bc3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.2.0 + uses: actions/checkout@v4.2.1 - name: Check scripts in repository are executable run: | @@ -70,7 +70,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.2.0 + uses: actions/checkout@v4.2.1 - name: Set up Python 3.10 uses: actions/setup-python@v5.2.0 From fa45a4bec8f9b26b1d3c83df2b6a890b34e06a01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 13 Oct 2024 22:16:42 +0200 Subject: [PATCH 322/413] Add color MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b4b67bc3..d2197ffb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,6 +7,11 @@ on: permissions: contents: read +env: + FORCE_COLOR: 1 + PYTHONUNBUFFERED: 1 + PYTHONUTF8: 1 + jobs: smoke-tests: if: github.event.pull_request.draft == false From 3956ca49b66eb12208d9a31e2dc1673285186ed6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 18 Oct 2024 20:19:02 +0200 Subject: [PATCH 323/413] Don't attempt writing to pihole.toml if file does not exit. FTL needs to be started first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4852929c..ec09b8ed 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2367,15 +2367,6 @@ main() { pihole -a -p "${pw}" fi - # write privacy level and logging to pihole.toml - # set on fresh installations by setPrivacyLevel() and setLogging( - if [ -n "${QUERY_LOGGING}" ]; then - pihole-FTL --config dns.queryLogging "${QUERY_LOGGING}" - fi - if [ -n "${PRIVACY_LEVEL}" ]; then - pihole-FTL --config misc.privacylevel "${PRIVACY_LEVEL}" - fi - # Migrate existing install to v6.0 migrate_dnsmasq_configs @@ -2401,6 +2392,16 @@ main() { restart_service pihole-FTL + # write privacy level and logging to pihole.toml + # needs to be done after FTL service has been started, otherwise pihole.toml does not exist + # set on fresh installations by setPrivacyLevel() and setLogging( + if [ -n "${QUERY_LOGGING}" ]; then + pihole-FTL --config dns.queryLogging "${QUERY_LOGGING}" + fi + if [ -n "${PRIVACY_LEVEL}" ]; then + pihole-FTL --config misc.privacylevel "${PRIVACY_LEVEL}" + fi + # Download and compile the aggregated block list runGravity From 6e905a7f80d2bcfb0f39095536a47b6b9a368bc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 18 Oct 2024 20:22:13 +0200 Subject: [PATCH 324/413] Use setFTLConfigValue to set privacy level and logging. The function is sourced already at this point from utils.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ec09b8ed..34e77ea0 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2396,10 +2396,10 @@ main() { # needs to be done after FTL service has been started, otherwise pihole.toml does not exist # set on fresh installations by setPrivacyLevel() and setLogging( if [ -n "${QUERY_LOGGING}" ]; then - pihole-FTL --config dns.queryLogging "${QUERY_LOGGING}" + setFTLConfigValue "dns.queryLogging" "${QUERY_LOGGING}" fi if [ -n "${PRIVACY_LEVEL}" ]; then - pihole-FTL --config misc.privacylevel "${PRIVACY_LEVEL}" + setFTLConfigValue "misc.privacylevel" "${PRIVACY_LEVEL}" fi # Download and compile the aggregated block list From 1ad135ad697fd0d4cf5df0b26014508356a643a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 18 Oct 2024 21:26:18 +0200 Subject: [PATCH 325/413] Do not print FTL update check details on pihole -up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/update.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index 2ccad27c..e4f784ba 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -144,7 +144,7 @@ main() { local binary binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL) - if FTLcheckUpdate "${binary}"; then + if FTLcheckUpdate "${binary}" &>/dev/null; then FTL_update=true echo -e " ${INFO} FTL:\\t\\t${COL_YELLOW}update available${COL_NC}" else From 2fd51bd244291d59c87a4b4dc184a2586f4ff2c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 18 Oct 2024 23:00:31 +0200 Subject: [PATCH 326/413] Fix gavity version 19 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/gravity.db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 9782a044..5436fb87 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -66,7 +66,7 @@ CREATE TABLE info value TEXT NOT NULL ); -INSERT INTO "info" VALUES('version','18'); +INSERT INTO "info" VALUES('version','19'); CREATE TABLE domain_audit ( From 8c39365db164e294b208f931bce4e2945fa3d602 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 19 Oct 2024 10:32:25 +0000 Subject: [PATCH 327/413] Bump tox from 4.21.2 to 4.23.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.21.2 to 4.23.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.21.2...4.23.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 903a6acc..5b28d6f3 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.2 pytest == 8.3.3 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.21.2 +tox == 4.23.0 pytest-clarity == 1.0.1 From 7ae0b207fc64c306a57c2fadfa0209dadd655dc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 20 Oct 2024 09:34:15 +0200 Subject: [PATCH 328/413] Exit 1 if update script failed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/update.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index e4f784ba..72ba294e 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -38,10 +38,13 @@ GitCheckUpdateAvail() { local curBranch directory="${1}" curdir=$PWD - cd "${directory}" || return + cd "${directory}" || exit 1 # Fetch latest changes in this repo - git fetch --quiet origin + if ! git fetch --quiet origin ; then + echo -e "\\n ${COL_LIGHT_RED}Error: Unable to update local repository. Contact Pi-hole Support.${COL_NC}" + exit 1 + fi # Check current branch. If it is master, then check for the latest available tag instead of latest commit. curBranch=$(git rev-parse --abbrev-ref HEAD) @@ -71,17 +74,17 @@ GitCheckUpdateAvail() { echo -e "\\n ${COL_LIGHT_RED}Error: Local revision could not be obtained, please contact Pi-hole Support" echo -e " Additional debugging output:${COL_NC}" git status - exit + exit 1 fi if [[ "${#REMOTE}" == 0 ]]; then echo -e "\\n ${COL_LIGHT_RED}Error: Remote revision could not be obtained, please contact Pi-hole Support" echo -e " Additional debugging output:${COL_NC}" git status - exit + exit 1 fi # Change back to original directory - cd "${curdir}" || exit + cd "${curdir}" || exit 1 if [[ "${LOCAL}" != "${REMOTE}" ]]; then # Local branch is behind remote branch -> Update From 8bf6480aed03f2ee2ee359064746c369af8d2de5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 20 Oct 2024 09:36:28 +0200 Subject: [PATCH 329/413] Set exit code explicitly everywhere MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/list.sh | 2 +- automated install/basic-install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 8c930f04..5c57f878 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -192,7 +192,7 @@ GetComment() { comment="$1" if [[ "${comment}" =~ [^a-zA-Z0-9_\#:/\.,\ -] ]]; then echo " ${CROSS} Found invalid characters in domain comment!" - exit + exit 1 fi } diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4852929c..18bccab1 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -409,7 +409,7 @@ package_manager_detect() { # we cannot install required packages printf " %b No supported package manager found\\n" "${CROSS}" # so exit the installer - exit + exit 1 fi } From 0e7706ffa36c89051b7725209b8e9b8c2e390149 Mon Sep 17 00:00:00 2001 From: Sebastian Dinator Date: Thu, 24 Oct 2024 02:05:13 -0300 Subject: [PATCH 330/413] Improved error message for invalid protocol in adlist download MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sebastian Dinator Signed-off-by: Sebastián Dinator --- gravity.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 6a4fb88e..f98ecdcf 100755 --- a/gravity.sh +++ b/gravity.sh @@ -598,7 +598,9 @@ gravity_DownloadBlocklistFromUrl() { # Check for allowed protocols if [[ $url != "http"* && $url != "https"* && $url != "file"* && $url != "ftp"* && $url != "ftps"* && $url != "sftp"* ]]; then - echo -e "${OVER} ${CROSS} ${str} Invalid protocol specified, ignoring list" + echo -e "${OVER} ${CROSS} ${str} Invalid protocol specified. Ignoring list." + echo -e " Ensure your URL starts with a valid protocol like http:// or https:// ." + echo -e " E.g., try http://localhost/adblock.txt instead of localhost/adblock.txt." download=false fi From 5589fa171cf3b519950e9d3364a70cd7744cc138 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Dinator?= Date: Thu, 24 Oct 2024 14:03:02 -0300 Subject: [PATCH 331/413] Remove example from error message on invalid protocol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sebastián Dinator --- gravity.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index f98ecdcf..a657727d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -599,8 +599,7 @@ gravity_DownloadBlocklistFromUrl() { # Check for allowed protocols if [[ $url != "http"* && $url != "https"* && $url != "file"* && $url != "ftp"* && $url != "ftps"* && $url != "sftp"* ]]; then echo -e "${OVER} ${CROSS} ${str} Invalid protocol specified. Ignoring list." - echo -e " Ensure your URL starts with a valid protocol like http:// or https:// ." - echo -e " E.g., try http://localhost/adblock.txt instead of localhost/adblock.txt." + echo -e "Ensure your URL starts with a valid protocol like http:// , https:// or file:// ." download=false fi From 8acfa029d94927972f3b11da997d4d15e2c543c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Oct 2024 10:07:38 +0000 Subject: [PATCH 332/413] Bump actions/checkout from 4.2.1 to 4.2.2 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.2.1 to 4.2.2. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.2.1...v4.2.2) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 21f6c075..77aacbec 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.2.1 + uses: actions/checkout@v4.2.2 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 8faba669..724c29d4 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.2.1 + uses: actions/checkout@v4.2.2 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 066a6a1d..7df1a32d 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.2.1 + uses: actions/checkout@v4.2.2 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d2197ffb..4a22c965 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.2.1 + uses: actions/checkout@v4.2.2 - name: Check scripts in repository are executable run: | @@ -75,7 +75,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.2.1 + uses: actions/checkout@v4.2.2 - name: Set up Python 3.10 uses: actions/setup-python@v5.2.0 From 9abb2401058e4a96a0c1de64551dabc3272bb908 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Oct 2024 10:07:42 +0000 Subject: [PATCH 333/413] Bump actions/setup-python from 5.2.0 to 5.3.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.2.0 to 5.3.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.2.0...v5.3.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d2197ffb..b23df5cd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -78,7 +78,7 @@ jobs: uses: actions/checkout@v4.2.1 - name: Set up Python 3.10 - uses: actions/setup-python@v5.2.0 + uses: actions/setup-python@v5.3.0 with: python-version: "3.10" From 6d8b4fdfc6fe7970e2324fc25845d15307bd8de2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Oct 2024 10:22:13 +0000 Subject: [PATCH 334/413] Bump tox from 4.23.0 to 4.23.2 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.23.0 to 4.23.2. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.23.0...4.23.2) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 5b28d6f3..93232df7 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.2 pytest == 8.3.3 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.23.0 +tox == 4.23.2 pytest-clarity == 1.0.1 From fd5b3be03f91adeed1e7124de9d4ba4dbe1e8a8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 28 Sep 2024 20:49:27 +0200 Subject: [PATCH 335/413] Create meta package on the fly ... Debian MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/update.sh | 3 +- automated install/basic-install.sh | 121 +++++++++++------------------ test/test_any_automated_install.py | 42 ++-------- test/test_any_utils.py | 2 +- 4 files changed, 56 insertions(+), 112 deletions(-) diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index 2ccad27c..9240c593 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -106,7 +106,8 @@ main() { # Install packages used by this installation script (necessary if users have removed e.g. git from their systems) package_manager_detect - install_dependent_packages "${INSTALLER_DEPS[@]}" + build_dependency_package + install_dependent_packages # This is unlikely if ! is_repo "${PI_HOLE_FILES_DIR}" ; then diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4852929c..4e112f81 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -102,6 +102,18 @@ fi r=20 c=70 +# Content of Pi-hole's meta package control file +PIHOLE_META_PACKAGE_CONTROL_DEBIAN=$( + cat < +Architecture: all +Description: Pi-hole dependency meta package +Depends: grep,dnsutils,binutils,git,iproute2,dialog,ca-certificates,cron,curl,iputils-ping,psmisc,sudo,unzip,libcap2-bin,dns-root-data,libcap2,netcat-openbsd,procps,jq,lshw,bash-completion +EOM +) + ######## Undocumented Flags. Shhh ######## # These are undocumented flags; some of which we can use when repairing an installation # The runUnattended flag is one example of this @@ -362,10 +374,6 @@ test_dpkg_lock() { # Compatibility package_manager_detect() { - # pull common packages for both distributions out into a common variable - OS_CHECK_COMMON_DEPS=(grep) - PIHOLE_COMMON_DEPS=(curl psmisc sudo unzip jq); - INSTALLER_COMMON_DEPS=(git dialog ca-certificates) # First check to see if apt-get is installed. if is_command apt-get; then @@ -375,17 +383,11 @@ package_manager_detect() { # A variable to store the command used to update the package cache UPDATE_PKG_CACHE="${PKG_MANAGER} update" # The command we will use to actually install packages - PKG_INSTALL=("${PKG_MANAGER}" -qq --no-install-recommends install) + PKG_INSTALL="${PKG_MANAGER} -qq --no-install-recommends install" # grep -c will return 1 if there are no matches. This is an acceptable condition, so we OR TRUE to prevent set -e exiting the script. PKG_COUNT="${PKG_MANAGER} -s -o Debug::NoLocking=true upgrade | grep -c ^Inst || true" # Update package cache update_package_cache || exit 1 - # Packages required to perform the os_check and FTL binary detection - OS_CHECK_DEPS=(dnsutils binutils) - # Packages required to run this install script - INSTALLER_DEPS=(iproute2) - # Packages required to run Pi-hole - PIHOLE_DEPS=(cron iputils-ping libcap2-bin dns-root-data libcap2 netcat-openbsd procps lshw bash-completion) # If apt-get is not found, check for rpm. elif is_command rpm; then @@ -397,7 +399,7 @@ package_manager_detect() { fi # These variable names match the ones for apt-get. See above for an explanation of what they are for. - PKG_INSTALL=("${PKG_MANAGER}" install -y) + PKG_INSTALL="${PKG_MANAGER} install -y" # CentOS package manager returns 100 when there are packages to update so we need to || true to prevent the script from exiting. PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src|.riscv64)' | wc -l || true" OS_CHECK_DEPS=(bind-utils) @@ -413,6 +415,24 @@ package_manager_detect() { fi } +build_dependency_package(){ + # This function will build a package that contains all the dependencies needed for Pi-hole + mkdir -p /tmp/pihole-meta + chmod 0755 /tmp/pihole-meta + + if is_command apt-get; then + # move into the directory + pushd /tmp &>/dev/null || return 1 + mkdir -p /tmp/pihole-meta/DEBIAN + chmod 0755 /tmp/pihole-meta/DEBIAN + touch /tmp/pihole-meta/DEBIAN/control + echo "${PIHOLE_META_PACKAGE_CONTROL_DEBIAN}" > /tmp/pihole-meta/DEBIAN/control + dpkg-deb --build --root-owner-group pihole-meta + # Move back into the directory the user started in + popd &> /dev/null || return 1 + fi +} + # A function for checking if a directory is a git repository is_repo() { # Use a named, local variable instead of the vague $1, which is the first argument passed to this function @@ -1390,61 +1410,20 @@ notify_package_updates_available() { } install_dependent_packages() { + # Install meta dependency package - # Install packages passed in via argument array - # No spinner - conflicts with set -e - declare -a installArray - - # Debian based package install - debconf will download the entire package list - # so we just create an array of packages not currently installed to cut down on the - # amount of download traffic. - # NOTE: We may be able to use this installArray in the future to create a list of package that were - # installed by us, and remove only the installed packages, and not the entire list. if is_command apt-get; then - # For each package, check if it's already installed (and if so, don't add it to the installArray) - for i in "$@"; do - printf " %b Checking for %s..." "${INFO}" "${i}" - if dpkg-query -W -f='${Status}' "${i}" 2>/dev/null | grep "ok installed" &>/dev/null; then - printf "%b %b Checking for %s\\n" "${OVER}" "${TICK}" "${i}" - else - printf "%b %b Checking for %s (will be installed)\\n" "${OVER}" "${INFO}" "${i}" - installArray+=("${i}") - fi - done - # If there's anything to install, install everything in the list. - if [[ "${#installArray[@]}" -gt 0 ]]; then - test_dpkg_lock - # Running apt-get install with minimal output can cause some issues with - # requiring user input (e.g password for phpmyadmin see #218) - printf " %b Processing %s install(s) for: %s, please wait...\\n" "${INFO}" "${PKG_MANAGER}" "${installArray[*]}" - printf '%*s\n' "${c}" '' | tr " " - - "${PKG_INSTALL[@]}" "${installArray[@]}" - printf '%*s\n' "${c}" '' | tr " " - - return + if [ -f /tmp/pihole-meta.deb ]; then + eval "${PKG_INSTALL}" "/tmp/pihole-meta.deb" + rm /tmp/pihole-meta.deb + else + printf " %b Error: Unable to find dependency meta package.\\n" "${COL_LIGHT_RED}" + return 1 fi - printf "\\n" - return 0 fi # Install Fedora/CentOS packages - for i in "$@"; do - # For each package, check if it's already installed (and if so, don't add it to the installArray) - printf " %b Checking for %s..." "${INFO}" "${i}" - if "${PKG_MANAGER}" -q list installed "${i}" &>/dev/null; then - printf "%b %b Checking for %s\\n" "${OVER}" "${TICK}" "${i}" - else - printf "%b %b Checking for %s (will be installed)\\n" "${OVER}" "${INFO}" "${i}" - installArray+=("${i}") - fi - done - # If there's anything to install, install everything in the list. - if [[ "${#installArray[@]}" -gt 0 ]]; then - printf " %b Processing %s install(s) for: %s, please wait...\\n" "${INFO}" "${PKG_MANAGER}" "${installArray[*]}" - printf '%*s\n' "${c}" '' | tr " " - - "${PKG_INSTALL[@]}" "${installArray[@]}" - printf '%*s\n' "${c}" '' | tr " " - - return - fi + printf "\\n" return 0 } @@ -2269,9 +2248,12 @@ main() { # Notify user of package availability notify_package_updates_available - # Install packages necessary to perform os_check - printf " %b Checking for / installing Required dependencies for OS Check...\\n" "${INFO}" - install_dependent_packages "${OS_CHECK_COMMON_DEPS[@]}" "${OS_CHECK_DEPS[@]}" + # Build dependecy package + build_dependency_package + + # Install Pi-hole dependencies + printf " %b Installing required dependencies ...\\n" "${INFO}" + install_dependent_packages # Check that the installed OS is officially supported - display warning if not os_check @@ -2286,10 +2268,6 @@ main() { exit 1 fi - # Install packages used by this installation script - printf " %b Checking for / installing Required dependencies for this install script...\\n" "${INFO}" - install_dependent_packages "${INSTALLER_COMMON_DEPS[@]}" "${INSTALLER_DEPS[@]}" - # in case of an update if [[ -f "${PI_HOLE_V6_CONFIG}" ]]; then # if it's running unattended, @@ -2331,13 +2309,6 @@ main() { # Download or update the scripts by updating the appropriate git repos clone_or_update_repos - # Install the Core dependencies - local dep_install_list=("${PIHOLE_COMMON_DEPS[@]}" "${PIHOLE_DEPS[@]}") - - # Install packages used by the actual software - printf " %b Checking for / installing Required dependencies for Pi-hole software...\\n" "${INFO}" - install_dependent_packages "${dep_install_list[@]}" - unset dep_install_list # Create the pihole user create_pihole_user diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 0930f0af..7d9a49ad 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -12,7 +12,7 @@ from .conftest import ( run_script, ) -FTL_BRANCH = "development-v6" +FTL_BRANCH = "development" def test_supported_package_manager(host): @@ -480,8 +480,8 @@ def test_os_check_fails(host): """ source /opt/pihole/basic-install.sh package_manager_detect - install_dependent_packages ${OS_CHECK_DEPS[@]} - install_dependent_packages ${INSTALLER_DEPS[@]} + build_dependency_package + install_dependent_packages cat < /etc/os-release ID=UnsupportedOS VERSION_ID="2" @@ -504,8 +504,8 @@ def test_os_check_passes(host): """ source /opt/pihole/basic-install.sh package_manager_detect - install_dependent_packages ${OS_CHECK_DEPS[@]} - install_dependent_packages ${INSTALLER_DEPS[@]} + build_dependency_package + install_dependent_packages """ ) detectOS = host.run( @@ -518,21 +518,6 @@ def test_os_check_passes(host): assert expected_stdout in detectOS.stdout -def test_package_manager_has_installer_deps(host): - """Confirms OS is able to install the required packages for the installer""" - mock_command("dialog", {"*": ("", "0")}, host) - output = host.run( - """ - source /opt/pihole/basic-install.sh - package_manager_detect - install_dependent_packages ${INSTALLER_DEPS[@]} - """ - ) - - assert "No package" not in output.stdout - assert output.rc == 0 - - def test_package_manager_has_pihole_deps(host): """Confirms OS is able to install the required packages for Pi-hole""" mock_command("dialog", {"*": ("", "0")}, host) @@ -540,24 +525,11 @@ def test_package_manager_has_pihole_deps(host): """ source /opt/pihole/basic-install.sh package_manager_detect - install_dependent_packages ${PIHOLE_DEPS[@]} + build_dependency_package + install_dependent_packages """ ) assert "No package" not in output.stdout assert output.rc == 0 - -def test_package_manager_has_web_deps(host): - """Confirms OS is able to install the required packages for web""" - mock_command("dialog", {"*": ("", "0")}, host) - output = host.run( - """ - source /opt/pihole/basic-install.sh - package_manager_detect - install_dependent_packages ${PIHOLE_WEB_DEPS[@]} - """ - ) - - assert "No package" not in output.stdout - assert output.rc == 0 diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 59745c48..46b4acca 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -105,7 +105,7 @@ def test_setFTLConfigValue_getFTLConfigValue(host): source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) - echo "development-v6" > /etc/pihole/ftlbranch + echo "development" > /etc/pihole/ftlbranch binary="pihole-FTL${funcOutput##*pihole-FTL}" theRest="${funcOutput%pihole-FTL*}" FTLdetect "${binary}" "${theRest}" From f08c54e166c7f324349483c3bdda486486eb6bf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 29 Sep 2024 19:26:19 +0200 Subject: [PATCH 336/413] Create dependency package on RPM distros MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 91 +++++++++++++++++++++++++++--- test/test_any_automated_install.py | 1 - 2 files changed, 82 insertions(+), 10 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4e112f81..abfb13f3 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -102,8 +102,8 @@ fi r=20 c=70 -# Content of Pi-hole's meta package control file -PIHOLE_META_PACKAGE_CONTROL_DEBIAN=$( +# Content of Pi-hole's meta package control file on APT based systems +PIHOLE_META_PACKAGE_CONTROL_APT=$( cat </dev/null || return 1 + + # Prepare directory structure and control file mkdir -p /tmp/pihole-meta/DEBIAN chmod 0755 /tmp/pihole-meta/DEBIAN touch /tmp/pihole-meta/DEBIAN/control - echo "${PIHOLE_META_PACKAGE_CONTROL_DEBIAN}" > /tmp/pihole-meta/DEBIAN/control + + # Write the control file + echo "${PIHOLE_META_PACKAGE_CONTROL_APT}" > /tmp/pihole-meta/DEBIAN/control + + # Build the package dpkg-deb --build --root-owner-group pihole-meta + # Move back into the directory the user started in popd &> /dev/null || return 1 fi + + if is_command rpm; then + # move into the tmp directory + pushd /tmp &>/dev/null || return 1 + + # Prepare directory structure and spec file + mkdir -p /tmp/pihole-meta/SPECS + touch /tmp/pihole-meta/SPECS/pihole-meta.spec + echo "${PIHOLE_META_PACKAGE_CONTROL_RPM}" > /tmp/pihole-meta/SPECS/pihole-meta.spec + + # check if we need to install the build dependencies + if ! is_command rpmbuild; then + local REMOVE_RPM_BUILD=true + eval "${PKG_INSTALL}" "rpm-build" + fi + + # Build the package + rpmbuild -bb /tmp/pihole-meta/SPECS/pihole-meta.spec + + # Move the package to the /tmp directory + mv /tmp/pihole-meta/RPMS/noarch/pihole-meta*.rpm /tmp/pihole-meta.rpm + + # Remove the build dependencies when we've installed them + if [ -n "${REMOVE_RPM_BUILD}" ]; then + local PKG_REMOVE + PKG_REMOVE="${PKG_MANAGER} remove -y" + eval "${PKG_REMOVE}" "rpm-build" + fi + + # Move back into the directory the user started in + popd &> /dev/null || return 1 + fi + + # Remove the build directory + rm -rf /tmp/pihole-meta } # A function for checking if a directory is a git repository @@ -1412,18 +1475,28 @@ notify_package_updates_available() { install_dependent_packages() { # Install meta dependency package + # Install Debian/Ubuntu packages if is_command apt-get; then if [ -f /tmp/pihole-meta.deb ]; then eval "${PKG_INSTALL}" "/tmp/pihole-meta.deb" rm /tmp/pihole-meta.deb else - printf " %b Error: Unable to find dependency meta package.\\n" "${COL_LIGHT_RED}" + printf " %b Error: Unable to find Pi-hole dependency meta package.\\n" "${COL_LIGHT_RED}" return 1 fi fi # Install Fedora/CentOS packages + if is_command rpm; then + if [ -f /tmp/pihole-meta.rpm ]; then + eval "${PKG_INSTALL}" "/tmp/pihole-meta.rpm" + rm /tmp/pihole-meta.rpm + else + printf " %b Error: Unable to find Pi-hole dependency meta package.\\n" "${COL_LIGHT_RED}" + return 1 + fi + fi printf "\\n" return 0 } @@ -2248,7 +2321,7 @@ main() { # Notify user of package availability notify_package_updates_available - # Build dependecy package + # Build dependency package build_dependency_package # Install Pi-hole dependencies diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 7d9a49ad..70f554e4 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -532,4 +532,3 @@ def test_package_manager_has_pihole_deps(host): assert "No package" not in output.stdout assert output.rc == 0 - From c47f8c2cd67b727fb549efaef15d967a539dc40e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 29 Sep 2024 22:21:44 +0200 Subject: [PATCH 337/413] Overhaul uninstall script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 7 +- automated install/uninstall.sh | 116 +++-------------------------- test/_centos_9.Dockerfile | 2 +- test/_debian_11.Dockerfile | 2 +- test/_debian_12.Dockerfile | 2 +- test/_fedora_39.Dockerfile | 2 +- test/_fedora_40.Dockerfile | 2 +- test/_ubuntu_20.Dockerfile | 2 +- test/_ubuntu_22.Dockerfile | 2 +- test/_ubuntu_23.Dockerfile | 2 +- test/_ubuntu_24.Dockerfile | 2 +- test/conftest.py | 5 +- test/test_any_automated_install.py | 22 ++++++ 13 files changed, 48 insertions(+), 120 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index abfb13f3..8bc55c20 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -409,6 +409,8 @@ package_manager_detect() { PKG_INSTALL="${PKG_MANAGER} -qq --no-install-recommends install" # grep -c will return 1 if there are no matches. This is an acceptable condition, so we OR TRUE to prevent set -e exiting the script. PKG_COUNT="${PKG_MANAGER} -s -o Debug::NoLocking=true upgrade | grep -c ^Inst || true" + # The command we will use to remove packages (used in the uninstaller) + PKG_REMOVE="${PKG_MANAGER} -y remove --purge" # Update package cache update_package_cache || exit 1 @@ -425,7 +427,8 @@ package_manager_detect() { PKG_INSTALL="${PKG_MANAGER} install -y" # CentOS package manager returns 100 when there are packages to update so we need to || true to prevent the script from exiting. PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src|.riscv64)' | wc -l || true" - + # The command we will use to remove packages (used in the uninstaller) + PKG_REMOVE="${PKG_MANAGER} remove -y" # If neither apt-get or yum/dnf package managers were found else # we cannot install required packages @@ -483,8 +486,6 @@ build_dependency_package(){ # Remove the build dependencies when we've installed them if [ -n "${REMOVE_RPM_BUILD}" ]; then - local PKG_REMOVE - PKG_REMOVE="${PKG_MANAGER} remove -y" eval "${PKG_REMOVE}" "rpm-build" fi diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index ac06da73..a073c319 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -38,68 +38,25 @@ fi readonly PI_HOLE_FILES_DIR="/etc/.pihole" SKIP_INSTALL="true" source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh" -# setupVars set in basic-install.sh -source "${setupVars}" # package_manager_detect() sourced from basic-install.sh package_manager_detect -# Uninstall packages used by the Pi-hole -DEPS=("${INSTALLER_COMMON_DEPS[@]}" "${PIHOLE_COMMON_DEPS[@]}" "${OS_CHECK_COMMON_DEPS[@]}" "${INSTALLER_DEPS[@]}" "${PIHOLE_DEPS[@]}" "${OS_CHECK_DEPS[@]}") -# Compatibility -if [ -x "$(command -v apt-get)" ]; then - # Debian Family - PKG_REMOVE=("${PKG_MANAGER}" -y remove --purge) - package_check() { - dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -c "ok installed" - } -elif [ -x "$(command -v rpm)" ]; then - # Fedora Family - PKG_REMOVE=("${PKG_MANAGER}" remove -y) - package_check() { - rpm -qa | grep "^$1-" > /dev/null - } -else - echo -e " ${CROSS} OS distribution not supported" - exit 1 -fi - -removeAndPurge() { - # Purge dependencies +removeMetaPackage() { + # Purge Pi-hole meta package echo "" - for i in "${DEPS[@]}"; do - if package_check "${i}" > /dev/null; then - while true; do - read -rp " ${QST} Do you wish to remove ${COL_WHITE}${i}${COL_NC} from your system? [Y/N] " answer - case ${answer} in - [Yy]* ) - echo -ne " ${INFO} Removing ${i}..."; - ${SUDO} "${PKG_REMOVE[@]}" "${i}" &> /dev/null; - echo -e "${OVER} ${INFO} Removed ${i}"; - break;; - [Nn]* ) echo -e " ${INFO} Skipped ${i}"; break;; - esac - done - else - echo -e " ${INFO} Package ${i} not installed" - fi - done + echo -ne " ${INFO} Removing Pi-hole meta package..."; + eval "${SUDO}" "${PKG_REMOVE}" "pihole-meta" &> /dev/null; + echo -e "${OVER} ${INFO} Removed Pi-hole meta package"; - # Remove dnsmasq config files - ${SUDO} rm -f /etc/dnsmasq.conf /etc/dnsmasq.conf.orig /etc/dnsmasq.d/*-pihole*.conf &> /dev/null - echo -e " ${TICK} Removing dnsmasq config files" - - # Call removeNoPurge to remove Pi-hole specific files - removeNoPurge } -removeNoPurge() { +removePiholeFiles() { # Only web directories/files that are created by Pi-hole should be removed echo -ne " ${INFO} Removing Web Interface..." ${SUDO} rm -rf /var/www/html/admin &> /dev/null - ${SUDO} rm -rf /var/www/html/pihole &> /dev/null - ${SUDO} rm -f /var/www/html/index.lighttpd.orig &> /dev/null + # If the web directory is empty after removing these files, then the parent html directory can be removed. if [ -d "/var/www/html" ]; then @@ -126,45 +83,6 @@ removeNoPurge() { echo -e " ${TICK} Removed /etc/cron.d/pihole" fi - if package_check lighttpd > /dev/null; then - # Attempt to preserve backwards compatibility with older versions - if [[ -f /etc/lighttpd/lighttpd.conf.orig ]]; then - ${SUDO} mv /etc/lighttpd/lighttpd.conf.orig /etc/lighttpd/lighttpd.conf - fi - - if [[ -f /etc/lighttpd/external.conf ]]; then - ${SUDO} rm /etc/lighttpd/external.conf - fi - - # Fedora-based - if [[ -f /etc/lighttpd/conf.d/pihole-admin.conf ]]; then - ${SUDO} rm /etc/lighttpd/conf.d/pihole-admin.conf - conf=/etc/lighttpd/lighttpd.conf - tconf=/tmp/lighttpd.conf.$$ - if awk '!/^include "\/etc\/lighttpd\/conf\.d\/pihole-admin\.conf"$/{print}' \ - $conf > $tconf && mv $tconf $conf; then - : - else - rm $tconf - fi - ${SUDO} chown root:root $conf - ${SUDO} chmod 644 $conf - fi - - # Debian-based - if [[ -f /etc/lighttpd/conf-available/pihole-admin.conf ]]; then - if is_command lighty-disable-mod ; then - ${SUDO} lighty-disable-mod pihole-admin > /dev/null || true - fi - ${SUDO} rm /etc/lighttpd/conf-available/15-pihole-admin.conf - fi - - echo -e " ${TICK} Removed lighttpd configs" - fi - - ${SUDO} rm -f /etc/dnsmasq.d/adList.conf &> /dev/null - ${SUDO} rm -f /etc/dnsmasq.d/01-pihole.conf &> /dev/null - ${SUDO} rm -f /etc/dnsmasq.d/06-rfc6761.conf &> /dev/null ${SUDO} rm -rf /var/log/*pihole* &> /dev/null ${SUDO} rm -rf /var/log/pihole/*pihole* &> /dev/null ${SUDO} rm -rf /etc/pihole/ &> /dev/null @@ -234,23 +152,11 @@ removeNoPurge() { If you need help, reach out to us on GitHub, Discourse, Reddit or Twitter Reinstall at any time: ${COL_WHITE}curl -sSL https://install.pi-hole.net | bash${COL_NC} - ${COL_LIGHT_RED}Please reset the DNS on your router/clients to restore internet connectivity + ${COL_LIGHT_RED}Please reset the DNS on your router/clients to restore internet connectivity${COL_NC} + ${INFO} Pi-hole's meta package has been removed, use the 'autoremove' function from your package manager to remove unused dependencies${COL_NC} ${COL_LIGHT_GREEN}Uninstallation Complete! ${COL_NC}" } ######### SCRIPT ########### -echo -e " ${INFO} Be sure to confirm if any dependencies should not be removed" -while true; do - echo -e " ${INFO} ${COL_YELLOW}The following dependencies may have been added by the Pi-hole install:" - echo -n " " - for i in "${DEPS[@]}"; do - echo -n "${i} " - done - echo "${COL_NC}" - read -rp " ${QST} Do you wish to go through each dependency for removal? (Choosing No will leave all dependencies installed) [Y/n] " answer - case ${answer} in - [Yy]* ) removeAndPurge; break;; - [Nn]* ) removeNoPurge; break;; - * ) removeAndPurge; break;; - esac -done +removeMetaPackage +removePiholeFiles diff --git a/test/_centos_9.Dockerfile b/test/_centos_9.Dockerfile index 7e3c5b3a..a5e7cf0b 100644 --- a/test/_centos_9.Dockerfile +++ b/test/_centos_9.Dockerfile @@ -8,7 +8,7 @@ ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR -RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ +RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR RUN true && \ diff --git a/test/_debian_11.Dockerfile b/test/_debian_11.Dockerfile index cb7d27cc..b8107244 100644 --- a/test/_debian_11.Dockerfile +++ b/test/_debian_11.Dockerfile @@ -5,7 +5,7 @@ ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR -RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ +RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR RUN true && \ diff --git a/test/_debian_12.Dockerfile b/test/_debian_12.Dockerfile index 50d709b1..7446711a 100644 --- a/test/_debian_12.Dockerfile +++ b/test/_debian_12.Dockerfile @@ -5,7 +5,7 @@ ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR -RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ +RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR RUN true && \ diff --git a/test/_fedora_39.Dockerfile b/test/_fedora_39.Dockerfile index 1d3dbc63..5312c2db 100644 --- a/test/_fedora_39.Dockerfile +++ b/test/_fedora_39.Dockerfile @@ -6,7 +6,7 @@ ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR -RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ +RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR RUN true && \ diff --git a/test/_fedora_40.Dockerfile b/test/_fedora_40.Dockerfile index e4879c92..20102a10 100644 --- a/test/_fedora_40.Dockerfile +++ b/test/_fedora_40.Dockerfile @@ -6,7 +6,7 @@ ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR -RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ +RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR RUN true && \ diff --git a/test/_ubuntu_20.Dockerfile b/test/_ubuntu_20.Dockerfile index 64d4f415..75c12673 100644 --- a/test/_ubuntu_20.Dockerfile +++ b/test/_ubuntu_20.Dockerfile @@ -5,7 +5,7 @@ ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR -RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ +RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR RUN true && \ diff --git a/test/_ubuntu_22.Dockerfile b/test/_ubuntu_22.Dockerfile index 34faa361..9206a46a 100644 --- a/test/_ubuntu_22.Dockerfile +++ b/test/_ubuntu_22.Dockerfile @@ -5,7 +5,7 @@ ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR -RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ +RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR ENV DEBIAN_FRONTEND=noninteractive diff --git a/test/_ubuntu_23.Dockerfile b/test/_ubuntu_23.Dockerfile index ea0ad245..4f88be7d 100644 --- a/test/_ubuntu_23.Dockerfile +++ b/test/_ubuntu_23.Dockerfile @@ -5,7 +5,7 @@ ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR -RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ +RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR ENV DEBIAN_FRONTEND=noninteractive diff --git a/test/_ubuntu_24.Dockerfile b/test/_ubuntu_24.Dockerfile index 4d581cd3..4cab43de 100644 --- a/test/_ubuntu_24.Dockerfile +++ b/test/_ubuntu_24.Dockerfile @@ -5,7 +5,7 @@ ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR -RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ +RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR ENV DEBIAN_FRONTEND=noninteractive diff --git a/test/conftest.py b/test/conftest.py index 164e8de5..dcf49790 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -5,9 +5,8 @@ import subprocess from textwrap import dedent IMAGE = "pytest_pihole:test_container" - -tick_box = "[\x1b[1;32m\u2713\x1b[0m]" -cross_box = "[\x1b[1;31m\u2717\x1b[0m]" +tick_box = "[✓]" +cross_box = "[✗]" info_box = "[i]" diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 70f554e4..19812122 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -532,3 +532,25 @@ def test_package_manager_has_pihole_deps(host): assert "No package" not in output.stdout assert output.rc == 0 + + +def test_meta_package_uninstall(host): + """Confirms OS is able to install and uninstall the Pi-hole meta package""" + mock_command("dialog", {"*": ("", "0")}, host) + install = host.run( + """ + source /opt/pihole/basic-install.sh + package_manager_detect + build_dependency_package + install_dependent_packages + """ + ) + assert install.rc == 0 + + uninstall = host.run( + """ + source /opt/pihole/uninstall.sh + removeMetaPackage + """ + ) + assert uninstall.rc == 0 From cd7e5abe25a76c4c7e854fd7e85f133fd9586006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 29 Sep 2024 22:45:47 +0200 Subject: [PATCH 338/413] Add TRAP to the installer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 8bc55c20..e57515d5 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -25,6 +25,9 @@ set -e # When using "su" an incomplete PATH could be passed: https://github.com/pi-hole/pi-hole/issues/3209 export PATH+=':/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' +# Trap any errors, then exit +trap abort INT QUIT TERM + ######## VARIABLES ######### # For better maintainability, we store as much information that can change in variables # This allows us to make a change in one place that can propagate to all instances of the variable @@ -195,6 +198,15 @@ show_ascii_berry() { " } +abort() { + echo -e "\\n\\n ${COL_LIGHT_RED}Installation was interrupted${COL_NC}\\n" + echo -e "Pi-hole's dependencies might be already installed. If you want to remove them you can try to\\n" + echo -e "a) run 'pihole uninstall' \\n" + echo -e "b) Remove the meta-package 'pihole-meta' manually \\n" + echo -e "E.g. sudo apt-get remove pihole-meta && apt-get autoremove \\n" + exit 1 +} + is_command() { # Checks to see if the given command (passed as a string argument) exists on the system. # The function returns 0 (success) if the command exists, and 1 if it doesn't. From 9478e35cb7bc45ef00480ea81a70ad095cca7daa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 12 Oct 2024 14:34:21 +0200 Subject: [PATCH 339/413] Create a random temp direcotry to build the packages and remove leftovers Co-authored-by: RD WebDesign Signed-off-by: yubiuser --- automated install/basic-install.sh | 44 ++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index e57515d5..75418f1c 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -120,7 +120,6 @@ EOM # Content of Pi-hole's meta package control file on RPM based systems PIHOLE_META_PACKAGE_CONTROL_RPM=$( cat </dev/null || return 1 + # remove leftover package if it exists from previous runs + rm -f /tmp/pihole-meta.deb + # Prepare directory structure and control file - mkdir -p /tmp/pihole-meta/DEBIAN - chmod 0755 /tmp/pihole-meta/DEBIAN - touch /tmp/pihole-meta/DEBIAN/control + mkdir -p "${tempdir}"/DEBIAN + chmod 0755 "${tempdir}"/DEBIAN + touch "${tempdir}"/DEBIAN/control # Write the control file - echo "${PIHOLE_META_PACKAGE_CONTROL_APT}" > /tmp/pihole-meta/DEBIAN/control + echo "${PIHOLE_META_PACKAGE_CONTROL_APT}" > "${tempdir}"/DEBIAN/control # Build the package - dpkg-deb --build --root-owner-group pihole-meta + dpkg-deb --build --root-owner-group "${tempdir}" pihole-meta.deb # Move back into the directory the user started in popd &> /dev/null || return 1 @@ -479,10 +490,13 @@ build_dependency_package(){ # move into the tmp directory pushd /tmp &>/dev/null || return 1 + # remove leftover package if it exists from previous runs + rm -f /tmp/pihole-meta.rpm + # Prepare directory structure and spec file - mkdir -p /tmp/pihole-meta/SPECS - touch /tmp/pihole-meta/SPECS/pihole-meta.spec - echo "${PIHOLE_META_PACKAGE_CONTROL_RPM}" > /tmp/pihole-meta/SPECS/pihole-meta.spec + mkdir -p "${tempdir}"/SPECS + touch "${tempdir}"/SPECS/pihole-meta.spec + echo "${PIHOLE_META_PACKAGE_CONTROL_RPM}" > "${tempdir}"/SPECS/pihole-meta.spec # check if we need to install the build dependencies if ! is_command rpmbuild; then @@ -491,10 +505,10 @@ build_dependency_package(){ fi # Build the package - rpmbuild -bb /tmp/pihole-meta/SPECS/pihole-meta.spec + rpmbuild -bb "${tempdir}"/SPECS/pihole-meta.spec --define "_topdir ${tempdir}" # Move the package to the /tmp directory - mv /tmp/pihole-meta/RPMS/noarch/pihole-meta*.rpm /tmp/pihole-meta.rpm + mv "${tempdir}"/RPMS/noarch/pihole-meta*.rpm /tmp/pihole-meta.rpm # Remove the build dependencies when we've installed them if [ -n "${REMOVE_RPM_BUILD}" ]; then @@ -506,7 +520,7 @@ build_dependency_package(){ fi # Remove the build directory - rm -rf /tmp/pihole-meta + rm -rf "${tempdir}" } # A function for checking if a directory is a git repository From 7341f85c2d5d02843d166afcf64f7d66e7412a51 Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Tue, 12 Nov 2024 16:48:52 -0300 Subject: [PATCH 340/413] Add Fedora 41 and remove Fedora 39 from tests Signed-off-by: RD WebDesign --- .github/workflows/test.yml | 2 +- test/{_fedora_39.Dockerfile => _fedora_41.Dockerfile} | 2 +- test/{tox.fedora_39.ini => tox.fedora_41.ini} | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename test/{_fedora_39.Dockerfile => _fedora_41.Dockerfile} (97%) rename test/{tox.fedora_39.ini => tox.fedora_41.ini} (85%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 381abd15..7c419e9a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,8 +68,8 @@ jobs: ubuntu_23, ubuntu_24, centos_9, - fedora_39, fedora_40, + fedora_41, ] env: DISTRO: ${{matrix.distro}} diff --git a/test/_fedora_39.Dockerfile b/test/_fedora_41.Dockerfile similarity index 97% rename from test/_fedora_39.Dockerfile rename to test/_fedora_41.Dockerfile index 1d3dbc63..59858f4e 100644 --- a/test/_fedora_39.Dockerfile +++ b/test/_fedora_41.Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:39 +FROM fedora:41 RUN dnf install -y git initscripts ENV GITDIR=/etc/.pihole diff --git a/test/tox.fedora_39.ini b/test/tox.fedora_41.ini similarity index 85% rename from test/tox.fedora_39.ini rename to test/tox.fedora_41.ini index aaa6b30e..f70da227 100644 --- a/test/tox.fedora_39.ini +++ b/test/tox.fedora_41.ini @@ -6,5 +6,5 @@ allowlist_externals = docker deps = -rrequirements.txt setenv = COLUMNS=120 -commands = docker buildx build --load --progress plain -f _fedora_39.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _fedora_41.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py From 2dc5bd15cc1f70ae1fad9b2a0b1be879436f1c33 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 16 Nov 2024 08:19:09 +0100 Subject: [PATCH 341/413] Remove remaining traces of audit log Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity-db.sh | 14 +++----------- .../Scripts/database_migration/gravity/1_to_2.sql | 7 ------- advanced/Templates/gravity.db.sql | 7 ------- advanced/Templates/gravity_copy.sql | 1 - gravity.sh | 5 +---- 5 files changed, 4 insertions(+), 30 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 8f84e2b0..b0982bcc 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -13,10 +13,9 @@ readonly scriptPath="/etc/.pihole/advanced/Scripts/database_migration/gravity" upgrade_gravityDB(){ - local database piholeDir auditFile version + local database piholeDir version database="${1}" piholeDir="${2}" - auditFile="${piholeDir}/auditlog.list" # Exit early if the database does not exist (e.g. in CI tests) if [[ ! -f "${database}" ]]; then @@ -27,18 +26,11 @@ upgrade_gravityDB(){ version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" if [[ "$version" == "1" ]]; then - # This migration script upgrades the gravity.db file by - # adding the domain_audit table + # This migration script upgraded the gravity.db file by + # adding the domain_audit table. It is now a no-op echo -e " ${INFO} Upgrading gravity database from version 1 to 2" pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/1_to_2.sql" version=2 - - # Store audit domains in database table - if [ -e "${auditFile}" ]; then - echo -e " ${INFO} Migrating content of ${auditFile} into new database" - # database_table_from_file is defined in gravity.sh - database_table_from_file "domain_audit" "${auditFile}" - fi fi if [[ "$version" == "2" ]]; then # This migration script upgrades the gravity.db file by diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql index ef445cc6..857d25a6 100644 --- a/advanced/Scripts/database_migration/gravity/1_to_2.sql +++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql @@ -2,13 +2,6 @@ BEGIN TRANSACTION; -CREATE TABLE domain_audit -( - id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) -); - UPDATE info SET value = 2 WHERE property = 'version'; COMMIT; diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 5436fb87..fbacc16a 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -68,13 +68,6 @@ CREATE TABLE info INSERT INTO "info" VALUES('version','19'); -CREATE TABLE domain_audit -( - id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) -); - CREATE TABLE domainlist_by_group ( domainlist_id INTEGER NOT NULL REFERENCES domainlist (id), diff --git a/advanced/Templates/gravity_copy.sql b/advanced/Templates/gravity_copy.sql index ed11b61a..f9f98446 100644 --- a/advanced/Templates/gravity_copy.sql +++ b/advanced/Templates/gravity_copy.sql @@ -9,7 +9,6 @@ DROP TRIGGER tr_client_add; DROP TRIGGER tr_adlist_add; INSERT OR REPLACE INTO "group" SELECT * FROM OLD."group"; -INSERT OR REPLACE INTO domain_audit SELECT * FROM OLD.domain_audit; INSERT OR REPLACE INTO domainlist SELECT * FROM OLD.domainlist; DELETE FROM OLD.domainlist_by_group WHERE domainlist_id NOT IN (SELECT id FROM OLD.domainlist); diff --git a/gravity.sh b/gravity.sh index a657727d..40023381 100755 --- a/gravity.sh +++ b/gravity.sh @@ -172,10 +172,7 @@ database_table_from_file() { grep -v '^ *#' <"${src}" | while IFS= read -r domain; do # Only add non-empty lines if [[ -n "${domain}" ]]; then - if [[ "${table}" == "domain_audit" ]]; then - # domain_audit table format (no enable or modified fields) - echo "${rowid},\"${domain}\",${timestamp}" >>"${tmpFile}" - elif [[ "${table}" == "adlist" ]]; then + if [[ "${table}" == "adlist" ]]; then # Adlist table format echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0,0,0" >>"${tmpFile}" else From 6ff30f729479d6a81afa9361088a5cd283c3773a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 16 Nov 2024 10:08:30 +0100 Subject: [PATCH 342/413] Backup a stripped-down version of gravity.db after each pihole -g run Signed-off-by: DL6ER --- gravity.sh | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index a657727d..1c4eb2cb 100755 --- a/gravity.sh +++ b/gravity.sh @@ -58,6 +58,7 @@ gravityDBfile_default="/etc/pihole/gravity.db" gravityTEMPfile="${GRAVITYDB}_temp" gravityDIR="$(dirname -- "${gravityDBfile}")" gravityOLDfile="${gravityDIR}/gravity_old.db" +gravityBCKfile="${gravityDIR}/gravity_backup.db" # Generate new SQLite3 file from schema template generate_gravity_database() { @@ -86,6 +87,15 @@ gravity_build_tree() { echo -e "${OVER} ${TICK} ${str}" } +# Rotate gravity backup files +rotate_gravity_backup() { + for i in {9..1}; do + if [ -f "${gravityBCKfile}.${i}" ]; then + mv "${gravityBCKfile}.${i}" "${gravityBCKfile}.$((i + 1))" + fi + done +} + # Copy data from old to new database file and swap them gravity_swap_databases() { str="Swapping databases" @@ -101,10 +111,28 @@ gravity_swap_databases() { oldAvail=false if [ "${availableBlocks}" -gt "$((gravityBlocks * 2))" ] && [ -f "${gravityDBfile}" ]; then oldAvail=true - mv "${gravityDBfile}" "${gravityOLDfile}" - else - rm "${gravityDBfile}" + cp "${gravityDBfile}" "${gravityOLDfile}" fi + + # Drop the gravity and antigravity tables + subsequent VACUUM the current + # database for compaction + output=$({ printf ".timeout 30000\\nDROP TABLE IF EXISTS gravity;\\nDROP TABLE IF EXISTS antigravity;\\nVACUUM;\\n" | pihole-FTL sqlite3 -ni "${gravityDBfile}"; } 2>&1) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to clean current database for backup\\n ${output}" + rotate=false + else + # If multiple gravityBCKfile's are present (appended with a number), rotate them + # We keep at most 10 backups + rotate_gravity_backup + + # Move the old database to the backup location + mv "${gravityDBfile}" "${gravityBCKfile}.1" + fi + + + # Move the new database to the correct location mv "${gravityTEMPfile}" "${gravityDBfile}" echo -e "${OVER} ${TICK} ${str}" From b2ad878f4ac6742ac018161f9f05077c6dd8cd1d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 16 Nov 2024 10:18:00 +0100 Subject: [PATCH 343/413] Try to restore from gravity backup on database preparation error (if backup is available) Signed-off-by: DL6ER --- gravity.sh | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 1c4eb2cb..7b1b0987 100755 --- a/gravity.sh +++ b/gravity.sh @@ -345,6 +345,43 @@ gravity_CheckDNSResolutionAvailable() { echo -e "${OVER} ${TICK} DNS resolution is available" } +# Function: try_restore_backup +# Description: Attempts to restore the previous Pi-hole gravity database from a +# backup file. If a backup exists, it copies the backup to the +# gravity database file and prepares a new gravity database. If the +# restoration is successful, it returns 0. Otherwise, it returns 1. +# Returns: +# 0 - If the backup is successfully restored. +# 1 - If no backup is available or if the restoration fails. +try_restore_backup () { + # Check if a backup exists + if [ -f "${gravityBCKfile}.1" ]; then + echo -e " ${INFO} Attempting to restore previous database from backup" + cp "${gravityBCKfile}.1" "${gravityDBfile}" + + # If the backup was successfully copied, prepare a new gravity database from + # it + if [ -f "${gravityDBfile}" ]; then + output=$({ pihole-FTL sqlite3 -ni "${gravityTEMPfile}" <<<"${copyGravity}"; } 2>&1) + status="$?" + + # Error checking + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n ${output}" + gravity_Cleanup "error" + fi + + echo -e " ${TICK} Successfully restored from backup (${gravityBCKfile}.1)" + return 0 + else + echo -e " ${CROSS} Unable to restore backup" + fi + fi + + echo -e " ${CROSS} No backup available" + return 1 +} + # Retrieve blocklist URLs and parse domains from adlist.list gravity_DownloadBlocklists() { echo -e " ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..." @@ -411,7 +448,11 @@ gravity_DownloadBlocklists() { if [[ "${status}" -ne 0 ]]; then echo -e "\\n ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n ${output}" - return 1 + + # Try to attempt a backup restore + if ! try_restore_backup; then + return 1 + fi fi echo -e "${OVER} ${TICK} ${str}" From 59e9bac79482e9d1a54ec4b654372a305ef8e079 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 16 Nov 2024 11:47:59 +0100 Subject: [PATCH 344/413] Only try to obtain sources after possible database restore following a corruption Signed-off-by: DL6ER --- gravity.sh | 56 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/gravity.sh b/gravity.sh index 7b1b0987..9b8eec11 100755 --- a/gravity.sh +++ b/gravity.sh @@ -121,7 +121,6 @@ gravity_swap_databases() { if [[ "${status}" -ne 0 ]]; then echo -e "\\n ${CROSS} Unable to clean current database for backup\\n ${output}" - rotate=false else # If multiple gravityBCKfile's are present (appended with a number), rotate them # We keep at most 10 backups @@ -390,32 +389,6 @@ gravity_DownloadBlocklists() { echo -e " ${INFO} Storing gravity database in ${COL_BOLD}${gravityDBfile}${COL_NC}" fi - # Retrieve source URLs from gravity database - # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true) - mapfile -t sources <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2>/dev/null)" - mapfile -t sourceIDs <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2>/dev/null)" - mapfile -t sourceTypes <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2>/dev/null)" - - # Parse source domains from $sources - mapfile -t sourceDomains <<<"$( - # Logic: Split by folder/port - awk -F '[/:]' '{ - # Remove URL protocol & optional username:password@ - gsub(/(.*:\/\/|.*:.*@)/, "", $0) - if(length($1)>0){print $1} - else {print "local"} - }' <<<"$(printf '%s\n' "${sources[@]}")" 2>/dev/null - )" - - local str="Pulling blocklist source list into range" - echo -e "${OVER} ${TICK} ${str}" - - if [[ -z "${sources[*]}" ]] || [[ -z "${sourceDomains[*]}" ]]; then - echo -e " ${INFO} No source list found, or it is empty" - echo "" - unset sources - fi - local url domain str target compression adlist_type echo "" @@ -453,9 +426,38 @@ gravity_DownloadBlocklists() { if ! try_restore_backup; then return 1 fi + + echo -e " ${TICK} ${str}" + else + echo -e "${OVER} ${TICK} ${str}" fi + + # Retrieve source URLs from gravity database + # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true) + mapfile -t sources <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2>/dev/null)" + mapfile -t sourceIDs <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2>/dev/null)" + mapfile -t sourceTypes <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2>/dev/null)" + + # Parse source domains from $sources + mapfile -t sourceDomains <<<"$( + # Logic: Split by folder/port + awk -F '[/:]' '{ + # Remove URL protocol & optional username:password@ + gsub(/(.*:\/\/|.*:.*@)/, "", $0) + if(length($1)>0){print $1} + else {print "local"} + }' <<<"$(printf '%s\n' "${sources[@]}")" 2>/dev/null + )" + + local str="Pulling blocklist source list into range" echo -e "${OVER} ${TICK} ${str}" + if [[ -z "${sources[*]}" ]] || [[ -z "${sourceDomains[*]}" ]]; then + echo -e " ${INFO} No source list found, or it is empty" + echo "" + unset sources + fi + # Use compression to reduce the amount of data that is transferred # between the Pi-hole and the ad list provider. Use this feature # only if it is supported by the locally available version of curl From a5cb07c76ef196b3991614892683ec259d2e4b8e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 16 Nov 2024 11:52:31 +0100 Subject: [PATCH 345/413] Attempt to restore from all possibly available gravity.db backups Signed-off-by: DL6ER --- gravity.sh | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/gravity.sh b/gravity.sh index 9b8eec11..3ca775a4 100755 --- a/gravity.sh +++ b/gravity.sh @@ -353,10 +353,12 @@ gravity_CheckDNSResolutionAvailable() { # 0 - If the backup is successfully restored. # 1 - If no backup is available or if the restoration fails. try_restore_backup () { + local num + num=$1 # Check if a backup exists - if [ -f "${gravityBCKfile}.1" ]; then + if [ -f "${gravityBCKfile}.${num}" ]; then echo -e " ${INFO} Attempting to restore previous database from backup" - cp "${gravityBCKfile}.1" "${gravityDBfile}" + cp "${gravityBCKfile}.${num}" "${gravityDBfile}" # If the backup was successfully copied, prepare a new gravity database from # it @@ -370,14 +372,14 @@ try_restore_backup () { gravity_Cleanup "error" fi - echo -e " ${TICK} Successfully restored from backup (${gravityBCKfile}.1)" + echo -e " ${TICK} Successfully restored from backup (${gravityBCKfile}.${num})" return 0 else - echo -e " ${CROSS} Unable to restore backup" + echo -e " ${CROSS} Unable to restore backup no. ${num}" fi fi - echo -e " ${CROSS} No backup available" + echo -e " ${CROSS} Backup no. ${num} not available" return 1 } @@ -423,7 +425,14 @@ gravity_DownloadBlocklists() { echo -e "\\n ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n ${output}" # Try to attempt a backup restore - if ! try_restore_backup; then + for i in {1..9}; do + if try_restore_backup "${i}"; then + break + fi + done + + # If none of the attempts worked, return 1 + if [[ "${i}" -eq 9 ]]; then return 1 fi From fdf44355d2dc6aad23b269aff29f61f4ff700706 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 16 Nov 2024 12:08:27 +0100 Subject: [PATCH 346/413] Keep up to 10 database backups Signed-off-by: DL6ER --- gravity.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 3ca775a4..ebef5d2f 100755 --- a/gravity.sh +++ b/gravity.sh @@ -425,14 +425,14 @@ gravity_DownloadBlocklists() { echo -e "\\n ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n ${output}" # Try to attempt a backup restore - for i in {1..9}; do + for i in {1..10}; do if try_restore_backup "${i}"; then break fi done # If none of the attempts worked, return 1 - if [[ "${i}" -eq 9 ]]; then + if [[ "${i}" -eq 10 ]]; then return 1 fi From 62b63f87e0b47e04e2d602f1909fe3ff2b70d407 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 16 Nov 2024 12:46:43 +0100 Subject: [PATCH 347/413] Use rpm instead of dnf to check for installed package. dnf changed to returning success even for not-installed packages wit Fedora 41 Signed-off-by: DL6ER --- automated install/basic-install.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 0931e4ca..8f5b9879 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1430,7 +1430,7 @@ install_dependent_packages() { for i in "$@"; do # For each package, check if it's already installed (and if so, don't add it to the installArray) printf " %b Checking for %s..." "${INFO}" "${i}" - if "${PKG_MANAGER}" -q list installed "${i}" &>/dev/null; then + if rpm -q "${i}" &>/dev/null; then printf "%b %b Checking for %s\\n" "${OVER}" "${TICK}" "${i}" else printf "%b %b Checking for %s (will be installed)\\n" "${OVER}" "${INFO}" "${i}" @@ -2270,7 +2270,7 @@ main() { notify_package_updates_available # Install packages necessary to perform os_check - printf " %b Checking for / installing Required dependencies for OS Check...\\n" "${INFO}" + printf " %b Checking for / installing required dependencies for OS Check...\\n" "${INFO}" install_dependent_packages "${OS_CHECK_COMMON_DEPS[@]}" "${OS_CHECK_DEPS[@]}" # Check that the installed OS is officially supported - display warning if not @@ -2287,7 +2287,7 @@ main() { fi # Install packages used by this installation script - printf " %b Checking for / installing Required dependencies for this install script...\\n" "${INFO}" + printf " %b Checking for / installing required dependencies for this install script...\\n" "${INFO}" install_dependent_packages "${INSTALLER_COMMON_DEPS[@]}" "${INSTALLER_DEPS[@]}" # in case of an update @@ -2335,7 +2335,7 @@ main() { local dep_install_list=("${PIHOLE_COMMON_DEPS[@]}" "${PIHOLE_DEPS[@]}") # Install packages used by the actual software - printf " %b Checking for / installing Required dependencies for Pi-hole software...\\n" "${INFO}" + printf " %b Checking for / installing required dependencies for Pi-hole software...\\n" "${INFO}" install_dependent_packages "${dep_install_list[@]}" unset dep_install_list From 9a9af719f030ed39c0d87f4b8e3412f95deed446 Mon Sep 17 00:00:00 2001 From: Dominik Date: Sun, 17 Nov 2024 17:05:26 +0100 Subject: [PATCH 348/413] Update gravity.sh Co-authored-by: RD WebDesign Signed-off-by: Dominik --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index ebef5d2f..b616b2e4 100755 --- a/gravity.sh +++ b/gravity.sh @@ -357,7 +357,7 @@ try_restore_backup () { num=$1 # Check if a backup exists if [ -f "${gravityBCKfile}.${num}" ]; then - echo -e " ${INFO} Attempting to restore previous database from backup" + echo -e " ${INFO} Attempting to restore previous database from backup no. ${num}" cp "${gravityBCKfile}.${num}" "${gravityDBfile}" # If the backup was successfully copied, prepare a new gravity database from From 633c971a4433f171acecf38176d849d5c2beac93 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 23 Nov 2024 09:41:59 +0100 Subject: [PATCH 349/413] Store failed gravity restoration status in message table Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 5 +++++ gravity.sh | 9 ++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 5436fb87..3e8603e9 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -67,6 +67,11 @@ CREATE TABLE info ); INSERT INTO "info" VALUES('version','19'); +/* This is a flag to indicate if gravity was restored from a backup + false = not restored, + failed = restoration failed due to no backup + other string = restoration successful with the string being the backup file used */ +INSERT INTO "info" VALUES('gravity_restored','false'); CREATE TABLE domain_audit ( diff --git a/gravity.sh b/gravity.sh index b616b2e4..4bf8e9c6 100755 --- a/gravity.sh +++ b/gravity.sh @@ -353,12 +353,13 @@ gravity_CheckDNSResolutionAvailable() { # 0 - If the backup is successfully restored. # 1 - If no backup is available or if the restoration fails. try_restore_backup () { - local num + local num filename num=$1 + filename="${gravityBCKfile}.${num}" # Check if a backup exists - if [ -f "${gravityBCKfile}.${num}" ]; then + if [ -f "${filename}" ]; then echo -e " ${INFO} Attempting to restore previous database from backup no. ${num}" - cp "${gravityBCKfile}.${num}" "${gravityDBfile}" + cp "${filename}" "${gravityDBfile}" # If the backup was successfully copied, prepare a new gravity database from # it @@ -372,6 +373,7 @@ try_restore_backup () { gravity_Cleanup "error" fi + pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) values ('gravity_restored','${filename}');" echo -e " ${TICK} Successfully restored from backup (${gravityBCKfile}.${num})" return 0 else @@ -434,6 +436,7 @@ gravity_DownloadBlocklists() { # If none of the attempts worked, return 1 if [[ "${i}" -eq 10 ]]; then return 1 + pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) values ('gravity_restored','failed');" fi echo -e " ${TICK} ${str}" From 531490397da3e23695de41b59dd918dc404520b0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Nov 2024 12:33:26 +0100 Subject: [PATCH 350/413] When new domains are added to gravity and users run the first time gravity in the terminal (not via web), the list.123.abc.com file is created as root and stays like that. This causes issues down the line when users later try to run gravity from the web interface where we do not have root capabilities. This commit checks for write permissions and suggests what to do on error. It always ensures ownership and permissions are correct Signed-off-by: DL6ER --- gravity.sh | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index 40023381..aebc58cb 100755 --- a/gravity.sh +++ b/gravity.sh @@ -59,14 +59,24 @@ gravityTEMPfile="${GRAVITYDB}_temp" gravityDIR="$(dirname -- "${gravityDBfile}")" gravityOLDfile="${gravityDIR}/gravity_old.db" +fix_owner_permissions() { + # Fix ownership and permissions for the specified file + # User and group are set to pihole:pihole + # Permissions are set to 664 (rw-rw-r--) + chown pihole:pihole "${1}" + chmod 664 "${1}" + + # Ensure the containing directory is group writable + chmod g+w "$(dirname -- "${1}")" +} + # Generate new SQLite3 file from schema template generate_gravity_database() { if ! pihole-FTL sqlite3 -ni "${gravityDBfile}" <"${gravityDBschema}"; then echo -e " ${CROSS} Unable to create ${gravityDBfile}" return 1 fi - chown pihole:pihole "${gravityDBfile}" - chmod g+w "${piholeDir}" "${gravityDBfile}" + fix_owner_permissions "${gravityDBfile}" } # Build gravity tree @@ -413,6 +423,19 @@ gravity_DownloadBlocklists() { saveLocation="${piholeDir}/list.${id}.${domain}.${domainsExtension}" activeDomains[$i]="${saveLocation}" + # Check if we can write to the save location file + if ! touch "${saveLocation}" 2>/dev/null; then + echo -e " ${CROSS} Unable to write to ${saveLocation}" + echo " Please run pihole -g as root" + echo "" + continue + fi + + # Chown the file to the pihole user + # This is necessary for the FTL to be able to update the file + # when gravity is run from the web interface + fix_owner_permissions "${saveLocation}" + echo -e " ${INFO} Target: ${url}" local regex check_url # Check for characters NOT allowed in URLs @@ -996,8 +1019,7 @@ fi update_gravity_timestamp # Ensure proper permissions are set for the database -chown pihole:pihole "${gravityTEMPfile}" -chmod g+w "${piholeDir}" "${gravityTEMPfile}" +fix_owner_permissions "${gravityTEMPfile}" # Build the tree timeit gravity_build_tree From d9288b896ea5bdd6b95d46c77a34746bf7b2cfbc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 26 Nov 2024 18:01:52 +0100 Subject: [PATCH 351/413] Do not touch list files as this creates them. This causes issues down the line in the Heisenberg compensator Signed-off-by: DL6ER --- gravity.sh | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index aebc58cb..6ffac4b0 100755 --- a/gravity.sh +++ b/gravity.sh @@ -358,7 +358,7 @@ gravity_DownloadBlocklists() { unset sources fi - local url domain str target compression adlist_type + local url domain str target compression adlist_type directory echo "" # Prepare new gravity database @@ -423,8 +423,18 @@ gravity_DownloadBlocklists() { saveLocation="${piholeDir}/list.${id}.${domain}.${domainsExtension}" activeDomains[$i]="${saveLocation}" - # Check if we can write to the save location file - if ! touch "${saveLocation}" 2>/dev/null; then + # Check if we can write to the save location file without actually creating + # it (in case it doesn't exist) + # First, check if the directory is writable + directory="$(dirname -- "${saveLocation}")" + if [ ! -w "${directory}" ]; then + echo -e " ${CROSS} Unable to write to ${directory}" + echo " Please run pihole -g as root" + echo "" + continue + fi + # Then, check if the file is writable (if it exists) + if [ -e "${saveLocation}" ] && [ ! -w "${saveLocation}" ]; then echo -e " ${CROSS} Unable to write to ${saveLocation}" echo " Please run pihole -g as root" echo "" @@ -464,6 +474,7 @@ compareLists() { if ! sha1sum --check --status --strict "${target}.sha1"; then # The list changed upstream, we need to update the checksum sha1sum "${target}" >"${target}.sha1" + fix_owner_permissions "${target}.sha1" echo " ${INFO} List has been updated" database_adlist_status "${adlistID}" "1" else @@ -473,6 +484,7 @@ compareLists() { else # No checksum available, create one for comparing on the next run sha1sum "${target}" >"${target}.sha1" + fix_owner_permissions "${target}.sha1" # We assume here it was changed upstream database_adlist_status "${adlistID}" "1" fi From 1c4a48258fd60edff32c719a353c88c505494c9d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 26 Nov 2024 18:16:15 +0100 Subject: [PATCH 352/413] Only set ownership/permissions once the file was created Signed-off-by: DL6ER --- gravity.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/gravity.sh b/gravity.sh index 6ffac4b0..ccba6c74 100755 --- a/gravity.sh +++ b/gravity.sh @@ -441,11 +441,6 @@ gravity_DownloadBlocklists() { continue fi - # Chown the file to the pihole user - # This is necessary for the FTL to be able to update the file - # when gravity is run from the web interface - fix_owner_permissions "${saveLocation}" - echo -e " ${INFO} Target: ${url}" local regex check_url # Check for characters NOT allowed in URLs @@ -746,7 +741,7 @@ gravity_ParseFileIntoDomains() { -e 's/^.*\s+//g' \ -e '/^$/d' "${destination}" - chmod 644 "${destination}" + fix_owner_permissions "${destination}" } # Report number of entries in a table From b23348916b3b7cf175128c7ad32ce69ea28dc294 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 26 Nov 2024 21:07:11 +0100 Subject: [PATCH 353/413] Remove Ubuntu 23 tests, it is EOL Signed-off-by: DL6ER --- .github/workflows/test.yml | 1 - test/tox.ubuntu_23.ini | 10 ---------- 2 files changed, 11 deletions(-) delete mode 100644 test/tox.ubuntu_23.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7c419e9a..983ca0bc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -65,7 +65,6 @@ jobs: debian_12, ubuntu_20, ubuntu_22, - ubuntu_23, ubuntu_24, centos_9, fedora_40, diff --git a/test/tox.ubuntu_23.ini b/test/tox.ubuntu_23.ini deleted file mode 100644 index f0a32a68..00000000 --- a/test/tox.ubuntu_23.ini +++ /dev/null @@ -1,10 +0,0 @@ -[tox] -envlist = py3 - -[testenv:py3] -allowlist_externals = docker -deps = -rrequirements.txt -setenv = - COLUMNS=120 -commands = docker buildx build --load --progress plain -f _ubuntu_23.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py From 5ef4a5e8b074aa878c6f82248b5f0c7f0c6f5a2e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Dec 2024 10:09:54 +0000 Subject: [PATCH 354/413] Bump pytest from 8.3.3 to 8.3.4 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.3.3 to 8.3.4. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.3.3...8.3.4) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 93232df7..d72475c4 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.2 -pytest == 8.3.3 +pytest == 8.3.4 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 tox == 4.23.2 From 11e00e04b5bb7cf99131552f566db7209b5f03c6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 7 Dec 2024 10:27:20 +0100 Subject: [PATCH 355/413] Fix ARP flush command Signed-off-by: DL6ER --- advanced/Scripts/piholeARPTable.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeARPTable.sh b/advanced/Scripts/piholeARPTable.sh index c04c5b33..f55b1320 100755 --- a/advanced/Scripts/piholeARPTable.sh +++ b/advanced/Scripts/piholeARPTable.sh @@ -32,7 +32,7 @@ flushARP(){ fi # Stop FTL to prevent database access - if ! output=$(pihole-FTL service stop 2>&1); then + if ! output=$(service pihole-FTL stop 2>&1); then echo -e "${OVER} ${CROSS} Failed to stop FTL" echo " Output: ${output}" return 1 @@ -64,7 +64,7 @@ flushARP(){ fi # Start FTL again - if ! output=$(pihole-FTL service restart 2>&1); then + if ! output=$(service pihole-FTL restart 2>&1); then echo -e "${OVER} ${CROSS} Failed to restart FTL" echo " Output: ${output}" return 1 From 8f1fce8f4bcc795bf721da21b242cdd6b7eab364 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 10 Dec 2024 19:01:58 +0000 Subject: [PATCH 356/413] move the sourcing of utils.sh outside of installPihole Signed-off-by: Adam Warner --- automated install/basic-install.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 90eea220..b2bbb219 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1685,15 +1685,6 @@ installPihole() { exit 1 fi - # /opt/pihole/utils.sh should be installed by installScripts now, so we can use it - if [ -f "${PI_HOLE_INSTALL_DIR}/utils.sh" ]; then - # shellcheck disable=SC1091 - source "${PI_HOLE_INSTALL_DIR}/utils.sh" - else - printf " %b Failure: /opt/pihole/utils.sh does not exist .\\n" "${CROSS}" - exit 1 - fi - remove_old_dnsmasq_ftl_configs remove_old_pihole_lighttpd_configs @@ -2426,6 +2417,15 @@ main() { # Install and log everything to a file installPihole | tee -a /proc/$$/fd/3 + # /opt/pihole/utils.sh should be installed by installScripts now, so we can use it + if [ -f "${PI_HOLE_INSTALL_DIR}/utils.sh" ]; then + # shellcheck disable=SC1091 + source "${PI_HOLE_INSTALL_DIR}/utils.sh" + else + printf " %b Failure: /opt/pihole/utils.sh does not exist .\\n" "${CROSS}" + exit 1 + fi + # Copy the temp log file into final log location for storage copy_to_install_log From 764aa48c143bb1f4b36b8a8002b52d9e67c67e10 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 11 Dec 2024 17:22:49 +0000 Subject: [PATCH 357/413] Remove no-longer-needed utils Signed-off-by: Adam Warner --- advanced/Scripts/utils.sh | 38 ----------------------- test/test_any_utils.py | 64 --------------------------------------- 2 files changed, 102 deletions(-) diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 67301394..63d51f87 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -25,7 +25,6 @@ # # Example usage: # addOrEditKeyValPair "/etc/pihole/setupVars.conf" "BLOCKING_ENABLED" "true" -# TODO: We miight not actually need this function in v6 ####################### addOrEditKeyValPair() { local file="${1}" @@ -44,43 +43,6 @@ addOrEditKeyValPair() { fi } -####################### -# Takes two arguments: file, and key. -# Adds a key to target file -# -# Example usage: -# addKey "/etc/dnsmasq.d/01-pihole.conf" "log-queries" -####################### -addKey(){ - local file="${1}" - local key="${2}" - - # touch file to prevent grep error if file does not exist yet - touch "${file}" - - # Match key against entire line, using both anchors. We assume - # that the file's keys never have bounding whitespace. Anchors - # are necessary to ensure the key is considered absent when it - # is a substring of another key present in the file. - if ! grep -q "^${key}$" "${file}"; then - # Key does not exist, add it. - echo "${key}" >> "${file}" - fi -} - -####################### -# Takes two arguments: file, and key. -# Deletes a key or key/value pair from target file -# -# Example usage: -# removeKey "/etc/pihole/setupVars.conf" "PIHOLE_DNS_1" -####################### -removeKey() { - local file="${1}" - local key="${2}" - sed -i "/^${key}/d" "${file}" -} - ####################### # returns FTL's PID based on the content of the pihole-FTL.pid file # diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 59745c48..cf6cc7e6 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -18,70 +18,6 @@ def test_key_val_replacement_works(host): assert expected_stdout == output.stdout -def test_key_addition_works(host): - """Confirms addKey adds a key (no value) to a file without duplicating it""" - host.run( - """ - source /opt/pihole/utils.sh - addKey "./testoutput" "KEY_ONE" - addKey "./testoutput" "KEY_ONE" - addKey "./testoutput" "KEY_TWO" - addKey "./testoutput" "KEY_TWO" - addKey "./testoutput" "KEY_THREE" - addKey "./testoutput" "KEY_THREE" - """ - ) - output = host.run( - """ - cat ./testoutput - """ - ) - expected_stdout = "KEY_ONE\nKEY_TWO\nKEY_THREE\n" - assert expected_stdout == output.stdout - - -def test_key_addition_substr(host): - """Confirms addKey adds substring keys (no value) to a file""" - host.run( - """ - source /opt/pihole/utils.sh - addKey "./testoutput" "KEY_ONE" - addKey "./testoutput" "KEY_O" - addKey "./testoutput" "KEY_TWO" - addKey "./testoutput" "Y_TWO" - """ - ) - output = host.run( - """ - cat ./testoutput - """ - ) - expected_stdout = "KEY_ONE\nKEY_O\nKEY_TWO\nY_TWO\n" - assert expected_stdout == output.stdout - - -def test_key_removal_works(host): - """Confirms removeKey removes a key or key/value pair""" - host.run( - """ - source /opt/pihole/utils.sh - addOrEditKeyValPair "./testoutput" "KEY_ONE" "value1" - addOrEditKeyValPair "./testoutput" "KEY_TWO" "value2" - addOrEditKeyValPair "./testoutput" "KEY_THREE" "value3" - addKey "./testoutput" "KEY_FOUR" - removeKey "./testoutput" "KEY_TWO" - removeKey "./testoutput" "KEY_FOUR" - """ - ) - output = host.run( - """ - cat ./testoutput - """ - ) - expected_stdout = "KEY_ONE=value1\nKEY_THREE=value3\n" - assert expected_stdout == output.stdout - - def test_getFTLPID_default(host): """Confirms getFTLPID returns the default value if FTL is not running""" output = host.run( From fdda40994b2e4f3515c9a84738c1e0bf5d7aaada Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 14 Dec 2024 22:26:05 +0100 Subject: [PATCH 358/413] Fix rare case when apt and rpm package managers are found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index b2bbb219..81b240e4 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -484,9 +484,8 @@ build_dependency_package(){ # Move back into the directory the user started in popd &> /dev/null || return 1 - fi - if is_command rpm; then + elif is_command rpm; then # move into the tmp directory pushd /tmp &>/dev/null || return 1 @@ -517,6 +516,13 @@ build_dependency_package(){ # Move back into the directory the user started in popd &> /dev/null || return 1 + + # If neither apt-get or yum/dnf package managers were found + else + # we cannot build required packages + printf " %b No supported package manager found\\n" "${CROSS}" + # so exit the installer + exit 1 fi # Remove the build directory From cc01c110f18a2137a2c3f7cd874de80a979b63a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 14 Dec 2024 22:31:02 +0100 Subject: [PATCH 359/413] Also check during installlation of the dependency package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 81b240e4..f1ef8887 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1517,11 +1517,9 @@ install_dependent_packages() { printf " %b Error: Unable to find Pi-hole dependency meta package.\\n" "${COL_LIGHT_RED}" return 1 fi - fi - # Install Fedora/CentOS packages - if is_command rpm; then - if [ -f /tmp/pihole-meta.rpm ]; then + elif is_command rpm; then + if [ -f /tmp/pihole-meta.rpm ]; then eval "${PKG_INSTALL}" "/tmp/pihole-meta.rpm" rm /tmp/pihole-meta.rpm else @@ -1529,7 +1527,14 @@ install_dependent_packages() { return 1 fi + # If neither apt-get or yum/dnf package managers were found + else + # we cannot install the dependency package + printf " %b No supported package manager found\\n" "${CROSS}" + # so exit the installer + exit 1 fi + printf "\\n" return 0 } From 26ef0be9d635bce7a410330d9c4932a661c45970 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 16 Dec 2024 11:56:04 +0100 Subject: [PATCH 360/413] One check less MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f1ef8887..a4ccf434 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2490,10 +2490,6 @@ main() { # Update local and remote versions via updatechecker /opt/pihole/updatecheck.sh - if [[ "${useUpdateVars}" == false ]]; then - displayFinalMessage "${pw}" - fi - # If there is a password if ((${#pw} > 0)); then # display the password @@ -2502,6 +2498,9 @@ main() { fi if [[ "${useUpdateVars}" == false ]]; then + # Display the completion dialog + displayFinalMessage "${pw}" + # If the Web interface was installed, printf " %b View the web interface at http://pi.hole:${WEBPORT}/admin or http://%s/admin\\n\\n" "${INFO}" "${IPV4_ADDRESS%/*}:${WEBPORT}" From 4d55bc1ae3d244e19e7158eeffdbf3bb3d2eb208 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 16 Dec 2024 12:08:16 +0100 Subject: [PATCH 361/413] pihole -a is gone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- manpages/pihole.8 | 34 ++++++------------------------ 2 files changed, 9 insertions(+), 29 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a4ccf434..ef1e372a 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2446,7 +2446,7 @@ main() { if [[ $(pihole-FTL --config webserver.api.pwhash) == '""' ]]; then # generate a random password pw=$(tr -dc _A-Z-a-z-0-9 0)); then # display the password printf " %b Web Interface password: %b%s%b\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${pw}" "${COL_NC}" - printf " %b This can be changed using 'pihole -a -p'\\n\\n" "${INFO}" + printf " %b This can be changed using 'pihole setpassword'\\n\\n" "${INFO}" fi if [[ "${useUpdateVars}" == false ]]; then diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 825a1fd1..5a312efe 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -7,13 +7,7 @@ Pi-hole : A black-hole for internet advertisements \fBpihole\fR (\fB-w\fR|\fB-b\fR|\fB--wild\fR|\fB--regex\fR) [options] domain(s) .br -\fBpihole -a\fR \fB-p\fR password -.br -\fBpihole -a\fR (\fB-c|-f|-k\fR) -.br -\fBpihole -a -i\fR interface -.br -\fBpihole -a -l\fR privacylevel +\fBpihole setpassword\fR password .br \fBpihole -c\fR [-j|-r|-e] .br @@ -120,25 +114,6 @@ Available commands and options: (regular expressions are supported) .br -\fB-a, admin\fR [options] -.br - - (Admin options): -.br - -p, password Set Web Interface password -.br - -c, celsius Set Celsius as preferred temperature unit -.br - -f, fahrenheit Set Fahrenheit as preferred temperature unit -.br - -k, kelvin Set Kelvin as preferred temperature unit -.br - -i, interface Specify dnsmasq's interface listening behavior -.br - -l, privacylevel Set privacy level - (0 = lowest, 3 = highest) -.br - \fB-g, updateGravity\fR .br Update the list of ad-serving domains @@ -211,6 +186,11 @@ Available commands and options: repositories .br +\fBsetpassword\fR +.br + Set Web Interface password +.br + \fBuninstall\fR .br Uninstall Pi-hole from your system @@ -303,7 +283,7 @@ Allow-/denylist manipulation Changing the Web Interface password .br -\fBpihole -a -p ExamplePassword\fR +\fBpihole setpassword ExamplePassword\fR .br Change the password to "ExamplePassword" .br From 3ac426b5d183e412c57673cef91b919bb78efac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 16 Dec 2024 12:42:07 +0100 Subject: [PATCH 362/413] Update manpage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- manpages/pihole.8 | 77 ++++++++++++++++++++--------------------------- pihole | 1 + 2 files changed, 34 insertions(+), 44 deletions(-) diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 5a312efe..97a6ec68 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -5,13 +5,15 @@ Pi-hole : A black-hole for internet advertisements .br .SH "SYNOPSIS" -\fBpihole\fR (\fB-w\fR|\fB-b\fR|\fB--wild\fR|\fB--regex\fR) [options] domain(s) +\fBpihole\fR (\fB-allow\fR|\fB-deny\fR) [options] domain(s) +.br +\fBpihole\fR (\fB--allow-regex\fR|\fB--regex\fR) [options] domain(s) +.br +\fBpihole\fR (\fB--allow-wild\fR|\fB--wild\fR) [options] domain(s) .br \fBpihole setpassword\fR password .br -\fBpihole -c\fR [-j|-r|-e] -.br -\fBpihole\fR \fB-d\fR [-a] +\fBpihole\fR \fB-d\fR [-a] [-c] .br \fBpihole -f .br @@ -19,7 +21,7 @@ pihole -r .br \fBpihole\fR \fB-t\fR [arg] .br -pihole -g\fR +\fBpihole -g\fR .br \fBpihole\fR -\fBq\fR [options] .br @@ -27,20 +29,22 @@ pihole -g\fR .br \fBpihole -up \fR[--check-only] .br -\fBpihole -v\fR [-p|-a|-f] [-c|-l|-hash] +\fBpihole -v\fR .br -\fBpihole uninstall +\fBpihole uninstall\fR .br -pihole status +\fBpihole status\fR .br -pihole reloaddns\fR +\fBpihole reloaddns\fR .br -pihole reloadlists\fR +\fBpihole reloadlists\fR .br \fBpihole\fR (\fBenable\fR|\fBdisable\fR [time]) .br \fBpihole\fR \fBcheckout\fR repo [branch] .br +\fBpihole\fR \api\fR endpoint +.br \fBpihole\fR \fBhelp\fR .br .SH "DESCRIPTION" @@ -93,6 +97,7 @@ Available commands and options: .br -a Enable automated debugging + -c Include a Pi-hole database integrity check .br \fB-f, flush\fR @@ -126,11 +131,9 @@ Available commands and options: (Query options): .br - -adlist Print the name of the block list URL + -partial Search the adlists for partially matching domains .br - -exact Search the block lists for exact domain matches -.br - -all Return all query matches within a block list + -all Return all query matches within a adlists .br \fB-h, --help, help\fR @@ -161,31 +164,11 @@ Available commands and options: --check-only Exit script before update is performed. .br -\fB-v, version\fR [repo] [options] +\fB-v, version\fR .br Show installed versions of Pi-hole, Web Interface & FTL .br -.br - (repo options): -.br - -p, --pihole Only retrieve info regarding Pi-hole repository -.br - -a, --admin Only retrieve info regarding web - repository -.br - -f, --ftl Only retrieve info regarding FTL repository -.br - (version options): -.br - -c, --current Return the current version -.br - -l, --latest Return the latest version -.br - --hash Return the GitHub hash from your local - repositories -.br - \fBsetpassword\fR .br Set Web Interface password @@ -201,14 +184,14 @@ Available commands and options: Display the running status of Pi-hole subsystems .br -\fBenable\fR +\fBenable\fR [time] .br - Enable Pi-hole subsystems + Enable Pi-hole blocking, optionally for a set duration .br \fBdisable\fR [time] .br - Disable Pi-hole subsystems, optionally for a set duration + Disable Pi-hole blocking, optionally for a set duration .br (time options): @@ -250,6 +233,11 @@ Available commands and options: .br branchname Update subsystems to the specified branchname .br + +\fBapi\fR endpoint +.br + Query the Pi-hole API at +.br .SH "EXAMPLE" Some usage examples @@ -263,7 +251,7 @@ Allow-/denylist manipulation Allow "iloveads.example.com" .br -\fBpihole deny not noads.example.com\fR +\fBpihole deny remove noads.example.com\fR .br Removes "noads.example.com" from denylist .br @@ -299,9 +287,9 @@ Updating lists from internet sources Displaying version information .br -\fBpihole -v -a -c\fR +\fBpihole -v\fR .br - Display the current version of web + Display the current version of Pi-hole .br Temporarily disabling Pi-hole @@ -330,10 +318,11 @@ Switching Pi-hole subsystem branches Flush information stored in Pi-hole's network tables .br -.SH "SEE ALSO" - -\fBlighttpd\fR(8), \fBpihole-FTL\fR(8) +\fBpihole api stats/summary\fR .br + Queries FTL for the stats/summary endpoint +.br + .SH "COLOPHON" Get sucked into the latest news and community activity by entering Pi-hole's orbit. Information about Pi-hole, and the latest version of the software can be found at https://pi-hole.net. diff --git a/pihole b/pihole index 5657637b..f3e14ac6 100755 --- a/pihole +++ b/pihole @@ -501,6 +501,7 @@ Options: uninstall Uninstall Pi-hole from your system status Display the running status of Pi-hole subsystems enable Enable Pi-hole subsystems + Add '-h' for more info on enable usage disable Disable Pi-hole subsystems Add '-h' for more info on disable usage reloaddns Update the lists and flush the cache without restarting the DNS server From 7dc542f3c294539a5349b0d90b6e9529ee6c6ad0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 16 Dec 2024 12:51:23 +0100 Subject: [PATCH 363/413] Get WEBPORT from pihole.toml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ef1e372a..bff59bb8 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -91,7 +91,7 @@ IPV4_ADDRESS=${IPV4_ADDRESS} IPV6_ADDRESS=${IPV6_ADDRESS} # Give settings their default values. These may be changed by prompts later in the script. QUERY_LOGGING= -WEBPORT=8080 +WEBPORT= PRIVACY_LEVEL= # Where old configs go to if a v6 migration is performed @@ -2498,6 +2498,9 @@ main() { fi if [[ "${useUpdateVars}" == false ]]; then + # Get the Web interface port, return only the first port + WEBPORT=$(getFTLConfigValue webserver.port|cut -d, -f1) + # Display the completion dialog displayFinalMessage "${pw}" From b2a556468534692c887d55490810f01147db2de7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 16 Dec 2024 13:06:25 +0100 Subject: [PATCH 364/413] FTL allows non-numieric port notation - we need to strip this MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index bff59bb8..20051b83 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2498,8 +2498,8 @@ main() { fi if [[ "${useUpdateVars}" == false ]]; then - # Get the Web interface port, return only the first port - WEBPORT=$(getFTLConfigValue webserver.port|cut -d, -f1) + # Get the Web interface port, return only the first port and strip all non-numeric characters + WEBPORT=$(getFTLConfigValue webserver.port|cut -d, -f1 | tr -cd '0-9') # Display the completion dialog displayFinalMessage "${pw}" From 83224e7729e91849b9b8f8506c33e85061f78d6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 16 Dec 2024 09:30:25 +0100 Subject: [PATCH 365/413] Improve dependency package output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 49 ++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f1ef8887..976bcf72 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -480,7 +480,16 @@ build_dependency_package(){ echo "${PIHOLE_META_PACKAGE_CONTROL_APT}" > "${tempdir}"/DEBIAN/control # Build the package - dpkg-deb --build --root-owner-group "${tempdir}" pihole-meta.deb + local str="Building dependency package pihole-meta.deb" + printf " %b %s..." "${INFO}" "${str}" + + if dpkg-deb --build --root-owner-group "${tempdir}" pihole-meta.deb &>/dev/null; then + printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" + else + printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" + printf "%b Error: Building pihole-meta.deb failed. %b\\n" "${COL_LIGHT_RED}" "${COL_NC}" + return 1 + fi # Move back into the directory the user started in popd &> /dev/null || return 1 @@ -504,7 +513,16 @@ build_dependency_package(){ fi # Build the package - rpmbuild -bb "${tempdir}"/SPECS/pihole-meta.spec --define "_topdir ${tempdir}" + local str="Building dependency package pihole-meta.rpm" + printf " %b %s..." "${INFO}" "${str}" + + if rpmbuild -bb "${tempdir}"/SPECS/pihole-meta.spec --define "_topdir ${tempdir}" &>/dev/null; then + printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" + else + printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" + printf "%b Error: Building pihole-meta.rpm failed. %b\\n" "${COL_LIGHT_RED}" "${COL_NC}" + return 1 + fi # Move the package to the /tmp directory mv "${tempdir}"/RPMS/noarch/pihole-meta*.rpm /tmp/pihole-meta.rpm @@ -1507,23 +1525,37 @@ notify_package_updates_available() { install_dependent_packages() { # Install meta dependency package + local str="Installing Pi-hole dependency package" + printf " %b %s..." "${INFO}" "${str}" # Install Debian/Ubuntu packages if is_command apt-get; then if [ -f /tmp/pihole-meta.deb ]; then - eval "${PKG_INSTALL}" "/tmp/pihole-meta.deb" - rm /tmp/pihole-meta.deb + if eval "${PKG_INSTALL}" "/tmp/pihole-meta.deb" &>/dev/null; then + printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" + rm /tmp/pihole-meta.deb + else + printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" + printf " %b Error: Unable to install Pi-hole dependency package.\\n" "${COL_LIGHT_RED}" + return 1 + fi else - printf " %b Error: Unable to find Pi-hole dependency meta package.\\n" "${COL_LIGHT_RED}" + printf " %b Error: Unable to find Pi-hole dependency package.\\n" "${COL_LIGHT_RED}" return 1 fi # Install Fedora/CentOS packages elif is_command rpm; then if [ -f /tmp/pihole-meta.rpm ]; then - eval "${PKG_INSTALL}" "/tmp/pihole-meta.rpm" - rm /tmp/pihole-meta.rpm + if eval "${PKG_INSTALL}" "/tmp/pihole-meta.rpm" &>/dev/null; then + printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" + rm /tmp/pihole-meta.rpm + else + printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" + printf " %b Error: Unable to install Pi-hole dependency package.\\n" "${COL_LIGHT_RED}" + return 1 + fi else - printf " %b Error: Unable to find Pi-hole dependency meta package.\\n" "${COL_LIGHT_RED}" + printf " %b Error: Unable to find Pi-hole dependency package.\\n" "${COL_LIGHT_RED}" return 1 fi @@ -2354,7 +2386,6 @@ main() { build_dependency_package # Install Pi-hole dependencies - printf " %b Installing required dependencies ...\\n" "${INFO}" install_dependent_packages # Check that the installed OS is officially supported - display warning if not From 4f268ed193f2b552b7518b64b531ff0b1a7654e3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 21 Dec 2024 07:10:46 +0100 Subject: [PATCH 366/413] Explicitly migrate from v5 to v6 Signed-off-by: DL6ER --- automated install/basic-install.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 755d8dd9..863ac432 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2326,6 +2326,12 @@ migrate_dnsmasq_configs() { mv /etc/dnsmasq.d/0{1,2,4,5}-pihole*.conf "${V6_CONF_MIGRATION_DIR}/" 2>/dev/null || true mv /etc/dnsmasq.d/06-rfc6761.conf "${V6_CONF_MIGRATION_DIR}/" 2>/dev/null || true + + # Finally, after everything is in place, we can create the new config file + # /etc/pihole/pihole.toml + # This file will be created with the default settings unless the user has + # changed settings via setupVars.conf or the other dnsmasq files moved above + pihole-FTL migrate v6 } main() { From 92a3c73f801c358fbfa015fd908ef2802f7a64a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 22 Dec 2024 21:46:32 +0100 Subject: [PATCH 367/413] Fix v5 -> v6 update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 755d8dd9..c5d2ad9c 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2401,8 +2401,8 @@ main() { exit 1 fi - # in case of an update - if [[ -f "${PI_HOLE_V6_CONFIG}" ]]; then + # in case of an update (can be a v5 -> v6 or v6 -> v6 update) + if [[ -f "${PI_HOLE_V6_CONFIG}" ]] || [[ -f "/etc/pihole/setupVars.conf" ]]; then # if it's running unattended, if [[ "${runUnattended}" == true ]]; then printf " %b Performing unattended setup, no dialogs will be displayed\\n" "${INFO}" From c777152c04b8cc1f8a1b07e05bdd61d38180fb38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 22 Dec 2024 21:55:25 +0100 Subject: [PATCH 368/413] Only separat data and status when needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/api.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 43c91d69..785b8309 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -165,15 +165,16 @@ GetFTLData() { # get the data from querying the API as well as the http status code response=$(curl -skS -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" ) - # status are the last 3 characters - status="${response#"${response%???}"}" - # data is everything from response without the last 3 characters - data="${response%???}" - if [ "${2}" = "raw" ]; then # return the raw response echo "${response}" else + + # status are the last 3 characters # status are the last 3 characters + status="${response#"${response%???}"}" + # data is everything from response without the last 3 characters + data="${response%???}" + # return only the data if [ "${status}" = 200 ]; then # response OK From 3011d48b6f22996f36b2e72cad633484e9e3d0b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 22 Dec 2024 23:30:14 +0100 Subject: [PATCH 369/413] Use tail instead of parameter expansion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/api.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 785b8309..c9c2dd49 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -170,8 +170,9 @@ GetFTLData() { echo "${response}" else - # status are the last 3 characters # status are the last 3 characters - status="${response#"${response%???}"}" + # status are the last 3 characters + # not using ${response#"${response%???}"}" here because it's extremely slow on big responses + status=$(printf "%s" "${response}" | tail -c 3) # data is everything from response without the last 3 characters data="${response%???}" @@ -265,7 +266,8 @@ apiFunc() { response=$(GetFTLData "$1" raw) # status are the last 3 characters - status="${response#"${response%???}"}" + # not using ${response#"${response%???}"}" here because it's extremely slow on big responses + status=$(printf "%s" "${response}" | tail -c 3) # data is everything from response without the last 3 characters data="${response%???}" From 8faa28d8a01cc5e8b67058502af25ba53a3bfa0f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 23 Dec 2024 08:48:05 +0100 Subject: [PATCH 370/413] Explicitly log migration process Signed-off-by: DL6ER --- automated install/basic-install.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 863ac432..c6ac0fbf 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2331,7 +2331,20 @@ migrate_dnsmasq_configs() { # /etc/pihole/pihole.toml # This file will be created with the default settings unless the user has # changed settings via setupVars.conf or the other dnsmasq files moved above - pihole-FTL migrate v6 + str="Migrating Pi-hole configuration to version 6" + printf " %b %s...\\n" "${INFO}" + local FTLoutput FTLstatus + FTLoutput=$(pihole-FTL migrate v6) + FTLstatus=$? + if [[ "${FTLstatus}" -eq 0 ]]; then + printf " %b %s\\n" "${TICK}" "${str}" + else + printf " %b %s\\n" "${CROSS}" "${str}" + fi + + # Print the output of the FTL migration prefacing every line with four + # spaces for alignment + printf "%b" "${FTLoutput}" | sed 's/^/ /' } main() { From a2a22c4e138398499d649135ccc7c53a3fcf019a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 23 Dec 2024 09:13:34 +0100 Subject: [PATCH 371/413] Exit early when neither service nor systemctl commands are available Signed-off-by: DL6ER --- automated install/basic-install.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index c5d2ad9c..368fa6c0 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2328,6 +2328,17 @@ migrate_dnsmasq_configs() { mv /etc/dnsmasq.d/06-rfc6761.conf "${V6_CONF_MIGRATION_DIR}/" 2>/dev/null || true } +# Check for availability of either the "service" or "systemctl" commands +check_service_command() { + # Check for the availability of the "service" command + if ! is_command service && ! is_command systemctl; then + # If neither the "service" nor the "systemctl" command is available, inform the user + printf " %b Neither the service nor the systemctl commands are available\\n" "${CROSS}" + printf " on this machine. This Pi-hole installer cannot continue.\\n" + exit 1 + fi +} + main() { ######## FIRST CHECK ######## # Must be root to install @@ -2376,6 +2387,9 @@ main() { # Check if SELinux is Enforcing and exit before doing anything else checkSelinux + # Check for availability of either the "service" or "systemctl" commands + check_service_command + # Check for supported package managers so that we may install dependencies package_manager_detect From e08f65d1c4ca40de27df124ac378736a3cf0f189 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 23 Dec 2024 09:30:37 +0100 Subject: [PATCH 372/413] Disable lighttpd if found Signed-off-by: DL6ER --- automated install/basic-install.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index c5d2ad9c..7411b2c5 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2300,6 +2300,21 @@ copy_to_install_log() { chown pihole:pihole "${installLogLoc}" } +disableLighttpd() { + # Lighttpd is not needed anymore, so disable it + # We keep all the configuration files in place, so the user can re-enable it + # if needed + + # Check if lighttpd is installed + if is_command lighttpd; then + # Stop the lighttpd service + stop_service lighttpd + + # Disable the lighttpd service + disable_service lighttpd + fi +} + migrate_dnsmasq_configs() { # Previously, Pi-hole created a number of files in /etc/dnsmasq.d # During migration, their content is copied into the new single source of @@ -2489,6 +2504,9 @@ main() { # but before starting or resttarting the ftl service disable_resolved_stublistener + # Disable lighttpd server + disableLighttpd + # Check if gravity database needs to be upgraded. If so, do it without rebuilding # gravity altogether. This may be a very long running task needlessly blocking # the update process. From 47d5a085652e05e4b0a0fbafe9393ff3112fce9e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 23 Dec 2024 19:12:47 +0100 Subject: [PATCH 373/413] Ask the user if they want to disable lighttpd Signed-off-by: DL6ER --- automated install/basic-install.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 7411b2c5..fc2b28ef 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2301,6 +2301,24 @@ copy_to_install_log() { } disableLighttpd() { + # Detect if the terminal is interactive + if [[ -t 0 ]]; then + # The terminal is interactive + dialog --no-shadow --keep-tite \ + --title "Pi-hole v6.0 does no longer need lighttpd" \ + --yesno "Pi-hole v6.0 has its own embedded web server so lighttpd is no longer needed *unless* you have custom configurations. In this case, you can opt-out of disabling lighttpd and pihole-FTL will try to bind to an alternative port such as 8080.\\n\\nDo you want to disable lighttpd (recommended)?" "${r}" "${c}" + response=$? + else + # The terminal is non-interactive, assume yes. Lighttpd will be stopped + # but keeps being installed and can easily be re-enabled by the user + response=0 + fi + + # If the user does not want to disable lighttpd, return early + if [[ "${response}" -ne 0 ]]; then + return + fi + # Lighttpd is not needed anymore, so disable it # We keep all the configuration files in place, so the user can re-enable it # if needed From d14273f958fa4b73b8cf321a936edd7e5364c035 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 24 Dec 2024 02:17:50 +0100 Subject: [PATCH 374/413] Add comment where setupVars.conf goes when v6 migration is done Signed-off-by: DL6ER --- automated install/basic-install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index c6ac0fbf..6d97d7e0 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2331,6 +2331,7 @@ migrate_dnsmasq_configs() { # /etc/pihole/pihole.toml # This file will be created with the default settings unless the user has # changed settings via setupVars.conf or the other dnsmasq files moved above + # During migration, setupVars.conf is moved to /etc/pihole/migration_backup_v6 str="Migrating Pi-hole configuration to version 6" printf " %b %s...\\n" "${INFO}" local FTLoutput FTLstatus From 50645c2924d8eb92f8b0e4a3dec5fdd762ba617f Mon Sep 17 00:00:00 2001 From: Dominik Date: Tue, 24 Dec 2024 02:19:37 +0100 Subject: [PATCH 375/413] Apply suggestions from code review Co-authored-by: Adam Warner Co-authored-by: RD WebDesign Signed-off-by: Dominik --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index fc2b28ef..13606758 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2305,8 +2305,8 @@ disableLighttpd() { if [[ -t 0 ]]; then # The terminal is interactive dialog --no-shadow --keep-tite \ - --title "Pi-hole v6.0 does no longer need lighttpd" \ - --yesno "Pi-hole v6.0 has its own embedded web server so lighttpd is no longer needed *unless* you have custom configurations. In this case, you can opt-out of disabling lighttpd and pihole-FTL will try to bind to an alternative port such as 8080.\\n\\nDo you want to disable lighttpd (recommended)?" "${r}" "${c}" + --title "Pi-hole v6.0 no longer uses lighttpd" \ + --yesno "\\n\\nPi-hole v6.0 has its own embedded web server so lighttpd is no longer needed *unless* you have custom configurations.\\n\\nIn this case, you can opt-out of disabling lighttpd and pihole-FTL will try to bind to an alternative port such as 8080.\\n\\nDo you want to disable lighttpd (recommended)?" "${r}" "${c}" response=$? else # The terminal is non-interactive, assume yes. Lighttpd will be stopped From 18358273718b7186d3f417d4b4c32375c75b6ab4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 30 Dec 2024 15:35:41 +0100 Subject: [PATCH 376/413] Ensure Yes to keeping lighttpd doesn't trigger set -e during the upgrade Signed-off-by: DL6ER --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 13606758..85587bb3 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2301,13 +2301,13 @@ copy_to_install_log() { } disableLighttpd() { + local response # Detect if the terminal is interactive if [[ -t 0 ]]; then # The terminal is interactive dialog --no-shadow --keep-tite \ --title "Pi-hole v6.0 no longer uses lighttpd" \ - --yesno "\\n\\nPi-hole v6.0 has its own embedded web server so lighttpd is no longer needed *unless* you have custom configurations.\\n\\nIn this case, you can opt-out of disabling lighttpd and pihole-FTL will try to bind to an alternative port such as 8080.\\n\\nDo you want to disable lighttpd (recommended)?" "${r}" "${c}" - response=$? + --yesno "\\n\\nPi-hole v6.0 has its own embedded web server so lighttpd is no longer needed *unless* you have custom configurations.\\n\\nIn this case, you can opt-out of disabling lighttpd and pihole-FTL will try to bind to an alternative port such as 8080.\\n\\nDo you want to disable lighttpd (recommended)?" "${r}" "${c}" && response=0 || response="$?" else # The terminal is non-interactive, assume yes. Lighttpd will be stopped # but keeps being installed and can easily be re-enabled by the user From bd9cb00574be123eb910d12bea5ea5b9dbf5df4b Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 31 Dec 2024 15:31:04 +0000 Subject: [PATCH 377/413] Perform os_check during update, in case the user has updated to an unsupported OS between Pi-hole updates. Signed-off-by: Adam Warner --- advanced/Scripts/update.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index 4f32382e..9ea63b4c 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -107,6 +107,9 @@ main() { web_update=false FTL_update=false + # Perform an OS check to ensure we're on an appropriate operating system + os_check + # Install packages used by this installation script (necessary if users have removed e.g. git from their systems) package_manager_detect build_dependency_package From 4fd77f57faead42a8d8accc3a9d59e80f92e957d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 31 Dec 2024 16:42:08 +0100 Subject: [PATCH 378/413] Remove now unused function test_dpkg_lock() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index b1608c70..c7d479d0 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -388,28 +388,6 @@ os_check() { fi } -# This function waits for dpkg to unlock, which signals that the previous apt-get command has finished. -test_dpkg_lock() { - i=0 - printf " %b Waiting for package manager to finish (up to 30 seconds)\\n" "${INFO}" - # fuser is a program to show which processes use the named files, sockets, or filesystems - # So while the lock is held, - while fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - # we wait half a second, - sleep 0.5 - # increase the iterator, - ((i = i + 1)) - # exit if waiting for more then 30 seconds - if [[ $i -gt 60 ]]; then - printf " %b %bError: Could not verify package manager finished and released lock. %b\\n" "${CROSS}" "${COL_LIGHT_RED}" "${COL_NC}" - printf " Attempt to install packages manually and retry.\\n" - exit 1 - fi - done - # and then report success once dpkg is unlocked. - return 0 -} - # Compatibility package_manager_detect() { From 79087fb252c1f88fae1d1b9eaeabea7830006672 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 Jan 2025 18:55:49 +0100 Subject: [PATCH 379/413] SHow lighttpd disable dialog only once during v6 migration and when lighttpd is actually installed Signed-off-by: DL6ER --- automated install/basic-install.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index b1608c70..88369165 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2301,6 +2301,11 @@ copy_to_install_log() { } disableLighttpd() { + # Return early when lighttpd is not installed + if [[ ! -f /etc/lighttpd/lighttpd.conf ]]; then + return + fi + local response # Detect if the terminal is interactive if [[ -t 0 ]]; then @@ -2345,6 +2350,9 @@ migrate_dnsmasq_configs() { return 0 fi + # Disable lighttpd server during v6 migration + disableLighttpd + # Create target directory /etc/pihole/migration_backup_v6 # and make it owned by pihole:pihole mkdir -p "${V6_CONF_MIGRATION_DIR}" @@ -2536,9 +2544,6 @@ main() { # but before starting or resttarting the ftl service disable_resolved_stublistener - # Disable lighttpd server - disableLighttpd - # Check if gravity database needs to be upgraded. If so, do it without rebuilding # gravity altogether. This may be a very long running task needlessly blocking # the update process. From 364281354c75eceb1d1eec88687cf0bd7debca64 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 2 Jan 2025 13:43:32 +0100 Subject: [PATCH 380/413] Check if service is currently enabled when checking if lighttpd may need to be disabled Signed-off-by: DL6ER --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 88369165..ea8e210e 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2301,8 +2301,8 @@ copy_to_install_log() { } disableLighttpd() { - # Return early when lighttpd is not installed - if [[ ! -f /etc/lighttpd/lighttpd.conf ]]; then + # Return early when lighttpd is not active + if ! check_service_active lighttpd; then return fi From 4d23187f864c143e526af069ae899ffdf6f891ac Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 6 Jan 2025 18:33:23 +0100 Subject: [PATCH 381/413] Add missing newline Signed-off-by: DL6ER --- automated install/basic-install.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index cb7324f5..7afdc313 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2365,6 +2365,9 @@ migrate_dnsmasq_configs() { # Print the output of the FTL migration prefacing every line with four # spaces for alignment printf "%b" "${FTLoutput}" | sed 's/^/ /' + + # Print a blank line for separation + printf "\\n" } # Check for availability of either the "service" or "systemctl" commands From 211a88188827d290c01b3441891047f1a911be9a Mon Sep 17 00:00:00 2001 From: Dominik Date: Wed, 8 Jan 2025 19:28:15 +0100 Subject: [PATCH 382/413] Apply suggestions from code review Co-authored-by: yubiuser Signed-off-by: Dominik --- automated install/basic-install.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 7afdc313..a274f22d 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2352,14 +2352,14 @@ migrate_dnsmasq_configs() { # changed settings via setupVars.conf or the other dnsmasq files moved above # During migration, setupVars.conf is moved to /etc/pihole/migration_backup_v6 str="Migrating Pi-hole configuration to version 6" - printf " %b %s...\\n" "${INFO}" + printf " %b %s...\\n" "${INFO}" "${str}" local FTLoutput FTLstatus FTLoutput=$(pihole-FTL migrate v6) FTLstatus=$? if [[ "${FTLstatus}" -eq 0 ]]; then - printf " %b %s\\n" "${TICK}" "${str}" + printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" else - printf " %b %s\\n" "${CROSS}" "${str}" + printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" fi # Print the output of the FTL migration prefacing every line with four From 4be130971e0f9e5d09c7747204ff17b70bcf554e Mon Sep 17 00:00:00 2001 From: Dominik Date: Thu, 9 Jan 2025 18:32:04 +0100 Subject: [PATCH 383/413] Update automated install/basic-install.sh Co-authored-by: yubiuser Signed-off-by: Dominik --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a274f22d..719a57fc 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2352,7 +2352,7 @@ migrate_dnsmasq_configs() { # changed settings via setupVars.conf or the other dnsmasq files moved above # During migration, setupVars.conf is moved to /etc/pihole/migration_backup_v6 str="Migrating Pi-hole configuration to version 6" - printf " %b %s...\\n" "${INFO}" "${str}" + printf " %b %s..." "${INFO}" "${str}" local FTLoutput FTLstatus FTLoutput=$(pihole-FTL migrate v6) FTLstatus=$? From bd9cc7cbb77d965d4b52015e1487872cfc7754fb Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Thu, 9 Jan 2025 16:06:38 -0300 Subject: [PATCH 384/413] Clean up unused WEBCALL code Debug Log is no longer available via web interface Signed-off-by: RD WebDesign --- advanced/Scripts/piholeDebug.sh | 32 ++++++++++++-------------------- pihole | 4 +--- 2 files changed, 13 insertions(+), 23 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 0a15972c..63d8c50b 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1317,19 +1317,16 @@ upload_to_tricorder() { curl_to_tricorder # If we're not running in automated mode, else - # if not being called from the web interface - if [[ ! "${WEBCALL}" ]]; then - echo "" - # give the user a choice of uploading it or not - # Users can review the log file locally (or the output of the script since they are the same) and try to self-diagnose their problem - read -r -p "[?] Would you like to upload the log? [y/N] " response - case ${response} in - # If they say yes, run our function for uploading the log - [yY][eE][sS]|[yY]) curl_to_tricorder;; - # If they choose no, just exit out of the script - *) log_write " * Log will ${COL_GREEN}NOT${COL_NC} be uploaded to tricorder.\\n * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG}${COL_NC}\\n";exit; - esac - fi + echo "" + # give the user a choice of uploading it or not + # Users can review the log file locally (or the output of the script since they are the same) and try to self-diagnose their problem + read -r -p "[?] Would you like to upload the log? [y/N] " response + case ${response} in + # If they say yes, run our function for uploading the log + [yY][eE][sS]|[yY]) curl_to_tricorder;; + # If they choose no, just exit out of the script + *) log_write " * Log will ${COL_GREEN}NOT${COL_NC} be uploaded to tricorder.\\n * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG}${COL_NC}\\n";exit; + esac fi # Check if tricorder.pi-hole.net is reachable and provide token # along with some additional useful information @@ -1349,13 +1346,8 @@ upload_to_tricorder() { # If no token was generated else # Show an error and some help instructions - # Skip this if being called from web interface and automatic mode was not chosen (users opt-out to upload) - if [[ "${WEBCALL}" ]] && [[ ! "${AUTOMATED}" ]]; then - : - else - log_write "${CROSS} ${COL_RED}There was an error uploading your debug log.${COL_NC}" - log_write " * Please try again or contact the Pi-hole team for assistance." - fi + log_write "${CROSS} ${COL_RED}There was an error uploading your debug log.${COL_NC}" + log_write " * Please try again or contact the Pi-hole team for assistance." fi # Finally, show where the log file is no matter the outcome of the function so users can look at it log_write " * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG}${COL_NC}\\n" diff --git a/pihole b/pihole index f3e14ac6..6424c793 100755 --- a/pihole +++ b/pihole @@ -73,19 +73,17 @@ listFunc() { debugFunc() { local automated - local web local check_database_integrity # Pull off the `debug` leaving passed call augmentation flags in $1 shift for value in "$@"; do [[ "$value" == *"-a"* ]] && automated="true" - [[ "$value" == *"-w"* ]] && web="true" [[ "$value" == *"-c"* ]] && check_database_integrity="true" [[ "$value" == *"--check_database"* ]] && check_database_integrity="true" done - AUTOMATED=${automated:-} WEBCALL=${web:-} CHECK_DATABASE=${check_database_integrity:-} "${PI_HOLE_SCRIPT_DIR}"/piholeDebug.sh + AUTOMATED=${automated:-} CHECK_DATABASE=${check_database_integrity:-} "${PI_HOLE_SCRIPT_DIR}"/piholeDebug.sh exit 0 } From 733dce1c4c4626cfa99ca0ac62a4114396969b83 Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Thu, 9 Jan 2025 17:34:05 -0300 Subject: [PATCH 385/413] If it's a docker container, consider the OS is supported Signed-off-by: RD WebDesign --- advanced/Scripts/piholeDebug.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 63d8c50b..1307382c 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -346,6 +346,9 @@ os_check() { fi done + # If it is a docker container, we can assume the OS is supported + [ -n "${DOCKER_VERSION}" ] && valid_os=true && valid_version=true + local finalmsg if [ "$valid_os" = true ]; then log_write "${TICK} Distro: ${COL_GREEN}${detected_os^}${COL_NC}" From 0590bbba2d17b9153640c8bf0fb7e63b266191b2 Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Thu, 9 Jan 2025 18:06:06 -0300 Subject: [PATCH 386/413] Skip some checks in docker container These commands always fail in containers with "command not found" (as expected). Signed-off-by: RD WebDesign --- advanced/Scripts/piholeDebug.sh | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 1307382c..b2ae94ee 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -492,13 +492,25 @@ run_and_print_command() { } hardware_check() { + # Note: the checks are skipped if Pi-hole is running in a docker container + + local skip_msg="${INFO} Not enough permissions inside Docker container ${COL_YELLOW}(skipped)${COL_NC}" + echo_current_diagnostic "System hardware configuration" - # Store the output of the command in a variable - run_and_print_command "lshw -short" + if [ -n "${DOCKER_VERSION}" ]; then + log_write "${skip_msg}" + else + # Store the output of the command in a variable + run_and_print_command "lshw -short" + fi echo_current_diagnostic "Processor details" - # Store the output of the command in a variable - run_and_print_command "lscpu" + if [ -n "${DOCKER_VERSION}" ]; then + log_write "${skip_msg}" + else + # Store the output of the command in a variable + run_and_print_command "lscpu" + fi } disk_usage() { From c78be8a89df47bcfbe3b2cd0e816b54ee0be1af1 Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Thu, 9 Jan 2025 18:49:08 -0300 Subject: [PATCH 387/413] No systemctl or service inside containers Signed-off-by: RD WebDesign --- advanced/Scripts/piholeDebug.sh | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index b2ae94ee..08b9bc96 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -823,26 +823,24 @@ dig_at() { process_status(){ # Check to make sure Pi-hole's services are running and active echo_current_diagnostic "Pi-hole processes" + # Local iterator local i + # For each process, for i in "${PIHOLE_PROCESSES[@]}"; do + local status_of_process + # If systemd if command -v systemctl &> /dev/null; then # get its status via systemctl - local status_of_process status_of_process=$(systemctl is-active "${i}") else # Otherwise, use the service command and mock the output of `systemctl is-active` - local status_of_process - # If DOCKER_VERSION is set, the output is slightly different (s6 init system on Docker) + # If it is a docker container, there is no systemctl or service. Do nothing. if [ -n "${DOCKER_VERSION}" ]; then - if service "${i}" status | grep -E '^up' &> /dev/null; then - status_of_process="active" - else - status_of_process="inactive" - fi + : else # non-Docker system if service "${i}" status | grep -E 'is\srunning' &> /dev/null; then @@ -852,8 +850,12 @@ process_status(){ fi fi fi + # and print it out to the user - if [[ "${status_of_process}" == "active" ]]; then + if [ -n "${DOCKER_VERSION}" ]; then + # If it's a Docker container, the test was skipped + log_write "${INFO} systemctl/service not installed inside docker container ${COL_YELLOW}(skipped)${COL_NC}" + elif [[ "${status_of_process}" == "active" ]]; then # If it's active, show it in green log_write "${TICK} ${COL_GREEN}${i}${COL_NC} daemon is ${COL_GREEN}${status_of_process}${COL_NC}" else @@ -870,6 +872,8 @@ ftl_full_status(){ if command -v systemctl &> /dev/null; then FTL_status=$(systemctl status --full --no-pager pihole-FTL.service) log_write " ${FTL_status}" + elif [ -n "${DOCKER_VERSION}" ]; then + log_write "${INFO} systemctl/service not installed inside docker container ${COL_YELLOW}(skipped)${COL_NC}" else log_write "${INFO} systemctl: command not found" fi From 27fb2bbe3a4bd0c13ae841f6f901b529434a37dd Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Thu, 9 Jan 2025 17:32:01 -0300 Subject: [PATCH 388/413] Replace some hardcoded paths with pihole.toml values Signed-off-by: RD WebDesign --- advanced/Scripts/piholeDebug.sh | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 08b9bc96..d0d61281 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -44,6 +44,14 @@ fi # shellcheck disable=SC1091 . /etc/pihole/versions +# Read the value of an FTL config key. The value is printed to stdout. +get_ftl_conf_value() { + local key=$1 + + # Obtain setting from FTL directly + pihole-FTL --config "${key}" +} + # FAQ URLs for use in showing the debug log FAQ_HARDWARE_REQUIREMENTS="${COL_CYAN}https://docs.pi-hole.net/main/prerequisites/${COL_NC}" FAQ_HARDWARE_REQUIREMENTS_PORTS="${COL_CYAN}https://docs.pi-hole.net/main/prerequisites/#ports${COL_NC}" @@ -61,10 +69,10 @@ DNSMASQ_D_DIRECTORY="/etc/dnsmasq.d" PIHOLE_DIRECTORY="/etc/pihole" PIHOLE_SCRIPTS_DIRECTORY="/opt/pihole" BIN_DIRECTORY="/usr/local/bin" -RUN_DIRECTORY="/run" LOG_DIRECTORY="/var/log/pihole" -HTML_DIRECTORY="/var/www/html" -WEB_GIT_DIRECTORY="${HTML_DIRECTORY}/admin" +HTML_DIRECTORY="$(get_ftl_conf_value "webserver.paths.webroot")" +WEBHOME_PATH="$(get_ftl_conf_value "webserver.paths.webhome")" +WEB_GIT_DIRECTORY="${HTML_DIRECTORY}${WEBHOME_PATH}" SHM_DIRECTORY="/dev/shm" ETC="/etc" @@ -79,14 +87,6 @@ PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml" PIHOLE_DNSMASQ_CONF_FILE="${PIHOLE_DIRECTORY}/dnsmasq.conf" PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" -# Read the value of an FTL config key. The value is printed to stdout. -get_ftl_conf_value() { - local key=$1 - - # Obtain setting from FTL directly - pihole-FTL --config "${key}" -} - PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "files.gravity")" PIHOLE_FTL_DB_FILE="$(get_ftl_conf_value "files.database")" @@ -94,7 +94,7 @@ PIHOLE_FTL_DB_FILE="$(get_ftl_conf_value "files.database")" PIHOLE_COMMAND="${BIN_DIRECTORY}/pihole" PIHOLE_COLTABLE_FILE="${BIN_DIRECTORY}/COL_TABLE" -FTL_PID="${RUN_DIRECTORY}/pihole-FTL.pid" +FTL_PID="$(get_ftl_conf_value "files.pid")" PIHOLE_LOG="${LOG_DIRECTORY}/pihole.log" PIHOLE_LOG_GZIPS="${LOG_DIRECTORY}/pihole.log.[0-9].*" From 83d464e85f16202a1189ff4ee449f0933b05990a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 11 Jan 2025 10:54:32 +0000 Subject: [PATCH 389/413] Bump eps1lon/actions-label-merge-conflict from 3.0.2 to 3.0.3 Bumps [eps1lon/actions-label-merge-conflict](https://github.com/eps1lon/actions-label-merge-conflict) from 3.0.2 to 3.0.3. - [Release notes](https://github.com/eps1lon/actions-label-merge-conflict/releases) - [Changelog](https://github.com/eps1lon/actions-label-merge-conflict/blob/main/CHANGELOG.md) - [Commits](https://github.com/eps1lon/actions-label-merge-conflict/compare/v3.0.2...v3.0.3) --- updated-dependencies: - dependency-name: eps1lon/actions-label-merge-conflict dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/merge-conflict.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge-conflict.yml b/.github/workflows/merge-conflict.yml index 4e56ae3e..f169ab6f 100644 --- a/.github/workflows/merge-conflict.yml +++ b/.github/workflows/merge-conflict.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check if PRs are have merge conflicts - uses: eps1lon/actions-label-merge-conflict@v3.0.2 + uses: eps1lon/actions-label-merge-conflict@v3.0.3 with: dirtyLabel: "PR: Merge Conflict" repoToken: "${{ secrets.GITHUB_TOKEN }}" From 12927f8a3d85a70983c04ed07a0f298ae7c94b6c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 12 Jan 2025 13:41:27 +0100 Subject: [PATCH 390/413] Put gravity backup files into dedicated directory for cleaness Signed-off-by: DL6ER --- gravity.sh | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/gravity.sh b/gravity.sh index ee00741a..0fe5690b 100755 --- a/gravity.sh +++ b/gravity.sh @@ -58,7 +58,8 @@ gravityDBfile_default="/etc/pihole/gravity.db" gravityTEMPfile="${GRAVITYDB}_temp" gravityDIR="$(dirname -- "${gravityDBfile}")" gravityOLDfile="${gravityDIR}/gravity_old.db" -gravityBCKfile="${gravityDIR}/gravity_backup.db" +gravityBCKdir="${gravityDIR}/gravity_backups" +gravityBCKfile="${gravityBCKdir}/gravity.db" fix_owner_permissions() { # Fix ownership and permissions for the specified file @@ -132,6 +133,11 @@ gravity_swap_databases() { if [[ "${status}" -ne 0 ]]; then echo -e "\\n ${CROSS} Unable to clean current database for backup\\n ${output}" else + # Check if the backup directory exists + if [ ! -d "${gravityBCKdir}" ]; then + mkdir -p "${gravityBCKdir}" + fi + # If multiple gravityBCKfile's are present (appended with a number), rotate them # We keep at most 10 backups rotate_gravity_backup @@ -434,11 +440,13 @@ gravity_DownloadBlocklists() { echo -e "\\n ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n ${output}" # Try to attempt a backup restore - for i in {1..10}; do - if try_restore_backup "${i}"; then - break - fi - done + if [[ -d "${gravityBCKdir}" ]]; then + for i in {1..10}; do + if try_restore_backup "${i}"; then + break + fi + done + fi # If none of the attempts worked, return 1 if [[ "${i}" -eq 10 ]]; then From cc25ee940e00d211c94cdbad1575c19eb634a35b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 12 Jan 2025 13:42:51 +0100 Subject: [PATCH 391/413] Fix unreachable code Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 0fe5690b..f35eb674 100755 --- a/gravity.sh +++ b/gravity.sh @@ -450,8 +450,8 @@ gravity_DownloadBlocklists() { # If none of the attempts worked, return 1 if [[ "${i}" -eq 10 ]]; then - return 1 pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) values ('gravity_restored','failed');" + return 1 fi echo -e " ${TICK} ${str}" From 0ae50eab879b3f6c5cfb50c28c47cd8881929c69 Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Sun, 12 Jan 2025 20:25:32 -0300 Subject: [PATCH 392/413] Suppress error message on `git describe` failure Signed-off-by: RD WebDesign --- advanced/Scripts/piholeDebug.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index d0d61281..80fe9a35 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -202,7 +202,7 @@ compare_local_version_to_git_version() { if git status &> /dev/null; then # The current version the user is on local local_version - local_version=$(git describe --tags --abbrev=0); + local_version=$(git describe --tags --abbrev=0 2>&1); # What branch they are on local local_branch local_branch=$(git rev-parse --abbrev-ref HEAD); From 254b67bca3cfc9ca97e05c674fed1f1bf8482cc9 Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Mon, 13 Jan 2025 20:11:41 -0300 Subject: [PATCH 393/413] Using more user-friendly messages on `git describe` failure Signed-off-by: RD WebDesign --- advanced/Scripts/piholeDebug.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 80fe9a35..e6d6197b 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -202,7 +202,7 @@ compare_local_version_to_git_version() { if git status &> /dev/null; then # The current version the user is on local local_version - local_version=$(git describe --tags --abbrev=0 2>&1); + local_version=$(git describe --tags --abbrev=0 2> /dev/null); # What branch they are on local local_branch local_branch=$(git rev-parse --abbrev-ref HEAD); @@ -213,7 +213,13 @@ compare_local_version_to_git_version() { local local_status local_status=$(git status -s) # echo this information out to the user in a nice format - log_write "${TICK} Version: ${local_version}" + if [ ${local_version} ]; then + log_write "${TICK} Version: ${local_version}" + elif [ -n "${DOCKER_VERSION}" ]; then + log_write "${TICK} Version: Pi-hole Docker Container ${COL_BOLD}${DOCKER_VERSION}${COL_NC}" + else + log_write "${CROSS} Version: not detected" + fi # Print the repo upstreams remotes=$(git remote -v) From 08e3257f37b0c4ae02371da49036f68fc33d6f96 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 25 Jan 2025 10:08:17 +0000 Subject: [PATCH 394/413] Bump actions/stale from 9.0.0 to 9.1.0 Bumps [actions/stale](https://github.com/actions/stale) from 9.0.0 to 9.1.0. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v9.0.0...v9.1.0) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yml | 2 +- .github/workflows/stale_pr.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 724c29d4..a0dd9a31 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -17,7 +17,7 @@ jobs: issues: write steps: - - uses: actions/stale@v9.0.0 + - uses: actions/stale@v9.1.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 30 diff --git a/.github/workflows/stale_pr.yml b/.github/workflows/stale_pr.yml index 96650818..6952dcab 100644 --- a/.github/workflows/stale_pr.yml +++ b/.github/workflows/stale_pr.yml @@ -17,7 +17,7 @@ jobs: pull-requests: write steps: - - uses: actions/stale@v9.0.0 + - uses: actions/stale@v9.1.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} # Do not automatically mark PR/issue as stale From 66757de3fb4ddaa8f9ea1f379e824b6f64f51bc2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 25 Jan 2025 10:34:03 +0000 Subject: [PATCH 395/413] Bump tox from 4.23.2 to 4.24.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.23.2 to 4.24.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.23.2...4.24.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index d72475c4..7b7e679f 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.2 pytest == 8.3.4 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.23.2 +tox == 4.24.1 pytest-clarity == 1.0.1 From 69bfb3ff3b955d3dfb76e3095e18b037b52eb0d6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 26 Jan 2025 19:52:46 +0100 Subject: [PATCH 396/413] Store timestamp of the backup creation instead of the filename on restoring gravity.db Signed-off-by: DL6ER --- gravity.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index f35eb674..68840b51 100755 --- a/gravity.sh +++ b/gravity.sh @@ -366,7 +366,7 @@ gravity_CheckDNSResolutionAvailable() { # 0 - If the backup is successfully restored. # 1 - If no backup is available or if the restoration fails. try_restore_backup () { - local num filename + local num filename timestamp num=$1 filename="${gravityBCKfile}.${num}" # Check if a backup exists @@ -386,8 +386,15 @@ try_restore_backup () { gravity_Cleanup "error" fi - pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) values ('gravity_restored','${filename}');" - echo -e " ${TICK} Successfully restored from backup (${gravityBCKfile}.${num})" + # Get the timestamp of the backup file in a human-readable format + # Note that this timestamp will be in the server timezone, this may be + # GMT, e.g., on a Raspberry Pi where the default timezone has never been + # changed + timestamp=$(date -r "${filename}" "+%Y-%m-%d %H:%M:%S %Z") + + # Add a record to the info table to indicate that the gravity database was restored + pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) values ('gravity_restored','${timestamp}');" + echo -e " ${TICK} Successfully restored from backup (${gravityBCKfile}.${num} at ${timestamp})" return 0 else echo -e " ${CROSS} Unable to restore backup no. ${num}" From c381109dcd43b8f1e3d96604fa9e2783ad53ad37 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 29 Jan 2025 17:52:39 +0100 Subject: [PATCH 397/413] Use a different method to identify if a gravity restore succeeded Signed-off-by: DL6ER --- gravity.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 68840b51..304c2d79 100755 --- a/gravity.sh +++ b/gravity.sh @@ -413,7 +413,7 @@ gravity_DownloadBlocklists() { echo -e " ${INFO} Storing gravity database in ${COL_BOLD}${gravityDBfile}${COL_NC}" fi - local url domain str target compression adlist_type directory + local url domain str target compression adlist_type directory success echo "" # Prepare new gravity database @@ -447,16 +447,18 @@ gravity_DownloadBlocklists() { echo -e "\\n ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n ${output}" # Try to attempt a backup restore + success=false if [[ -d "${gravityBCKdir}" ]]; then for i in {1..10}; do if try_restore_backup "${i}"; then + success=true break fi done fi # If none of the attempts worked, return 1 - if [[ "${i}" -eq 10 ]]; then + if [[ "${success}" == false ]]; then pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) values ('gravity_restored','failed');" return 1 fi From cee021c358afc2ba98e412993bf2c52bf7bed715 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 29 Jan 2025 16:58:23 +0100 Subject: [PATCH 398/413] Use HTTP ETags in addition to the If-Modified-Since header if supported by curl to avoid downloading from servers not using the latter method (raw.githubcontent.com) Signed-off-by: DL6ER --- gravity.sh | 52 ++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/gravity.sh b/gravity.sh index 68840b51..f34cec40 100755 --- a/gravity.sh +++ b/gravity.sh @@ -44,6 +44,7 @@ gravityDBcopy="${piholeGitDir}/advanced/Templates/gravity_copy.sql" domainsExtension="domains" curl_connect_timeout=10 +etag_support=false # Check gravity temp directory if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then @@ -502,6 +503,20 @@ gravity_DownloadBlocklists() { compression="" echo -e " ${INFO} Libz compression not available\n" fi + + # Check if etag is supported by the locally available version of curl by + # comparing the version string being >= 7.68.0 (released Jan 2020) + # https://github.com/curl/curl/pull/4543 followed by + # https://github.com/curl/curl/pull/4678 + if curl -V | grep -q "curl 7\.[6-9][8-9]"; then + etag_support=true + else + # Check if the version is >= 8 + if curl -V | grep -q "curl 8"; then + etag_support=true + fi + fi + # Loop through $sources and download each one for ((i = 0; i < "${#sources[@]}"; i++)); do url="${sources[$i]}" @@ -595,12 +610,37 @@ gravity_DownloadBlocklistFromUrl() { mv "${listCurlBuffer}" "${listCurlBuffer%.*}.phgpb" listCurlBuffer="${listCurlBuffer%.*}.phgpb" - # Determine if $saveLocation has read permission - if [[ -r "${saveLocation}" && $url != "file"* ]]; then - # Have curl determine if a remote file has been modified since last retrieval - # Uses "Last-Modified" header, which certain web servers do not provide (e.g: raw github urls) - # Note: Don't do this for local files, always download them - heisenbergCompensator="-z ${saveLocation}" + # For all remote files, we try to determine if the file has changed to skip + # downloading them whenever possible. + if [[ $url != "file"* ]]; then + # Use the HTTP ETag header to determine if the file has changed if supported + # by curl. Using ETags is supported by raw.githubusercontent.com URLs. + if [[ "${etag_support}" == true ]]; then + # Save HTTP ETag to the specified file. An ETag is a caching related header, + # usually returned in a response. If no ETag is sent by the server, an empty + # file is created and can later be used consistently. + heisenbergCompensator="--etag-save ${saveLocation}.etag" + + if [[ -f "${saveLocation}.etag" ]]; then + # This option makes a conditional HTTP request for the specific ETag read + # from the given file by sending a custom If-None-Match header using the + # stored ETag. This way, the server will only send the file if it has + # changed since the last request. + heisenbergCompensator="${heisenbergCompensator} --etag-compare ${saveLocation}.etag" + fi + fi + + # Add If-Modified-Since header to the request if we did already download the + # file once + if [[ -f "${saveLocation}" ]]; then + # Request a file that has been modified later than the given time and + # date. We provide a file here which makes curl use the modification + # timestamp (mtime) of this file. + # Interstingly, this option is not supported by raw.githubusercontent.com + # URLs, however, it is still supported by many older web servers which may + # not support the HTTP ETag method so we keep it as a fallback. + heisenbergCompensator="${heisenbergCompensator} -z ${saveLocation}" + fi fi str="Status:" From 4322299c39e6657d9973e4c19b1fbf26865ecfba Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 29 Jan 2025 16:58:54 +0100 Subject: [PATCH 399/413] Drive-by fixing of errors reported by shellcheck for gravity.sh Signed-off-by: DL6ER --- gravity.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index f34cec40..c39a0f26 100755 --- a/gravity.sh +++ b/gravity.sh @@ -534,7 +534,7 @@ gravity_DownloadBlocklists() { # Save the file as list.#.domain saveLocation="${piholeDir}/list.${id}.${domain}.${domainsExtension}" - activeDomains[$i]="${saveLocation}" + activeDomains[i]="${saveLocation}" # Check if we can write to the save location file without actually creating # it (in case it doesn't exist) @@ -1055,7 +1055,7 @@ timeit(){ elapsed_time=$((end_time - start_time)) # Display the elapsed time - printf " %b--> took %d.%03d seconds%b\n" ${COL_BLUE} $((elapsed_time / 1000)) $((elapsed_time % 1000)) ${COL_NC} + printf " %b--> took %d.%03d seconds%b\n" "${COL_BLUE}" $((elapsed_time / 1000)) $((elapsed_time % 1000)) "${COL_NC}" return $ret } From 6e0b16ed1acbca2512e0f22ec02ddffef6ef619b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 29 Jan 2025 19:17:04 +0100 Subject: [PATCH 400/413] Rename heisenbergCompensator -> modifiedOptions Signed-off-by: DL6ER --- gravity.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index c39a0f26..e9da1a97 100755 --- a/gravity.sh +++ b/gravity.sh @@ -601,7 +601,7 @@ compareLists() { # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" gravity_type="${6}" domain="${7}" - local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext + local modifiedOptions="" listCurlBuffer str httpCode success="" ip cmd_ext local file_path permissions ip_addr port blocked=false download=true # Create temp file to store content on disk instead of RAM @@ -619,14 +619,14 @@ gravity_DownloadBlocklistFromUrl() { # Save HTTP ETag to the specified file. An ETag is a caching related header, # usually returned in a response. If no ETag is sent by the server, an empty # file is created and can later be used consistently. - heisenbergCompensator="--etag-save ${saveLocation}.etag" + modifiedOptions="--etag-save ${saveLocation}.etag" if [[ -f "${saveLocation}.etag" ]]; then # This option makes a conditional HTTP request for the specific ETag read # from the given file by sending a custom If-None-Match header using the # stored ETag. This way, the server will only send the file if it has # changed since the last request. - heisenbergCompensator="${heisenbergCompensator} --etag-compare ${saveLocation}.etag" + modifiedOptions="${modifiedOptions} --etag-compare ${saveLocation}.etag" fi fi @@ -639,7 +639,7 @@ gravity_DownloadBlocklistFromUrl() { # Interstingly, this option is not supported by raw.githubusercontent.com # URLs, however, it is still supported by many older web servers which may # not support the HTTP ETag method so we keep it as a fallback. - heisenbergCompensator="${heisenbergCompensator} -z ${saveLocation}" + modifiedOptions="${modifiedOptions} -z ${saveLocation}" fi fi @@ -770,7 +770,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ "${download}" == true ]]; then # shellcheck disable=SC2086 - httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2>/dev/null) + httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${modifiedOptions} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2>/dev/null) fi case $url in From f50c6962531db57d7621d8a140eb6efb0ed0ef54 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 29 Jan 2025 19:19:01 +0100 Subject: [PATCH 401/413] Use temp variable for curl -V output Signed-off-by: DL6ER --- gravity.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index e9da1a97..bc11867f 100755 --- a/gravity.sh +++ b/gravity.sh @@ -414,7 +414,7 @@ gravity_DownloadBlocklists() { echo -e " ${INFO} Storing gravity database in ${COL_BOLD}${gravityDBfile}${COL_NC}" fi - local url domain str target compression adlist_type directory + local url domain str target compression adlist_type directory curlVersion echo "" # Prepare new gravity database @@ -496,7 +496,8 @@ gravity_DownloadBlocklists() { # Use compression to reduce the amount of data that is transferred # between the Pi-hole and the ad list provider. Use this feature # only if it is supported by the locally available version of curl - if curl -V | grep -q "Features:.* libz"; then + curlVersion=$(curl -V) + if echo "${curlVersion}" | grep -q "Features:.* libz"; then compression="--compressed" echo -e " ${INFO} Using libz compression\n" else @@ -508,11 +509,11 @@ gravity_DownloadBlocklists() { # comparing the version string being >= 7.68.0 (released Jan 2020) # https://github.com/curl/curl/pull/4543 followed by # https://github.com/curl/curl/pull/4678 - if curl -V | grep -q "curl 7\.[6-9][8-9]"; then + if echo "${curlVersion}" | grep -q "curl 7\.[6-9][8-9]"; then etag_support=true else # Check if the version is >= 8 - if curl -V | grep -q "curl 8"; then + if echo "${curlVersion}" | grep -q "curl 8"; then etag_support=true fi fi From bf899fa57f9d3b5743d1260df5be008a7f3cd6d2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 30 Jan 2025 06:12:49 +0100 Subject: [PATCH 402/413] Migrate list's files (lists themselves, their SHA1 sum and, later, their ETags) into a dedicated directory /etc/pihole/listsCache in an attempt to clean up /etc/pihole Signed-off-by: DL6ER --- gravity.sh | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index 304c2d79..557047ab 100755 --- a/gravity.sh +++ b/gravity.sh @@ -30,6 +30,9 @@ PIHOLE_COMMAND="/usr/local/bin/${basename}" piholeDir="/etc/${basename}" +# Gravity aux files directory +listsCacheDir="${piholeDir}/listsCache" + # Legacy (pre v5.0) list file locations whitelistFile="${piholeDir}/whitelist.txt" blacklistFile="${piholeDir}/blacklist.txt" @@ -520,8 +523,8 @@ gravity_DownloadBlocklists() { fi # Save the file as list.#.domain - saveLocation="${piholeDir}/list.${id}.${domain}.${domainsExtension}" - activeDomains[$i]="${saveLocation}" + saveLocation="${listsCacheDir}/list.${id}.${domain}.${domainsExtension}" + activeDomains[i]="${saveLocation}" # Check if we can write to the save location file without actually creating # it (in case it doesn't exist) @@ -1022,6 +1025,28 @@ timeit(){ return $ret } +migrate_to_listsCache_dir() { + # If the ${listsCacheDir} directory already exists, this has been done before + if [[ -d "${listsCacheDir}" ]]; then + return + fi + + # If not, we need to migrate the old files to the new directory + local str="Migrating the list's cache directory to new location" + echo -ne " ${INFO} ${str}..." + mkdir -p "${listsCacheDir}" + + # Move the old files to the new directory + if mv "${piholeDir}"/list.* "${listsCacheDir}/" 2>/dev/null; then + echo -e "${OVER} ${TICK} ${str}" + else + echo -e "${OVER} ${CROSS} ${str}" + fi + + # Update the list's paths in the corresponding .sha1 files to the new location + sed -i "s|${piholeDir}/|${listsCacheDir}/|g" "${listsCacheDir}"/*.sha1 +} + helpFunc() { echo "Usage: pihole -g Update domains from blocklists specified in adlists.list @@ -1097,6 +1122,9 @@ if [[ "${recover_database:-}" == true ]]; then timeit database_recovery "$4" fi +# Migrate scattered list files to the new cache directory +migrate_to_listsCache_dir + # Move possibly existing legacy files to the gravity database if ! timeit migrate_to_database; then echo -e " ${CROSS} Unable to migrate to database. Please contact support." @@ -1107,7 +1135,7 @@ if [[ "${forceDelete:-}" == true ]]; then str="Deleting existing list cache" echo -ne "${INFO} ${str}..." - rm /etc/pihole/list.* 2>/dev/null || true + rm "${listsCacheDir}/list.*" 2>/dev/null || true echo -e "${OVER} ${TICK} ${str}" fi From 9faef22061e1815f01bdabe0721dacd9e56fb5bc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 31 Jan 2025 16:46:12 +0100 Subject: [PATCH 403/413] Decide if the content was changed *before* passing over to FTL Signed-off-by: DL6ER --- gravity.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index 304c2d79..855f15e2 100755 --- a/gravity.sh +++ b/gravity.sh @@ -775,9 +775,10 @@ gravity_DownloadBlocklistFromUrl() { # Determine if the blocklist was downloaded and saved correctly if [[ "${success}" == true ]]; then if [[ "${httpCode}" == "304" ]]; then + # Set list status to "unchanged/cached" + database_adlist_status "${adlistID}" "2" # Add domains to database table file pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" - database_adlist_status "${adlistID}" "2" done="true" # Check if $listCurlBuffer is a non-zero length file elif [[ -s "${listCurlBuffer}" ]]; then @@ -785,10 +786,10 @@ gravity_DownloadBlocklistFromUrl() { gravity_ParseFileIntoDomains "${listCurlBuffer}" "${saveLocation}" # Remove curl buffer file after its use rm "${listCurlBuffer}" + # Compare lists if are they identical + compareLists "${adlistID}" "${saveLocation}" # Add domains to database table file pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" - # Compare lists, are they identical? - compareLists "${adlistID}" "${saveLocation}" done="true" else # Fall back to previously cached list if $listCurlBuffer is empty @@ -801,9 +802,10 @@ gravity_DownloadBlocklistFromUrl() { # Determine if cached list has read permission if [[ -r "${saveLocation}" ]]; then echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" + # Set list status to "download-failed/cached" + database_adlist_status "${adlistID}" "3" # Add domains to database table file pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" - database_adlist_status "${adlistID}" "3" else echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" # Manually reset these two numbers because we do not call parseList here From aadea31e5227c2eb819537c229d204bfb4e5c494 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Feb 2025 10:14:46 +0000 Subject: [PATCH 404/413] Bump actions/setup-python from 5.3.0 to 5.4.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.3.0 to 5.4.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.3.0...v5.4.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 983ca0bc..e96bf7d5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -77,7 +77,7 @@ jobs: uses: actions/checkout@v4.2.2 - name: Set up Python 3.10 - uses: actions/setup-python@v5.3.0 + uses: actions/setup-python@v5.4.0 with: python-version: "3.10" From 37718c65e9efd348824948aec3dc29483b49ec55 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 1 Feb 2025 19:36:25 +0100 Subject: [PATCH 405/413] Simplify cURL etag feature detection following discussion in #5873 Signed-off-by: DL6ER --- gravity.sh | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/gravity.sh b/gravity.sh index 276e59c2..f6d09ac5 100755 --- a/gravity.sh +++ b/gravity.sh @@ -507,17 +507,12 @@ gravity_DownloadBlocklists() { echo -e " ${INFO} Libz compression not available\n" fi - # Check if etag is supported by the locally available version of curl by - # comparing the version string being >= 7.68.0 (released Jan 2020) - # https://github.com/curl/curl/pull/4543 followed by + # Check if etag is supported by the locally available version of curl + # (available as of curl 7.68.0, released Jan 2020) + # https://github.com/curl/curl/pull/4543 + # https://github.com/curl/curl/pull/4678 - if echo "${curlVersion}" | grep -q "curl 7\.[6-9][8-9]"; then + if curl --help all | grep -q "etag-save"; then etag_support=true - else - # Check if the version is >= 8 - if echo "${curlVersion}" | grep -q "curl 8"; then - etag_support=true - fi fi # Loop through $sources and download each one From 9da1e9ab88f3a694adce3eda9fff2a3ee74bb91f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 1 Feb 2025 19:42:19 +0100 Subject: [PATCH 406/413] Fix counting of domains at the end of pihole -g. The existing schme used the SQL VIEWs for simplicitly, however, they have the unintended side-effect of - when being queried without a GROUP BY clause - returning entries multiple times if they are assigned to several groups Signed-off-by: DL6ER --- gravity.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index 304c2d79..39f2fd8c 100755 --- a/gravity.sh +++ b/gravity.sh @@ -864,11 +864,11 @@ gravity_Table_Count() { gravity_ShowCount() { # Here we use the table "gravity" instead of the view "vw_gravity" for speed. # It's safe to replace it here, because right after a gravity run both will show the exactly same number of domains. - gravity_Table_Count "gravity" "gravity domains" "" - gravity_Table_Count "vw_blacklist" "exact denied domains" - gravity_Table_Count "vw_regex_blacklist" "regex denied filters" - gravity_Table_Count "vw_whitelist" "exact allowed domains" - gravity_Table_Count "vw_regex_whitelist" "regex allowed filters" + gravity_Table_Count "gravity" "gravity domains" + gravity_Table_Count "domainlist WHERE type = 1" "exact denied domains" + gravity_Table_Count "domainlist WHERE type = 3" "regex denied filters" + gravity_Table_Count "domainlist WHERE type = 0" "exact allowed domains" + gravity_Table_Count "domainlist WHERE type = 2" "regex allowed filters" } # Trap Ctrl-C From 7162be18e2585dfd14db65f2275d2bd6a1379af6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 2 Feb 2025 20:57:13 +0100 Subject: [PATCH 407/413] Remove curl -V temp variable Signed-off-by: DL6ER --- gravity.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index f6d09ac5..c01b2823 100755 --- a/gravity.sh +++ b/gravity.sh @@ -414,7 +414,7 @@ gravity_DownloadBlocklists() { echo -e " ${INFO} Storing gravity database in ${COL_BOLD}${gravityDBfile}${COL_NC}" fi - local url domain str target compression adlist_type directory curlVersion success + local url domain str target compression adlist_type directory success echo "" # Prepare new gravity database @@ -498,8 +498,7 @@ gravity_DownloadBlocklists() { # Use compression to reduce the amount of data that is transferred # between the Pi-hole and the ad list provider. Use this feature # only if it is supported by the locally available version of curl - curlVersion=$(curl -V) - if echo "${curlVersion}" | grep -q "Features:.* libz"; then + if curl -V | grep -q "Features:.* libz"; then compression="--compressed" echo -e " ${INFO} Using libz compression\n" else From 0da97e762f851a9a801b905e4816d47bb665899c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 3 Feb 2025 08:31:33 +0100 Subject: [PATCH 408/413] Remove outdated dns-servers.conf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 7 ------- test/test_any_automated_install.py | 5 ----- 2 files changed, 12 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 719a57fc..2e202861 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1279,13 +1279,6 @@ installConfigs() { # Ensure that permissions are correctly set chown -R pihole:pihole /etc/pihole - # Install list of DNS servers - # Format: Name;Primary IPv4;Secondary IPv4;Primary IPv6;Secondary IPv6 - # Some values may be empty (for example: DNS servers without IPv6 support) - echo "${DNS_SERVERS}" >"${PI_HOLE_CONFIG_DIR}/dns-servers.conf" - chmod 644 "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" - chown pihole:pihole "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" - # Install empty custom.list file if it does not exist if [[ ! -r "${PI_HOLE_CONFIG_DIR}/hosts/custom.list" ]]; then if ! install -D -T -o pihole -g pihole -m 660 /dev/null "${PI_HOLE_CONFIG_DIR}/hosts/custom.list" &>/dev/null; then diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 19812122..73da7eef 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -119,11 +119,6 @@ def test_installPihole_fresh_install_readableFiles(host): assert exit_status_success == actual_rc check_leases = test_cmd.format("w", "/etc/pihole/dhcp.leases", piholeuser) actual_rc = host.run(check_leases).rc - # readable dns-servers.conf - assert exit_status_success == actual_rc - check_servers = test_cmd.format("r", "/etc/pihole/dns-servers.conf", piholeuser) - actual_rc = host.run(check_servers).rc - assert exit_status_success == actual_rc # readable install.log check_install = test_cmd.format("r", "/etc/pihole/install.log", piholeuser) actual_rc = host.run(check_install).rc From 88cfda721046be2ccbfd00b28225022b91a68fbf Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 3 Feb 2025 19:16:24 +0100 Subject: [PATCH 409/413] Show only enabled domains/regex in the final gravity message Signed-off-by: DL6ER --- gravity.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index ad8863ca..5bc0af09 100755 --- a/gravity.sh +++ b/gravity.sh @@ -900,10 +900,10 @@ gravity_ShowCount() { # Here we use the table "gravity" instead of the view "vw_gravity" for speed. # It's safe to replace it here, because right after a gravity run both will show the exactly same number of domains. gravity_Table_Count "gravity" "gravity domains" - gravity_Table_Count "domainlist WHERE type = 1" "exact denied domains" - gravity_Table_Count "domainlist WHERE type = 3" "regex denied filters" - gravity_Table_Count "domainlist WHERE type = 0" "exact allowed domains" - gravity_Table_Count "domainlist WHERE type = 2" "regex allowed filters" + gravity_Table_Count "domainlist WHERE type = 1 AND enabled = 1" "exact denied domains" + gravity_Table_Count "domainlist WHERE type = 3 AND enabled = 1" "regex denied filters" + gravity_Table_Count "domainlist WHERE type = 0 AND enabled = 1" "exact allowed domains" + gravity_Table_Count "domainlist WHERE type = 2 AND enabled = 1" "regex allowed filters" } # Trap Ctrl-C From 8fb3ccc147b1c86a0accf62dcc40f017a505dec7 Mon Sep 17 00:00:00 2001 From: Nick Rosbrook Date: Mon, 3 Feb 2025 11:18:50 -0500 Subject: [PATCH 410/413] installer: use a drop-in to disable systemd-resolved stub listener systemd-resolved supports drop-in config snippets, e.g. by placing .conf files in /etc/systemd/resolved.conf.d. During install, use a drop-in config to set DNSStubListener=no, instead of modifying the main config. This is generally better practice, and also prevents conflicts when distribution packages are upgraded, which may include new versions of /etc/systemd/resolved.conf. Signed-off-by: Nick Rosbrook --- automated install/basic-install.sh | 23 +++++++++-------------- automated install/uninstall.sh | 5 +++-- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 719a57fc..b40d4e60 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1436,20 +1436,15 @@ disable_resolved_stublistener() { printf " %b Testing if systemd-resolved is enabled\\n" "${INFO}" # Check if Systemd-resolved's DNSStubListener is enabled and active on port 53 if check_service_active "systemd-resolved"; then - # Check if DNSStubListener is enabled - printf " %b %b Testing if systemd-resolved DNSStub-Listener is active" "${OVER}" "${INFO}" - if (grep -E '#?DNSStubListener=yes' /etc/systemd/resolved.conf &>/dev/null); then - # Disable the DNSStubListener to unbind it from port 53 - # Note that this breaks dns functionality on host until ftl are up and running - printf "%b %b Disabling systemd-resolved DNSStubListener" "${OVER}" "${TICK}" - # Make a backup of the original /etc/systemd/resolved.conf - # (This will need to be restored on uninstallation) - sed -r -i.orig 's/#?DNSStubListener=yes/DNSStubListener=no/g' /etc/systemd/resolved.conf - printf " and restarting systemd-resolved\\n" - systemctl reload-or-restart systemd-resolved - else - printf "%b %b Systemd-resolved does not need to be restarted\\n" "${OVER}" "${INFO}" - fi + # Disable the DNSStubListener to unbind it from port 53 + # Note that this breaks dns functionality on host until FTL is up and running + printf "%b %b Disabling systemd-resolved DNSStubListener\\n" "${OVER}" "${TICK}" + mkdir -p /etc/systemd/resolved.conf.d + cat > /etc/systemd/resolved.conf.d/90-pi-hole-disable-stub-listener.conf << EOF +[Resolve] +DNSStubListener=no +EOF + systemctl reload-or-restart systemd-resolved else printf "%b %b Systemd-resolved is not enabled\\n" "${OVER}" "${INFO}" fi diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index a073c319..39c13037 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -94,8 +94,9 @@ removePiholeFiles() { echo -e " ${TICK} Removed config files" # Restore Resolved - if [[ -e /etc/systemd/resolved.conf.orig ]]; then - ${SUDO} cp -p /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf + if [[ -e /etc/systemd/resolved.conf.orig ]] || [[ -e /etc/systemd/resolved.conf.d/90-pi-hole-disable-stub-listener.conf ]]; then + ${SUDO} cp -p /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf &> /dev/null || true + ${SUDO} rm -f /etc/systemd/resolved.conf.d/90-pi-hole-disable-stub-listener.conf systemctl reload-or-restart systemd-resolved fi From 1bebeff72063aa8ce90698dfeac6019251df45f7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 9 Feb 2025 10:06:24 +0100 Subject: [PATCH 411/413] Add further info message that building the tree may have failed because of too little RAM baing available. We have see this a few times, either with way too many domains on devices like a Pi Zero or, e.g., only 256 MB RAM being allocated to VMs Signed-off-by: DL6ER --- gravity.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/gravity.sh b/gravity.sh index bd6285c5..0a34599b 100755 --- a/gravity.sh +++ b/gravity.sh @@ -97,6 +97,7 @@ gravity_build_tree() { if [[ "${status}" -ne 0 ]]; then echo -e "\\n ${CROSS} Unable to build gravity tree in ${gravityTEMPfile}\\n ${output}" + echo -e " ${INFO} If you have a large amount of domains, make sure your Pi-hole has enough RAM available\\n" return 1 fi echo -e "${OVER} ${TICK} ${str}" From d1639740d98755e407cfce5fa3408d0988561411 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 10 Feb 2025 20:29:05 +0100 Subject: [PATCH 412/413] Fix database integrity check in debug log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index e6d6197b..4886588e 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1221,7 +1221,7 @@ database_integrity_check(){ local database="${1}" log_write "${INFO} Checking integrity of ${database} ... (this can take several minutes)" - result="$(pihole-FTL "${database}" "PRAGMA integrity_check" 2>&1 & spinner)" + result="$(pihole-FTL sqlite3 -ni "${database}" "PRAGMA integrity_check" 2>&1 & spinner)" if [[ ${result} = "ok" ]]; then log_write "${TICK} Integrity of ${database} intact" From c9e58adc48aa3c4846079d2fdbd3fe05365055ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 10 Feb 2025 20:41:18 +0100 Subject: [PATCH 413/413] DHCP discover timeout was reduced by FTL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 4886588e..f4226299 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1137,7 +1137,7 @@ show_FTL_db_entries() { } check_dhcp_servers() { - echo_current_diagnostic "Discovering active DHCP servers (takes 10 seconds)" + echo_current_diagnostic "Discovering active DHCP servers (takes 6 seconds)" OLD_IFS="$IFS" IFS=$'\n'