From 9f31ab8a6fd2dfe361c2383f4b59648eb9cd6e3f Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 1 Jan 2023 12:34:40 +0000 Subject: [PATCH 001/200] Debug log does not need to check php/lighttpd Signed-off-by: Adam Warner --- advanced/Scripts/piholeDebug.sh | 63 +++++++-------------------------- 1 file changed, 12 insertions(+), 51 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index fa1cebbb..7462cba3 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -64,10 +64,8 @@ PIHOLE_SCRIPTS_DIRECTORY="/opt/pihole" BIN_DIRECTORY="/usr/local/bin" RUN_DIRECTORY="/run" LOG_DIRECTORY="/var/log/pihole" -WEB_SERVER_LOG_DIRECTORY="/var/log/lighttpd" -WEB_SERVER_CONFIG_DIRECTORY="/etc/lighttpd" -WEB_SERVER_CONFIG_DIRECTORY_FEDORA="${WEB_SERVER_CONFIG_DIRECTORY}/conf.d" -WEB_SERVER_CONFIG_DIRECTORY_DEBIAN="${WEB_SERVER_CONFIG_DIRECTORY}/conf-enabled" +#WEB_SERVER_LOG_DIRECTORY="/var/log/lighttpd" #TODO: FTL access log? +#WEB_SERVER_CONFIG_DIRECTORY="/etc/lighttpd" #TODO: FTL access log? HTML_DIRECTORY="/var/www/html" WEB_GIT_DIRECTORY="${HTML_DIRECTORY}/admin" SHM_DIRECTORY="/dev/shm" @@ -77,10 +75,8 @@ ETC="/etc" # https://discourse.pi-hole.net/t/what-files-does-pi-hole-use/1684 PIHOLE_CRON_FILE="${CRON_D_DIRECTORY}/pihole" -WEB_SERVER_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/lighttpd.conf" -WEB_SERVER_CUSTOM_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/external.conf" -WEB_SERVER_PIHOLE_CONFIG_FILE_DEBIAN="${WEB_SERVER_CONFIG_DIRECTORY_DEBIAN}/15-pihole-admin.conf" -WEB_SERVER_PIHOLE_CONFIG_FILE_FEDORA="${WEB_SERVER_CONFIG_DIRECTORY_FEDORA}/pihole-admin.conf" +#WEB_SERVER_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/lighttpd.conf" +#WEB_SERVER_CUSTOM_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/external.conf" PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" @@ -131,21 +127,19 @@ PIHOLE_LOG_GZIPS="${LOG_DIRECTORY}/pihole.log.[0-9].*" PIHOLE_DEBUG_LOG="${LOG_DIRECTORY}/pihole_debug.log" PIHOLE_FTL_LOG="$(get_ftl_conf_value "LOGFILE" "${LOG_DIRECTORY}/FTL.log")" -PIHOLE_WEB_SERVER_ACCESS_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/access-pihole.log" -PIHOLE_WEB_SERVER_ERROR_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/error-pihole.log" +# PIHOLE_WEB_SERVER_ACCESS_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/access-pihole.log" #TODO: FTL Error log? +# PIHOLE_WEB_SERVER_ERROR_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/error-pihole.log" #TODO: FTL Error log? RESOLVCONF="${ETC}/resolv.conf" DNSMASQ_CONF="${ETC}/dnsmasq.conf" # Store Pi-hole's processes in an array for easy use and parsing -PIHOLE_PROCESSES=( "lighttpd" "pihole-FTL" ) +PIHOLE_PROCESSES=( "pihole-FTL" ) # Store the required directories in an array so it can be parsed through REQUIRED_FILES=("${PIHOLE_CRON_FILE}" -"${WEB_SERVER_CONFIG_FILE}" -"${WEB_SERVER_CUSTOM_CONFIG_FILE}" -"${WEB_SERVER_PIHOLE_CONFIG_FILE_DEBIAN}" -"${WEB_SERVER_PIHOLE_CONFIG_FILE_FEDORA}" +# "${WEB_SERVER_CONFIG_FILE}" +# "${WEB_SERVER_CUSTOM_CONFIG_FILE}" "${PIHOLE_INSTALL_LOG_FILE}" "${PIHOLE_RAW_BLOCKLIST_FILES}" "${PIHOLE_LOCAL_HOSTS_FILE}" @@ -351,39 +345,6 @@ check_component_versions() { check_ftl_version } - -get_program_version() { - local program_name="${1}" - # Create a local variable so this function can be safely reused - local program_version - echo_current_diagnostic "${program_name} version" - # Evaluate the program we are checking, if it is any of the ones below, show the version - case "${program_name}" in - "lighttpd") program_version="$(${program_name} -v 2> /dev/null | head -n1 | cut -d '/' -f2 | cut -d ' ' -f1)" - ;; - "php") program_version="$(${program_name} -v 2> /dev/null | head -n1 | cut -d '-' -f1 | cut -d ' ' -f2)" - ;; - # If a match is not found, show an error - *) echo "Unrecognized program"; - esac - # If the program does not have a version (the variable is empty) - if [[ -z "${program_version}" ]]; then - # Display and error - log_write "${CROSS} ${COL_RED}${program_name} version could not be detected.${COL_NC}" - else - # Otherwise, display the version - log_write "${INFO} ${program_version}" - fi -} - -# These are the most critical dependencies of Pi-hole, so we check for them -# and their versions, using the functions above. -check_critical_program_versions() { - # Use the function created earlier and bundle them into one function that checks all the version numbers - get_program_version "lighttpd" - get_program_version "php" -} - os_check() { # This function gets a list of supported OS versions from a TXT record at versions.pi-hole.net # and determines whether or not the script is running on one of those systems @@ -737,10 +698,10 @@ compare_port_to_service_assigned() { check_required_ports() { echo_current_diagnostic "Ports in use" - # Since Pi-hole needs 53, 80, and 4711, check what they are being used by + # Since Pi-hole needs 53 and 4711, check what they are being used by # so we can detect any issues local resolver="pihole-FTL" - local web_server="lighttpd" + local web_server="pihole-FTL" local ftl="pihole-FTL" # Create an array for these ports in use ports_in_use=() @@ -1520,7 +1481,7 @@ initialize_debug # available to the other functions source_setup_variables check_component_versions -check_critical_program_versions +# check_critical_program_versions diagnose_operating_system check_selinux check_firewalld From 31ee15200d89ffd27fc16819ecebf3c360345993 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 1 Jan 2023 12:45:14 +0000 Subject: [PATCH 002/200] gut the install script of references to web server/lighttpd Signed-off-by: Adam Warner --- automated install/basic-install.sh | 233 ----------------------------- 1 file changed, 233 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 24fe9e54..3615804c 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -58,8 +58,6 @@ EOM installLogLoc="/etc/pihole/install.log" # This is an important file as it contains information specific to the machine it's being installed on setupVars="/etc/pihole/setupVars.conf" -# Pi-hole uses lighttpd as a Web server, and this is the config file for it -lighttpdConfig="/etc/lighttpd/lighttpd.conf" # This is a file used for the colorized output coltable="/opt/pihole/COL_TABLE" @@ -110,13 +108,11 @@ c=70 # The runUnattended flag is one example of this reconfigure=false runUnattended=false -INSTALL_WEB_SERVER=true # Check arguments for the undocumented flags for var in "$@"; do case "$var" in "--reconfigure" ) reconfigure=true;; "--unattended" ) runUnattended=true;; - "--disable-install-webserver" ) INSTALL_WEB_SERVER=false;; esac done @@ -308,42 +304,12 @@ package_manager_detect() { PKG_COUNT="${PKG_MANAGER} -s -o Debug::NoLocking=true upgrade | grep -c ^Inst || true" # Update package cache update_package_cache || exit 1 - # Check for and determine version number (major and minor) of current php install - local phpVer="php" - if is_command php ; then - phpVer="$(php <<< "")" - # Check if the first character of the string is numeric - if [[ ${phpVer:0:1} =~ [1-9] ]]; then - printf " %b Existing PHP installation detected : PHP version %s\\n" "${INFO}" "${phpVer}" - printf -v phpInsMajor "%d" "$(php <<< "")" - printf -v phpInsMinor "%d" "$(php <<< "")" - phpVer="php$phpInsMajor.$phpInsMinor" - else - printf " %b No valid PHP installation detected!\\n" "${CROSS}" - printf " %b PHP version : %s\\n" "${INFO}" "${phpVer}" - printf " %b Aborting installation.\\n" "${CROSS}" - exit 1 - fi - fi # Packages required to perform the os_check (stored as an array) OS_CHECK_DEPS=(grep dnsutils) # Packages required to run this install script (stored as an array) INSTALLER_DEPS=(git iproute2 dialog ca-certificates) # Packages required to run Pi-hole (stored as an array) PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) - # Packages required for the Web admin interface (stored as an array) - # It's useful to separate this from Pi-hole, since the two repos are also setup separately - PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-sqlite3" "${phpVer}-xml" "${phpVer}-intl") - # Prior to PHP8.0, JSON functionality is provided as dedicated module, required by Pi-hole AdminLTE: https://www.php.net/manual/json.installation.php - if [[ -z "${phpInsMajor}" || "${phpInsMajor}" -lt 8 ]]; then - PIHOLE_WEB_DEPS+=("${phpVer}-json") - fi - # The Web server user, - LIGHTTPD_USER="www-data" - # group, - LIGHTTPD_GROUP="www-data" - # and config file - LIGHTTPD_CFG="lighttpd.conf.debian" # If apt-get is not found, check for rpm. elif is_command rpm ; then @@ -361,25 +327,6 @@ package_manager_detect() { OS_CHECK_DEPS=(grep bind-utils) INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates) PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc libcap nmap-ncat jq) - PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo php-xml php-json php-intl) - LIGHTTPD_USER="lighttpd" - LIGHTTPD_GROUP="lighttpd" - LIGHTTPD_CFG="lighttpd.conf.fedora" - - # If the host OS is centos (or a derivative), epel is required for lighttpd - if ! grep -qiE 'fedora|fedberry' /etc/redhat-release; then - if rpm -qa | grep -qi 'epel'; then - printf " %b EPEL repository already installed\\n" "${TICK}" - else - local RH_RELEASE EPEL_PKG - # EPEL not already installed, add it based on the release version - RH_RELEASE=$(grep -oP '(?<= )[0-9]+(?=\.?)' /etc/redhat-release) - EPEL_PKG="https://dl.fedoraproject.org/pub/epel/epel-release-latest-${RH_RELEASE}.noarch.rpm" - printf " %b Enabling EPEL package repository (https://fedoraproject.org/wiki/EPEL)\\n" "${INFO}" - "${PKG_INSTALL[@]}" "${EPEL_PKG}" - printf " %b Installed %s\\n" "${TICK}" "${EPEL_PKG}" - fi - fi # If neither apt-get or yum/dnf package managers were found else @@ -1121,7 +1068,6 @@ setAdminFlag() { printf " %b Not installing Admin Web Interface\\n" "${INFO}" # Set the flag to not install the web interface INSTALL_WEB_INTERFACE=false - INSTALL_WEB_SERVER=false ;; "${DIALOG_ESC}") # User pressed @@ -1129,43 +1075,6 @@ setAdminFlag() { exit 1 ;; esac - - # If the user wants to install the Web admin interface (i.e. it has not been deselected above) and did not deselect the web server via command-line argument - if [[ "${INSTALL_WEB_INTERFACE}" == true && "${INSTALL_WEB_SERVER}" == true ]]; then - # Get list of required PHP modules, excluding base package (common) and handler (cgi) - local i php_modules - for i in "${PIHOLE_WEB_DEPS[@]}"; do [[ $i == 'php'* && $i != *'-common' && $i != *'-cgi' ]] && php_modules+=" ${i#*-}"; done - dialog --no-shadow --keep-tite \ - --backtitle "Pi-hole Installation" \ - --title "Web Server" \ - --yesno "\\n\\nA web server is required for the Admin Web Interface.\ -\\n\\nDo you want to install lighttpd and the required PHP modules?\ -\\n\\nNB: If you disable this, and, do not have an existing web server \ -and required PHP modules (${php_modules# }) installed, the web interface \ -will not function. Additionally the web server user needs to be member of \ -the \"pihole\" group for full functionality." \ - "${r}" "${c}" && result=0 || result=$? - - case ${result} in - "${DIALOG_OK}") - # If they chose yes, - printf " %b Installing lighttpd\\n" "${INFO}" - # Set the flag to install the web server - INSTALL_WEB_SERVER=true - ;; - "${DIALOG_CANCEL}") - # If they chose no, - printf " %b Not installing lighttpd\\n" "${INFO}" - # Set the flag to not install the web server - INSTALL_WEB_SERVER=false - ;; - "${DIALOG_ESC}") - # User pressed - printf " %b Escape pressed, exiting installer at web server choice.%b\\n" "${COL_LIGHT_RED}" "${COL_NC}" - exit 1 - ;; - esac - fi } # A function to display a list of example blocklists for users to select @@ -1397,79 +1306,6 @@ installConfigs() { fi install -T -m 0755 "${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole-FTL-prestart.sh" "${PI_HOLE_INSTALL_DIR}/pihole-FTL-prestart.sh" install -T -m 0755 "${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole-FTL-poststop.sh" "${PI_HOLE_INSTALL_DIR}/pihole-FTL-poststop.sh" - - # If the user chose to install the dashboard, - if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - # set permissions on /etc/lighttpd/lighttpd.conf so pihole user (other) can read the file - chmod o+x /etc/lighttpd - chmod o+r "${lighttpdConfig}" - - # Ensure /run/lighttpd exists and is owned by lighttpd user - # Needed for the php socket - mkdir -p /run/lighttpd - chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} /run/lighttpd - - if grep -q -F "OVERWRITTEN BY PI-HOLE" "${lighttpdConfig}"; then - # Attempt to preserve backwards compatibility with older versions - install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} "${lighttpdConfig}" - # Make the directories if they do not exist and set the owners - mkdir -p /var/cache/lighttpd/compress - chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} /var/cache/lighttpd/compress - mkdir -p /var/cache/lighttpd/uploads - chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} /var/cache/lighttpd/uploads - fi - # Copy the config file to include for pihole admin interface - if [[ -d "/etc/lighttpd/conf.d" ]]; then - install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/pihole-admin.conf /etc/lighttpd/conf.d/pihole-admin.conf - if grep -q -F 'include "/etc/lighttpd/conf.d/pihole-admin.conf"' "${lighttpdConfig}"; then - : - else - echo 'include "/etc/lighttpd/conf.d/pihole-admin.conf"' >> "${lighttpdConfig}" - fi - # Avoid some warnings trace from lighttpd, which might break tests - conf=/etc/lighttpd/conf.d/pihole-admin.conf - if lighttpd -f "${lighttpdConfig}" -tt 2>&1 | grep -q -F "WARNING: unknown config-key: dir-listing\."; then - echo '# Avoid some warnings trace from lighttpd, which might break tests' >> $conf - echo 'server.modules += ( "mod_dirlisting" )' >> $conf - fi - if lighttpd -f "${lighttpdConfig}" -tt 2>&1 | grep -q -F "warning: please use server.use-ipv6"; then - echo '# Avoid some warnings trace from lighttpd, which might break tests' >> $conf - echo 'server.use-ipv6 := "disable"' >> $conf - fi - elif [[ -d "/etc/lighttpd/conf-available" ]]; then - conf=/etc/lighttpd/conf-available/15-pihole-admin.conf - install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/pihole-admin.conf $conf - - # Get the version number of lighttpd - version=$(dpkg-query -f='${Version}\n' --show lighttpd) - # Test if that version is greater than or euqal to 1.4.56 - if dpkg --compare-versions "$version" "ge" "1.4.56"; then - # If it is, then we don't need to disable the modules - # (server.modules duplication is ignored in lighttpd 1.4.56+) - : - else - # disable server.modules += ( ... ) in $conf to avoid module dups - if awk '!/^server\.modules/{print}' $conf > $conf.$$ && mv $conf.$$ $conf; then - : - else - rm $conf.$$ - fi - fi - - chmod 644 $conf - if is_command lighty-enable-mod ; then - lighty-enable-mod pihole-admin access accesslog redirect fastcgi setenv > /dev/null || true - else - # Otherwise, show info about installing them - printf " %b Warning: 'lighty-enable-mod' utility not found\\n" "${INFO}" - printf " Please ensure fastcgi is enabled if you experience issues\\n" - fi - else - # lighttpd config include dir not found - printf " %b Warning: lighttpd config include dir not found\\n" "${INFO}" - printf " Please manually install pihole-admin.conf\\n" - fi - fi } install_manpage() { @@ -1717,29 +1553,6 @@ install_dependent_packages() { return 0 } -# Install the Web interface dashboard -installPiholeWeb() { - # Install Sudoers file - local str="Installing sudoer file" - printf "\\n %b %s..." "${INFO}" "${str}" - # Make the .d directory if it doesn't exist, - install -d -m 755 /etc/sudoers.d/ - # and copy in the pihole sudoers file - install -m 0640 ${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole.sudo /etc/sudoers.d/pihole - # Add lighttpd user (OS dependent) to sudoers file - echo "${LIGHTTPD_USER} ALL=NOPASSWD: ${PI_HOLE_BIN_DIR}/pihole" >> /etc/sudoers.d/pihole - - # If the Web server user is lighttpd, - if [[ "$LIGHTTPD_USER" == "lighttpd" ]]; then - # Allow executing pihole via sudo with Fedora - # Usually /usr/local/bin ${PI_HOLE_BIN_DIR} is not permitted as directory for sudoable programs - echo "Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin:${PI_HOLE_BIN_DIR}" >> /etc/sudoers.d/pihole - fi - # Set the strict permissions on the file - chmod 0440 /etc/sudoers.d/pihole - printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" -} - # Installs a cron file installCron() { # Install the cron job @@ -1838,9 +1651,7 @@ finalExports() { addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_1" "${PIHOLE_DNS_1}" addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_2" "${PIHOLE_DNS_2}" addOrEditKeyValPair "${setupVars}" "QUERY_LOGGING" "${QUERY_LOGGING}" - addOrEditKeyValPair "${setupVars}" "INSTALL_WEB_SERVER" "${INSTALL_WEB_SERVER}" addOrEditKeyValPair "${setupVars}" "INSTALL_WEB_INTERFACE" "${INSTALL_WEB_INTERFACE}" - addOrEditKeyValPair "${setupVars}" "LIGHTTPD_ENABLED" "${LIGHTTPD_ENABLED}" addOrEditKeyValPair "${setupVars}" "CACHE_SIZE" "${CACHE_SIZE}" addOrEditKeyValPair "${setupVars}" "DNS_FQDN_REQUIRED" "${DNS_FQDN_REQUIRED:-true}" addOrEditKeyValPair "${setupVars}" "DNS_BOGUS_PRIV" "${DNS_BOGUS_PRIV:-true}" @@ -1904,25 +1715,6 @@ installLogrotate() { # Install base files and web interface installPihole() { - # If the user wants to install the Web interface, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - if [[ ! -d "${webroot}" ]]; then - # make the Web directory if necessary - install -d -m 0755 ${webroot} - fi - - if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - # Set the owner and permissions - chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} ${webroot} - chmod 0775 ${webroot} - # Repair permissions if webroot is not world readable - chmod a+rx /var/www - chmod a+rx ${webroot} - # Give lighttpd access to the pihole group so the web interface can - # manage the gravity.db database - usermod -a -G pihole ${LIGHTTPD_USER} - fi - fi # Install base files and web interface if ! installScripts; then printf " %b Failure in dependent script copy function.\\n" "${CROSS}" @@ -2624,27 +2416,12 @@ main() { # Install the Core dependencies local dep_install_list=("${PIHOLE_DEPS[@]}") - if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - # And, if the setting says so, install the Web admin interface dependencies - dep_install_list+=("${PIHOLE_WEB_DEPS[@]}") - fi # Install packages used by the actual software printf " %b Checking for / installing Required dependencies for Pi-hole software...\\n" "${INFO}" install_dependent_packages "${dep_install_list[@]}" unset dep_install_list - # On some systems, lighttpd is not enabled on first install. We need to enable it here if the user - # has chosen to install the web interface, else the LIGHTTPD_ENABLED check will fail - if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - enable_service lighttpd - fi - # Determine if lighttpd is correctly enabled - if check_service_active "lighttpd"; then - LIGHTTPD_ENABLED=true - else - LIGHTTPD_ENABLED=false - fi # Create the pihole user create_pihole_user @@ -2685,16 +2462,6 @@ main() { # but before starting or resarting the dnsmasq or ftl services disable_resolved_stublistener - # If the Web server was installed, - if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - if [[ "${LIGHTTPD_ENABLED}" == true ]]; then - restart_service lighttpd - enable_service lighttpd - else - printf " %b Lighttpd is disabled, skipping service restart\\n" "${INFO}" - fi - fi - printf " %b Restarting services...\\n" "${INFO}" # Start services From c520b293264f1761cd008e73f4ff3d5d22db8822 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 1 Jan 2023 12:47:00 +0000 Subject: [PATCH 003/200] some more lighttpd refs (tests can come later) Signed-off-by: Adam Warner --- advanced/Scripts/piholeDebug.sh | 61 +++++++++++------------ advanced/lighttpd.conf.debian | 73 --------------------------- advanced/lighttpd.conf.fedora | 87 --------------------------------- advanced/pihole-admin.conf | 82 ------------------------------- 4 files changed, 28 insertions(+), 275 deletions(-) delete mode 100644 advanced/lighttpd.conf.debian delete mode 100644 advanced/lighttpd.conf.fedora delete mode 100644 advanced/pihole-admin.conf diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 7462cba3..9f895aab 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -64,8 +64,6 @@ PIHOLE_SCRIPTS_DIRECTORY="/opt/pihole" BIN_DIRECTORY="/usr/local/bin" RUN_DIRECTORY="/run" LOG_DIRECTORY="/var/log/pihole" -#WEB_SERVER_LOG_DIRECTORY="/var/log/lighttpd" #TODO: FTL access log? -#WEB_SERVER_CONFIG_DIRECTORY="/etc/lighttpd" #TODO: FTL access log? HTML_DIRECTORY="/var/www/html" WEB_GIT_DIRECTORY="${HTML_DIRECTORY}/admin" SHM_DIRECTORY="/dev/shm" @@ -75,9 +73,6 @@ ETC="/etc" # https://discourse.pi-hole.net/t/what-files-does-pi-hole-use/1684 PIHOLE_CRON_FILE="${CRON_D_DIRECTORY}/pihole" -#WEB_SERVER_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/lighttpd.conf" -#WEB_SERVER_CUSTOM_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/external.conf" - PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" @@ -762,34 +757,34 @@ check_networking() { [ -z "${DOCKER_VERSION}" ] && check_required_ports } -check_x_headers() { - # The X-Headers allow us to determine from the command line if the Web - # lighttpd.conf has a directive to show "X-Pi-hole: A black hole for Internet advertisements." - # in the header of any Pi-holed domain - # Similarly, it will show "X-Pi-hole: The Pi-hole Web interface is working!" if you view the header returned - # when accessing the dashboard (i.e curl -I pi.hole/admin/) - # server is operating correctly - echo_current_diagnostic "Dashboard headers" - # Use curl -I to get the header and parse out just the X-Pi-hole one - local full_curl_output_dashboard - local dashboard - full_curl_output_dashboard="$(curl -Is localhost/admin/)" - dashboard=$(echo "${full_curl_output_dashboard}" | awk '/X-Pi-hole/' | tr -d '\r') - # Store what the X-Header should be in variables for comparison later - local dashboard_working - dashboard_working="X-Pi-hole: The Pi-hole Web interface is working!" - - # If the X-Header matches what a working system should have, - if [[ $dashboard == "$dashboard_working" ]]; then - # then we can show a success - log_write "$TICK Web interface X-Header: ${COL_GREEN}${dashboard}${COL_NC}" - else - # Otherwise, it's a failure since the X-Headers either don't exist or have been modified in some way - log_write "$CROSS Web interface X-Header: ${COL_RED}X-Header does not match or could not be retrieved.${COL_NC}" - - log_write "${COL_RED}${full_curl_output_dashboard}${COL_NC}" - fi -} +# check_x_headers() { +# # The X-Headers allow us to determine from the command line if the Web +# # lighttpd.conf has a directive to show "X-Pi-hole: A black hole for Internet advertisements." +# # in the header of any Pi-holed domain +# # Similarly, it will show "X-Pi-hole: The Pi-hole Web interface is working!" if you view the header returned +# # when accessing the dashboard (i.e curl -I pi.hole/admin/) +# # server is operating correctly +# echo_current_diagnostic "Dashboard headers" +# # Use curl -I to get the header and parse out just the X-Pi-hole one +# local full_curl_output_dashboard +# local dashboard +# full_curl_output_dashboard="$(curl -Is localhost/admin/)" +# dashboard=$(echo "${full_curl_output_dashboard}" | awk '/X-Pi-hole/' | tr -d '\r') +# # Store what the X-Header should be in variables for comparison later +# local dashboard_working +# dashboard_working="X-Pi-hole: The Pi-hole Web interface is working!" + +# # If the X-Header matches what a working system should have, +# if [[ $dashboard == "$dashboard_working" ]]; then +# # then we can show a success +# log_write "$TICK Web interface X-Header: ${COL_GREEN}${dashboard}${COL_NC}" +# else +# # Otherwise, it's a failure since the X-Headers either don't exist or have been modified in some way +# log_write "$CROSS Web interface X-Header: ${COL_RED}X-Header does not match or could not be retrieved.${COL_NC}" + +# log_write "${COL_RED}${full_curl_output_dashboard}${COL_NC}" +# fi +# } dig_at() { # We need to test if Pi-hole can properly resolve domain names diff --git a/advanced/lighttpd.conf.debian b/advanced/lighttpd.conf.debian deleted file mode 100644 index f31f7bcd..00000000 --- a/advanced/lighttpd.conf.debian +++ /dev/null @@ -1,73 +0,0 @@ -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Lighttpd config for Pi-hole -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. - -################################################################################################### -# IF THIS HEADER EXISTS, THE FILE WILL BE OVERWRITTEN BY PI-HOLE'S UPDATE PROCEDURE. # -# ANY CHANGES MADE TO THIS FILE WILL BE LOST ON THE NEXT UPDATE UNLESS YOU REMOVE THIS HEADER # -# # -# ENSURE THAT YOU DO NOT REMOVE THE REQUIRED LINE: # -# # -# include "/etc/lighttpd/conf-enabled/*.conf" # -# # -################################################################################################### - -server.modules = ( - "mod_access", - "mod_auth", - "mod_expire", - "mod_redirect", - "mod_setenv", - "mod_rewrite" -) - -server.document-root = "/var/www/html" -server.upload-dirs = ( "/var/cache/lighttpd/uploads" ) -server.errorlog = "/var/log/lighttpd/error-pihole.log" -server.pid-file = "/run/lighttpd.pid" -server.username = "www-data" -server.groupname = "www-data" -# For lighttpd version 1.4.46 or above, the port can be overwritten in `/etc/lighttpd/external.conf` using the := operator -# e.g. server.port := 8000 -server.port = 80 - -# Allow streaming response -# reference: https://redmine.lighttpd.net/projects/lighttpd/wiki/Server_stream-response-bodyDetails -server.stream-response-body = 1 -#ssl.read-ahead = "disable" - -index-file.names = ( "index.php", "index.html", "index.lighttpd.html" ) -url.access-deny = ( "~", ".inc", ".md", ".yml", ".ini" ) -static-file.exclude-extensions = ( ".php", ".pl", ".fcgi" ) - -mimetype.assign = ( - ".ico" => "image/x-icon", - ".jpeg" => "image/jpeg", - ".jpg" => "image/jpeg", - ".png" => "image/png", - ".svg" => "image/svg+xml", - ".css" => "text/css; charset=utf-8", - ".html" => "text/html; charset=utf-8", - ".js" => "text/javascript; charset=utf-8", - ".json" => "application/json; charset=utf-8", - ".map" => "application/json; charset=utf-8", - ".txt" => "text/plain; charset=utf-8", - ".eot" => "application/vnd.ms-fontobject", - ".otf" => "font/otf", - ".ttc" => "font/collection", - ".ttf" => "font/ttf", - ".woff" => "font/woff", - ".woff2" => "font/woff2" -) - -# Add user chosen options held in (optional) external file -include "external*.conf" - -# default listening port for IPv6 falls back to the IPv4 port -include_shell "/usr/share/lighttpd/use-ipv6.pl " + server.port -include "/etc/lighttpd/conf-enabled/*.conf" diff --git a/advanced/lighttpd.conf.fedora b/advanced/lighttpd.conf.fedora deleted file mode 100644 index e09d7760..00000000 --- a/advanced/lighttpd.conf.fedora +++ /dev/null @@ -1,87 +0,0 @@ -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Lighttpd config for Pi-hole -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. - -################################################################################################### -# IF THIS HEADER EXISTS, THE FILE WILL BE OVERWRITTEN BY PI-HOLE'S UPDATE PROCEDURE. # -# ANY CHANGES MADE TO THIS FILE WILL BE LOST ON THE NEXT UPDATE UNLESS YOU REMOVE THIS HEADER # -# # -# ENSURE THAT YOU DO NOT REMOVE THE REQUIRED LINE: # -# # -# include "/etc/lighttpd/conf.d/pihole-admin.conf" # -# # -################################################################################################### - -server.modules = ( - "mod_access", - "mod_auth", - "mod_expire", - "mod_fastcgi", - "mod_accesslog", - "mod_redirect", - "mod_setenv", - "mod_rewrite" -) - -server.document-root = "/var/www/html" -server.upload-dirs = ( "/var/cache/lighttpd/uploads" ) -server.errorlog = "/var/log/lighttpd/error-pihole.log" -server.pid-file = "/run/lighttpd.pid" -server.username = "lighttpd" -server.groupname = "lighttpd" -# For lighttpd version 1.4.46 or above, the port can be overwritten in `/etc/lighttpd/external.conf` using the := operator -# e.g. server.port := 8000 -server.port = 80 - -# Allow streaming response -# reference: https://redmine.lighttpd.net/projects/lighttpd/wiki/Server_stream-response-bodyDetails -server.stream-response-body = 1 -#ssl.read-ahead = "disable" - -index-file.names = ( "index.php", "index.html", "index.lighttpd.html" ) -url.access-deny = ( "~", ".inc", ".md", ".yml", ".ini" ) -static-file.exclude-extensions = ( ".php", ".pl", ".fcgi" ) - -mimetype.assign = ( - ".ico" => "image/x-icon", - ".jpeg" => "image/jpeg", - ".jpg" => "image/jpeg", - ".png" => "image/png", - ".svg" => "image/svg+xml", - ".css" => "text/css; charset=utf-8", - ".html" => "text/html; charset=utf-8", - ".js" => "text/javascript; charset=utf-8", - ".json" => "application/json; charset=utf-8", - ".map" => "application/json; charset=utf-8", - ".txt" => "text/plain; charset=utf-8", - ".eot" => "application/vnd.ms-fontobject", - ".otf" => "font/otf", - ".ttc" => "font/collection", - ".ttf" => "font/ttf", - ".woff" => "font/woff", - ".woff2" => "font/woff2" -) - -# Add user chosen options held in (optional) external file -include "external*.conf" - -# default listening port for IPv6 falls back to the IPv4 port -#include_shell "/usr/share/lighttpd/use-ipv6.pl " + server.port -#include_shell "/usr/share/lighttpd/create-mime.assign.pl" -#include_shell "/usr/share/lighttpd/include-conf-enabled.pl" - -fastcgi.server = ( - ".php" => ( - "localhost" => ( - "socket" => "/tmp/php-fastcgi.socket", - "bin-path" => "/usr/bin/php-cgi" - ) - ) -) - -include "/etc/lighttpd/conf.d/pihole-admin.conf" diff --git a/advanced/pihole-admin.conf b/advanced/pihole-admin.conf deleted file mode 100644 index 0bb6eac9..00000000 --- a/advanced/pihole-admin.conf +++ /dev/null @@ -1,82 +0,0 @@ -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Lighttpd config for Pi-hole -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. - -############################################################################### -# FILE AUTOMATICALLY OVERWRITTEN BY PI-HOLE INSTALL/UPDATE PROCEDURE. # -# ANY CHANGES MADE TO THIS FILE AFTER INSTALL WILL BE LOST ON THE NEXT UPDATE # -############################################################################### - -server.errorlog := "/var/log/lighttpd/error-pihole.log" - -$HTTP["url"] =~ "^/admin/" { - server.document-root = "/var/www/html" - server.stream-response-body = 1 - accesslog.filename = "/var/log/lighttpd/access-pihole.log" - accesslog.format = "%{%s}t|%h|%V|%r|%s|%b" - - fastcgi.server = ( - ".php" => ( - "localhost" => ( - "socket" => "/run/lighttpd/pihole-php-fastcgi.socket", - "bin-path" => "/usr/bin/php-cgi", - "min-procs" => 1, - "max-procs" => 1, - "bin-environment" => ( - "PHP_FCGI_CHILDREN" => "4", - "PHP_FCGI_MAX_REQUESTS" => "10000", - ), - "bin-copy-environment" => ( - "PATH", "SHELL", "USER" - ), - "broken-scriptfilename" => "enable", - ) - ) - ) - - # X-Pi-hole is a response header for debugging using curl -I - # X-Frame-Options prevents clickjacking attacks and helps ensure your content is not embedded into other sites via < frame >, < iframe > or < object >. - # X-XSS-Protection sets the configuration for the cross-site scripting filters built into most browsers. This is important because it tells the browser to block the response if a malicious script has been inserted from a user input. (deprecated; disabled) - # X-Content-Type-Options stops a browser from trying to MIME-sniff the content type and forces it to stick with the declared content-type. This is important because the browser will only load external resources if their content-type matches what is expected, and not malicious hidden code. - # Content-Security-Policy tells the browser where resources are allowed to be loaded and if it’s allowed to parse/run inline styles or Javascript. This is important because it prevents content injection attacks, such as Cross Site Scripting (XSS). - # X-Permitted-Cross-Domain-Policies is an XML document that grants a web client, such as Adobe Flash Player or Adobe Acrobat (though not necessarily limited to these), permission to handle data across domains. - # Referrer-Policy allows control/restriction of the amount of information present in the referral header for links away from your page—the URL path or even if the header is sent at all. - setenv.add-response-header = ( - "X-Pi-hole" => "The Pi-hole Web interface is working!", - "X-Frame-Options" => "DENY", - "X-XSS-Protection" => "0", - "X-Content-Type-Options" => "nosniff", - "Content-Security-Policy" => "default-src 'self' 'unsafe-inline';", - "X-Permitted-Cross-Domain-Policies" => "none", - "Referrer-Policy" => "same-origin" - ) - - # Block . files from being served, such as .git, .github, .gitignore - $HTTP["url"] =~ "^/admin/\." { - url.access-deny = ("") - } - - # allow teleporter and API qr code iframe on settings page - $HTTP["url"] =~ "/(teleporter|api_token)\.php$" { - $HTTP["referer"] =~ "/admin/settings\.php" { - setenv.set-response-header = ( "X-Frame-Options" => "SAMEORIGIN" ) - } - } -} -else $HTTP["url"] == "/admin" { - url.redirect = ("" => "/admin/") -} - -$HTTP["host"] == "pi.hole" { - $HTTP["url"] == "/" { - url.redirect = ("" => "/admin/") - } -} - -# (keep this on one line for basic-install.sh filtering during install) -server.modules += ( "mod_access", "mod_accesslog", "mod_redirect", "mod_fastcgi", "mod_setenv" ) From d2d11959289fedfc0490921269a35e864742cf82 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 1 Jan 2023 12:49:07 +0000 Subject: [PATCH 004/200] Needs tweaking, offer chance to change web interface port Signed-off-by: Adam Warner --- automated install/basic-install.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 3615804c..d9e22e08 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -92,6 +92,7 @@ IPV6_ADDRESS=${IPV6_ADDRESS} # Give settings their default values. These may be changed by prompts later in the script. QUERY_LOGGING=true INSTALL_WEB_INTERFACE=true +WEBPORT=8080 PRIVACY_LEVEL=0 CACHE_SIZE=10000 @@ -1062,6 +1063,14 @@ setAdminFlag() { printf " %b Installing Admin Web Interface\\n" "${INFO}" # Set the flag to install the web interface INSTALL_WEB_INTERFACE=true + + # Web port TODO: Below whiptail copy pasted from a previous go at this. needs converting to dialog + # Ask for the IPv4 address + WEBPORT=$(whiptail --backtitle "Setting web interface port" --title "Web Port" --inputbox "By default, pihole-FTL listens for http traffic on port 8080. If you wish to change the port, you may do so now. You can also do it later by editing /etc/pihole/pihole-FTL.conf" "${r}" "${c}" "${WEBPORT}" 3>&1 1>&2 2>&3) || \ + # Canceling IPv4 settings window + { echo -e " ${COL_LIGHT_RED}Cancel was selected, exiting installer${COL_NC}"; exit 1; } + printf " %b The Web interface will be accessible on port: %s\\n" "${INFO}" "${WEBPORT}" + ;; "${DIALOG_CANCEL}") # If they chose no, @@ -1662,6 +1671,9 @@ finalExports() { # Set the privacy level addOrEditKeyValPair "${FTL_CONFIG_FILE}" "PRIVACYLEVEL" "${PRIVACY_LEVEL}" + # Set the web port + addOrEditKeyValPair "${FTL_CONFIG_FILE}" "WEBPORT" "${PRIVACY_LEVEL}" + # Bring in the current settings and the functions to manipulate them source "${setupVars}" # shellcheck source=advanced/Scripts/webpage.sh @@ -1819,7 +1831,7 @@ displayFinalMessage() { # If the user wants to install the dashboard, if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then # Store a message in a variable and display it - additional="View the web interface at http://pi.hole/admin or http://${IPV4_ADDRESS%/*}/admin\\n\\nYour Admin Webpage login password is ${pwstring}" + additional="View the web interface at http://pi.hole/admin:${WEBPORT} or http://${IPV4_ADDRESS%/*}:${WEBPORT}/admin\\n\\nYour Admin Webpage login password is ${pwstring}" fi # Final completion message to user @@ -2524,7 +2536,7 @@ main() { if [[ "${useUpdateVars}" == false ]]; then # If the Web interface was installed, if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - printf " %b View the web interface at http://pi.hole/admin or http://%s/admin\\n\\n" "${INFO}" "${IPV4_ADDRESS%/*}" + printf " %b View the web interface at http://pi.hole:${WEBPORT}/admin or http://%s/admin\\n\\n" "${INFO}" "${IPV4_ADDRESS%/*}:${WEBPORT}" fi # Explain to the user how to use Pi-hole as their DNS server printf " %b You may now configure your devices to use the Pi-hole as their DNS server\\n" "${INFO}" From 31f16510e2e469b48780625ed45d0f4312f9a9da Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 13 Jan 2023 21:23:50 +0000 Subject: [PATCH 005/200] Clear out some no longer needed ftl/dnsmasq config code Signed-off-by: Adam Warner --- advanced/01-pihole.conf | 35 -- advanced/06-rfc6761.conf | 42 -- advanced/dnsmasq.conf.original | 648 ----------------------------- automated install/basic-install.sh | 114 +---- 4 files changed, 20 insertions(+), 819 deletions(-) delete mode 100644 advanced/01-pihole.conf delete mode 100644 advanced/06-rfc6761.conf delete mode 100644 advanced/dnsmasq.conf.original diff --git a/advanced/01-pihole.conf b/advanced/01-pihole.conf deleted file mode 100644 index 677910f6..00000000 --- a/advanced/01-pihole.conf +++ /dev/null @@ -1,35 +0,0 @@ -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Dnsmasq config for Pi-hole's FTLDNS -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. - -############################################################################### -# FILE AUTOMATICALLY POPULATED BY PI-HOLE INSTALL/UPDATE PROCEDURE. # -# ANY CHANGES MADE TO THIS FILE AFTER INSTALL WILL BE LOST ON THE NEXT UPDATE # -# # -# IF YOU WISH TO CHANGE THE UPSTREAM SERVERS, CHANGE THEM IN: # -# /etc/pihole/setupVars.conf # -# # -# ANY OTHER CHANGES SHOULD BE MADE IN A SEPARATE CONFIG FILE # -# WITHIN /etc/dnsmasq.d/yourname.conf # -############################################################################### - -addn-hosts=/etc/pihole/local.list -addn-hosts=/etc/pihole/custom.list - -domain-needed - -localise-queries - -bogus-priv - -no-resolv - -log-queries -log-facility=/var/log/pihole/pihole.log - -log-async diff --git a/advanced/06-rfc6761.conf b/advanced/06-rfc6761.conf deleted file mode 100644 index fcdd0010..00000000 --- a/advanced/06-rfc6761.conf +++ /dev/null @@ -1,42 +0,0 @@ -# Pi-hole: A black hole for Internet advertisements -# (c) 2021 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# RFC 6761 config file for Pi-hole -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. - -############################################################################### -# FILE AUTOMATICALLY POPULATED BY PI-HOLE INSTALL/UPDATE PROCEDURE. # -# ANY CHANGES MADE TO THIS FILE AFTER INSTALL WILL BE LOST ON THE NEXT UPDATE # -# # -# CHANGES SHOULD BE MADE IN A SEPARATE CONFIG FILE # -# WITHIN /etc/dnsmasq.d/yourname.conf # -############################################################################### - -# RFC 6761: Caching DNS servers SHOULD recognize -# test, localhost, invalid -# names as special and SHOULD NOT attempt to look up NS records for them, or -# otherwise query authoritative DNS servers in an attempt to resolve these -# names. -server=/test/ -server=/localhost/ -server=/invalid/ - -# The same RFC requests something similar for -# 10.in-addr.arpa. 21.172.in-addr.arpa. 27.172.in-addr.arpa. -# 16.172.in-addr.arpa. 22.172.in-addr.arpa. 28.172.in-addr.arpa. -# 17.172.in-addr.arpa. 23.172.in-addr.arpa. 29.172.in-addr.arpa. -# 18.172.in-addr.arpa. 24.172.in-addr.arpa. 30.172.in-addr.arpa. -# 19.172.in-addr.arpa. 25.172.in-addr.arpa. 31.172.in-addr.arpa. -# 20.172.in-addr.arpa. 26.172.in-addr.arpa. 168.192.in-addr.arpa. -# Pi-hole implements this via the dnsmasq option "bogus-priv" (see -# 01-pihole.conf) because this also covers IPv6. - -# OpenWRT furthermore blocks bind, local, onion domains -# see https://git.openwrt.org/?p=openwrt/openwrt.git;a=blob_plain;f=package/network/services/dnsmasq/files/rfc6761.conf;hb=HEAD -# and https://www.iana.org/assignments/special-use-domain-names/special-use-domain-names.xhtml -# We do not include the ".local" rule ourselves, see https://github.com/pi-hole/pi-hole/pull/4282#discussion_r689112972 -server=/bind/ -server=/onion/ diff --git a/advanced/dnsmasq.conf.original b/advanced/dnsmasq.conf.original deleted file mode 100644 index 4aa5a8bf..00000000 --- a/advanced/dnsmasq.conf.original +++ /dev/null @@ -1,648 +0,0 @@ -# Configuration file for dnsmasq. -# -# Format is one option per line, legal options are the same -# as the long options legal on the command line. See -# "/usr/sbin/dnsmasq --help" or "man 8 dnsmasq" for details. - -# Listen on this specific port instead of the standard DNS port -# (53). Setting this to zero completely disables DNS function, -# leaving only DHCP and/or TFTP. -#port=5353 - -# The following two options make you a better netizen, since they -# tell dnsmasq to filter out queries which the public DNS cannot -# answer, and which load the servers (especially the root servers) -# unnecessarily. If you have a dial-on-demand link they also stop -# these requests from bringing up the link unnecessarily. - -# Never forward plain names (without a dot or domain part) -#domain-needed -# Never forward addresses in the non-routed address spaces. -#bogus-priv - -# Uncomment these to enable DNSSEC validation and caching: -# (Requires dnsmasq to be built with DNSSEC option.) -#conf-file=%%PREFIX%%/share/dnsmasq/trust-anchors.conf -#dnssec - -# Replies which are not DNSSEC signed may be legitimate, because the domain -# is unsigned, or may be forgeries. Setting this option tells dnsmasq to -# check that an unsigned reply is OK, by finding a secure proof that a DS -# record somewhere between the root and the domain does not exist. -# The cost of setting this is that even queries in unsigned domains will need -# one or more extra DNS queries to verify. -#dnssec-check-unsigned - -# Uncomment this to filter useless windows-originated DNS requests -# which can trigger dial-on-demand links needlessly. -# Note that (amongst other things) this blocks all SRV requests, -# so don't use it if you use eg Kerberos, SIP, XMMP or Google-talk. -# This option only affects forwarding, SRV records originating for -# dnsmasq (via srv-host= lines) are not suppressed by it. -#filterwin2k - -# Change this line if you want dns to get its upstream servers from -# somewhere other that /etc/resolv.conf -#resolv-file= - -# By default, dnsmasq will send queries to any of the upstream -# servers it knows about and tries to favor servers to are known -# to be up. Uncommenting this forces dnsmasq to try each query -# with each server strictly in the order they appear in -# /etc/resolv.conf -#strict-order - -# If you don't want dnsmasq to read /etc/resolv.conf or any other -# file, getting its servers from this file instead (see below), then -# uncomment this. -#no-resolv - -# If you don't want dnsmasq to poll /etc/resolv.conf or other resolv -# files for changes and re-read them then uncomment this. -#no-poll - -# Add other name servers here, with domain specs if they are for -# non-public domains. -#server=/localnet/192.168.0.1 - -# Example of routing PTR queries to nameservers: this will send all -# address->name queries for 192.168.3/24 to nameserver 10.1.2.3 -#server=/3.168.192.in-addr.arpa/10.1.2.3 - -# Add local-only domains here, queries in these domains are answered -# from /etc/hosts or DHCP only. -#local=/localnet/ - -# Add domains which you want to force to an IP address here. -# The example below send any host in double-click.net to a local -# web-server. -#address=/double-click.net/127.0.0.1 - -# --address (and --server) work with IPv6 addresses too. -#address=/www.thekelleys.org.uk/fe80::20d:60ff:fe36:f83 - -# Add the IPs of all queries to yahoo.com, google.com, and their -# subdomains to the vpn and search ipsets: -#ipset=/yahoo.com/google.com/vpn,search - -# You can control how dnsmasq talks to a server: this forces -# queries to 10.1.2.3 to be routed via eth1 -# server=10.1.2.3@eth1 - -# and this sets the source (ie local) address used to talk to -# 10.1.2.3 to 192.168.1.1 port 55 (there must be a interface with that -# IP on the machine, obviously). -# server=10.1.2.3@192.168.1.1#55 - -# If you want dnsmasq to change uid and gid to something other -# than the default, edit the following lines. -#user= -#group= - -# If you want dnsmasq to listen for DHCP and DNS requests only on -# specified interfaces (and the loopback) give the name of the -# interface (eg eth0) here. -# Repeat the line for more than one interface. -#interface= -# Or you can specify which interface _not_ to listen on -#except-interface= -# Or which to listen on by address (remember to include 127.0.0.1 if -# you use this.) -#listen-address= -# If you want dnsmasq to provide only DNS service on an interface, -# configure it as shown above, and then use the following line to -# disable DHCP and TFTP on it. -#no-dhcp-interface= - -# On systems which support it, dnsmasq binds the wildcard address, -# even when it is listening on only some interfaces. It then discards -# requests that it shouldn't reply to. This has the advantage of -# working even when interfaces come and go and change address. If you -# want dnsmasq to really bind only the interfaces it is listening on, -# uncomment this option. About the only time you may need this is when -# running another nameserver on the same machine. -#bind-interfaces - -# If you don't want dnsmasq to read /etc/hosts, uncomment the -# following line. -#no-hosts -# or if you want it to read another file, as well as /etc/hosts, use -# this. -#addn-hosts=/etc/banner_add_hosts - -# Set this (and domain: see below) if you want to have a domain -# automatically added to simple names in a hosts-file. -#expand-hosts - -# Set the domain for dnsmasq. this is optional, but if it is set, it -# does the following things. -# 1) Allows DHCP hosts to have fully qualified domain names, as long -# as the domain part matches this setting. -# 2) Sets the "domain" DHCP option thereby potentially setting the -# domain of all systems configured by DHCP -# 3) Provides the domain part for "expand-hosts" -#domain=thekelleys.org.uk - -# Set a different domain for a particular subnet -#domain=wireless.thekelleys.org.uk,192.168.2.0/24 - -# Same idea, but range rather then subnet -#domain=reserved.thekelleys.org.uk,192.68.3.100,192.168.3.200 - -# Uncomment this to enable the integrated DHCP server, you need -# to supply the range of addresses available for lease and optionally -# a lease time. If you have more than one network, you will need to -# repeat this for each network on which you want to supply DHCP -# service. -#dhcp-range=192.168.0.50,192.168.0.150,12h - -# This is an example of a DHCP range where the netmask is given. This -# is needed for networks we reach the dnsmasq DHCP server via a relay -# agent. If you don't know what a DHCP relay agent is, you probably -# don't need to worry about this. -#dhcp-range=192.168.0.50,192.168.0.150,255.255.255.0,12h - -# This is an example of a DHCP range which sets a tag, so that -# some DHCP options may be set only for this network. -#dhcp-range=set:red,192.168.0.50,192.168.0.150 - -# Use this DHCP range only when the tag "green" is set. -#dhcp-range=tag:green,192.168.0.50,192.168.0.150,12h - -# Specify a subnet which can't be used for dynamic address allocation, -# is available for hosts with matching --dhcp-host lines. Note that -# dhcp-host declarations will be ignored unless there is a dhcp-range -# of some type for the subnet in question. -# In this case the netmask is implied (it comes from the network -# configuration on the machine running dnsmasq) it is possible to give -# an explicit netmask instead. -#dhcp-range=192.168.0.0,static - -# Enable DHCPv6. Note that the prefix-length does not need to be specified -# and defaults to 64 if missing/ -#dhcp-range=1234::2, 1234::500, 64, 12h - -# Do Router Advertisements, BUT NOT DHCP for this subnet. -#dhcp-range=1234::, ra-only - -# Do Router Advertisements, BUT NOT DHCP for this subnet, also try and -# add names to the DNS for the IPv6 address of SLAAC-configured dual-stack -# hosts. Use the DHCPv4 lease to derive the name, network segment and -# MAC address and assume that the host will also have an -# IPv6 address calculated using the SLAAC algorithm. -#dhcp-range=1234::, ra-names - -# Do Router Advertisements, BUT NOT DHCP for this subnet. -# Set the lifetime to 46 hours. (Note: minimum lifetime is 2 hours.) -#dhcp-range=1234::, ra-only, 48h - -# Do DHCP and Router Advertisements for this subnet. Set the A bit in the RA -# so that clients can use SLAAC addresses as well as DHCP ones. -#dhcp-range=1234::2, 1234::500, slaac - -# Do Router Advertisements and stateless DHCP for this subnet. Clients will -# not get addresses from DHCP, but they will get other configuration information. -# They will use SLAAC for addresses. -#dhcp-range=1234::, ra-stateless - -# Do stateless DHCP, SLAAC, and generate DNS names for SLAAC addresses -# from DHCPv4 leases. -#dhcp-range=1234::, ra-stateless, ra-names - -# Do router advertisements for all subnets where we're doing DHCPv6 -# Unless overridden by ra-stateless, ra-names, et al, the router -# advertisements will have the M and O bits set, so that the clients -# get addresses and configuration from DHCPv6, and the A bit reset, so the -# clients don't use SLAAC addresses. -#enable-ra - -# Supply parameters for specified hosts using DHCP. There are lots -# of valid alternatives, so we will give examples of each. Note that -# IP addresses DO NOT have to be in the range given above, they just -# need to be on the same network. The order of the parameters in these -# do not matter, it's permissible to give name, address and MAC in any -# order. - -# Always allocate the host with Ethernet address 11:22:33:44:55:66 -# The IP address 192.168.0.60 -#dhcp-host=11:22:33:44:55:66,192.168.0.60 - -# Always set the name of the host with hardware address -# 11:22:33:44:55:66 to be "fred" -#dhcp-host=11:22:33:44:55:66,fred - -# Always give the host with Ethernet address 11:22:33:44:55:66 -# the name fred and IP address 192.168.0.60 and lease time 45 minutes -#dhcp-host=11:22:33:44:55:66,fred,192.168.0.60,45m - -# Give a host with Ethernet address 11:22:33:44:55:66 or -# 12:34:56:78:90:12 the IP address 192.168.0.60. Dnsmasq will assume -# that these two Ethernet interfaces will never be in use at the same -# time, and give the IP address to the second, even if it is already -# in use by the first. Useful for laptops with wired and wireless -# addresses. -#dhcp-host=11:22:33:44:55:66,12:34:56:78:90:12,192.168.0.60 - -# Give the machine which says its name is "bert" IP address -# 192.168.0.70 and an infinite lease -#dhcp-host=bert,192.168.0.70,infinite - -# Always give the host with client identifier 01:02:02:04 -# the IP address 192.168.0.60 -#dhcp-host=id:01:02:02:04,192.168.0.60 - -# Always give the host with client identifier "marjorie" -# the IP address 192.168.0.60 -#dhcp-host=id:marjorie,192.168.0.60 - -# Enable the address given for "judge" in /etc/hosts -# to be given to a machine presenting the name "judge" when -# it asks for a DHCP lease. -#dhcp-host=judge - -# Never offer DHCP service to a machine whose Ethernet -# address is 11:22:33:44:55:66 -#dhcp-host=11:22:33:44:55:66,ignore - -# Ignore any client-id presented by the machine with Ethernet -# address 11:22:33:44:55:66. This is useful to prevent a machine -# being treated differently when running under different OS's or -# between PXE boot and OS boot. -#dhcp-host=11:22:33:44:55:66,id:* - -# Send extra options which are tagged as "red" to -# the machine with Ethernet address 11:22:33:44:55:66 -#dhcp-host=11:22:33:44:55:66,set:red - -# Send extra options which are tagged as "red" to -# any machine with Ethernet address starting 11:22:33: -#dhcp-host=11:22:33:*:*:*,set:red - -# Give a fixed IPv6 address and name to client with -# DUID 00:01:00:01:16:d2:83:fc:92:d4:19:e2:d8:b2 -# Note the MAC addresses CANNOT be used to identify DHCPv6 clients. -# Note also the they [] around the IPv6 address are obligatory. -#dhcp-host=id:00:01:00:01:16:d2:83:fc:92:d4:19:e2:d8:b2, fred, [1234::5] - -# Ignore any clients which are not specified in dhcp-host lines -# or /etc/ethers. Equivalent to ISC "deny unknown-clients". -# This relies on the special "known" tag which is set when -# a host is matched. -#dhcp-ignore=tag:!known - -# Send extra options which are tagged as "red" to any machine whose -# DHCP vendorclass string includes the substring "Linux" -#dhcp-vendorclass=set:red,Linux - -# Send extra options which are tagged as "red" to any machine one -# of whose DHCP userclass strings includes the substring "accounts" -#dhcp-userclass=set:red,accounts - -# Send extra options which are tagged as "red" to any machine whose -# MAC address matches the pattern. -#dhcp-mac=set:red,00:60:8C:*:*:* - -# If this line is uncommented, dnsmasq will read /etc/ethers and act -# on the ethernet-address/IP pairs found there just as if they had -# been given as --dhcp-host options. Useful if you keep -# MAC-address/host mappings there for other purposes. -#read-ethers - -# Send options to hosts which ask for a DHCP lease. -# See RFC 2132 for details of available options. -# Common options can be given to dnsmasq by name: -# run "dnsmasq --help dhcp" to get a list. -# Note that all the common settings, such as netmask and -# broadcast address, DNS server and default route, are given -# sane defaults by dnsmasq. You very likely will not need -# any dhcp-options. If you use Windows clients and Samba, there -# are some options which are recommended, they are detailed at the -# end of this section. - -# Override the default route supplied by dnsmasq, which assumes the -# router is the same machine as the one running dnsmasq. -#dhcp-option=3,1.2.3.4 - -# Do the same thing, but using the option name -#dhcp-option=option:router,1.2.3.4 - -# Override the default route supplied by dnsmasq and send no default -# route at all. Note that this only works for the options sent by -# default (1, 3, 6, 12, 28) the same line will send a zero-length option -# for all other option numbers. -#dhcp-option=3 - -# Set the NTP time server addresses to 192.168.0.4 and 10.10.0.5 -#dhcp-option=option:ntp-server,192.168.0.4,10.10.0.5 - -# Send DHCPv6 option. Note [] around IPv6 addresses. -#dhcp-option=option6:dns-server,[1234::77],[1234::88] - -# Send DHCPv6 option for namservers as the machine running -# dnsmasq and another. -#dhcp-option=option6:dns-server,[::],[1234::88] - -# Ask client to poll for option changes every six hours. (RFC4242) -#dhcp-option=option6:information-refresh-time,6h - -# Set the NTP time server address to be the same machine as -# is running dnsmasq -#dhcp-option=42,0.0.0.0 - -# Set the NIS domain name to "welly" -#dhcp-option=40,welly - -# Set the default time-to-live to 50 -#dhcp-option=23,50 - -# Set the "all subnets are local" flag -#dhcp-option=27,1 - -# Send the etherboot magic flag and then etherboot options (a string). -#dhcp-option=128,e4:45:74:68:00:00 -#dhcp-option=129,NIC=eepro100 - -# Specify an option which will only be sent to the "red" network -# (see dhcp-range for the declaration of the "red" network) -# Note that the tag: part must precede the option: part. -#dhcp-option = tag:red, option:ntp-server, 192.168.1.1 - -# The following DHCP options set up dnsmasq in the same way as is specified -# for the ISC dhcpcd in -# http://www.samba.org/samba/ftp/docs/textdocs/DHCP-Server-Configuration.txt -# adapted for a typical dnsmasq installation where the host running -# dnsmasq is also the host running samba. -# you may want to uncomment some or all of them if you use -# Windows clients and Samba. -#dhcp-option=19,0 # option ip-forwarding off -#dhcp-option=44,0.0.0.0 # set netbios-over-TCP/IP nameserver(s) aka WINS server(s) -#dhcp-option=45,0.0.0.0 # netbios datagram distribution server -#dhcp-option=46,8 # netbios node type - -# Send an empty WPAD option. This may be REQUIRED to get windows 7 to behave. -#dhcp-option=252,"\n" - -# Send RFC-3397 DNS domain search DHCP option. WARNING: Your DHCP client -# probably doesn't support this...... -#dhcp-option=option:domain-search,eng.apple.com,marketing.apple.com - -# Send RFC-3442 classless static routes (note the netmask encoding) -#dhcp-option=121,192.168.1.0/24,1.2.3.4,10.0.0.0/8,5.6.7.8 - -# Send vendor-class specific options encapsulated in DHCP option 43. -# The meaning of the options is defined by the vendor-class so -# options are sent only when the client supplied vendor class -# matches the class given here. (A substring match is OK, so "MSFT" -# matches "MSFT" and "MSFT 5.0"). This example sets the -# mtftp address to 0.0.0.0 for PXEClients. -#dhcp-option=vendor:PXEClient,1,0.0.0.0 - -# Send microsoft-specific option to tell windows to release the DHCP lease -# when it shuts down. Note the "i" flag, to tell dnsmasq to send the -# value as a four-byte integer - that's what microsoft wants. See -# http://technet2.microsoft.com/WindowsServer/en/library/a70f1bb7-d2d4-49f0-96d6-4b7414ecfaae1033.mspx?mfr=true -#dhcp-option=vendor:MSFT,2,1i - -# Send the Encapsulated-vendor-class ID needed by some configurations of -# Etherboot to allow is to recognize the DHCP server. -#dhcp-option=vendor:Etherboot,60,"Etherboot" - -# Send options to PXELinux. Note that we need to send the options even -# though they don't appear in the parameter request list, so we need -# to use dhcp-option-force here. -# See http://syslinux.zytor.com/pxe.php#special for details. -# Magic number - needed before anything else is recognized -#dhcp-option-force=208,f1:00:74:7e -# Configuration file name -#dhcp-option-force=209,configs/common -# Path prefix -#dhcp-option-force=210,/tftpboot/pxelinux/files/ -# Reboot time. (Note 'i' to send 32-bit value) -#dhcp-option-force=211,30i - -# Set the boot filename for netboot/PXE. You will only need -# this is you want to boot machines over the network and you will need -# a TFTP server; either dnsmasq's built in TFTP server or an -# external one. (See below for how to enable the TFTP server.) -#dhcp-boot=pxelinux.0 - -# The same as above, but use custom tftp-server instead machine running dnsmasq -#dhcp-boot=pxelinux,server.name,192.168.1.100 - -# Boot for Etherboot gPXE. The idea is to send two different -# filenames, the first loads gPXE, and the second tells gPXE what to -# load. The dhcp-match sets the gpxe tag for requests from gPXE. -#dhcp-match=set:gpxe,175 # gPXE sends a 175 option. -#dhcp-boot=tag:!gpxe,undionly.kpxe -#dhcp-boot=mybootimage - -# Encapsulated options for Etherboot gPXE. All the options are -# encapsulated within option 175 -#dhcp-option=encap:175, 1, 5b # priority code -#dhcp-option=encap:175, 176, 1b # no-proxydhcp -#dhcp-option=encap:175, 177, string # bus-id -#dhcp-option=encap:175, 189, 1b # BIOS drive code -#dhcp-option=encap:175, 190, user # iSCSI username -#dhcp-option=encap:175, 191, pass # iSCSI password - -# Test for the architecture of a netboot client. PXE clients are -# supposed to send their architecture as option 93. (See RFC 4578) -#dhcp-match=peecees, option:client-arch, 0 #x86-32 -#dhcp-match=itanics, option:client-arch, 2 #IA64 -#dhcp-match=hammers, option:client-arch, 6 #x86-64 -#dhcp-match=mactels, option:client-arch, 7 #EFI x86-64 - -# Do real PXE, rather than just booting a single file, this is an -# alternative to dhcp-boot. -#pxe-prompt="What system shall I netboot?" -# or with timeout before first available action is taken: -#pxe-prompt="Press F8 for menu.", 60 - -# Available boot services. for PXE. -#pxe-service=x86PC, "Boot from local disk" - -# Loads /pxelinux.0 from dnsmasq TFTP server. -#pxe-service=x86PC, "Install Linux", pxelinux - -# Loads /pxelinux.0 from TFTP server at 1.2.3.4. -# Beware this fails on old PXE ROMS. -#pxe-service=x86PC, "Install Linux", pxelinux, 1.2.3.4 - -# Use bootserver on network, found my multicast or broadcast. -#pxe-service=x86PC, "Install windows from RIS server", 1 - -# Use bootserver at a known IP address. -#pxe-service=x86PC, "Install windows from RIS server", 1, 1.2.3.4 - -# If you have multicast-FTP available, -# information for that can be passed in a similar way using options 1 -# to 5. See page 19 of -# http://download.intel.com/design/archives/wfm/downloads/pxespec.pdf - - -# Enable dnsmasq's built-in TFTP server -#enable-tftp - -# Set the root directory for files available via FTP. -#tftp-root=/var/ftpd - -# Make the TFTP server more secure: with this set, only files owned by -# the user dnsmasq is running as will be send over the net. -#tftp-secure - -# This option stops dnsmasq from negotiating a larger blocksize for TFTP -# transfers. It will slow things down, but may rescue some broken TFTP -# clients. -#tftp-no-blocksize - -# Set the boot file name only when the "red" tag is set. -#dhcp-boot=tag:red,pxelinux.red-net - -# An example of dhcp-boot with an external TFTP server: the name and IP -# address of the server are given after the filename. -# Can fail with old PXE ROMS. Overridden by --pxe-service. -#dhcp-boot=/var/ftpd/pxelinux.0,boothost,192.168.0.3 - -# If there are multiple external tftp servers having a same name -# (using /etc/hosts) then that name can be specified as the -# tftp_servername (the third option to dhcp-boot) and in that -# case dnsmasq resolves this name and returns the resultant IP -# addresses in round robin fashion. This facility can be used to -# load balance the tftp load among a set of servers. -#dhcp-boot=/var/ftpd/pxelinux.0,boothost,tftp_server_name - -# Set the limit on DHCP leases, the default is 150 -#dhcp-lease-max=150 - -# The DHCP server needs somewhere on disk to keep its lease database. -# This defaults to a sane location, but if you want to change it, use -# the line below. -#dhcp-leasefile=/var/lib/misc/dnsmasq.leases - -# Set the DHCP server to authoritative mode. In this mode it will barge in -# and take over the lease for any client which broadcasts on the network, -# whether it has a record of the lease or not. This avoids long timeouts -# when a machine wakes up on a new network. DO NOT enable this if there's -# the slightest chance that you might end up accidentally configuring a DHCP -# server for your campus/company accidentally. The ISC server uses -# the same option, and this URL provides more information: -# http://www.isc.org/files/auth.html -#dhcp-authoritative - -# Run an executable when a DHCP lease is created or destroyed. -# The arguments sent to the script are "add" or "del", -# then the MAC address, the IP address and finally the hostname -# if there is one. -#dhcp-script=/bin/echo - -# Set the cachesize here. -#cache-size=150 - -# If you want to disable negative caching, uncomment this. -#no-negcache - -# Normally responses which come from /etc/hosts and the DHCP lease -# file have Time-To-Live set as zero, which conventionally means -# do not cache further. If you are happy to trade lower load on the -# server for potentially stale date, you can set a time-to-live (in -# seconds) here. -#local-ttl= - -# If you want dnsmasq to detect attempts by Verisign to send queries -# to unregistered .com and .net hosts to its sitefinder service and -# have dnsmasq instead return the correct NXDOMAIN response, uncomment -# this line. You can add similar lines to do the same for other -# registries which have implemented wildcard A records. -#bogus-nxdomain=64.94.110.11 - -# If you want to fix up DNS results from upstream servers, use the -# alias option. This only works for IPv4. -# This alias makes a result of 1.2.3.4 appear as 5.6.7.8 -#alias=1.2.3.4,5.6.7.8 -# and this maps 1.2.3.x to 5.6.7.x -#alias=1.2.3.0,5.6.7.0,255.255.255.0 -# and this maps 192.168.0.10->192.168.0.40 to 10.0.0.10->10.0.0.40 -#alias=192.168.0.10-192.168.0.40,10.0.0.0,255.255.255.0 - -# Change these lines if you want dnsmasq to serve MX records. - -# Return an MX record named "maildomain.com" with target -# servermachine.com and preference 50 -#mx-host=maildomain.com,servermachine.com,50 - -# Set the default target for MX records created using the localmx option. -#mx-target=servermachine.com - -# Return an MX record pointing to the mx-target for all local -# machines. -#localmx - -# Return an MX record pointing to itself for all local machines. -#selfmx - -# Change the following lines if you want dnsmasq to serve SRV -# records. These are useful if you want to serve ldap requests for -# Active Directory and other windows-originated DNS requests. -# See RFC 2782. -# You may add multiple srv-host lines. -# The fields are ,,,, -# If the domain part if missing from the name (so that is just has the -# service and protocol sections) then the domain given by the domain= -# config option is used. (Note that expand-hosts does not need to be -# set for this to work.) - -# A SRV record sending LDAP for the example.com domain to -# ldapserver.example.com port 389 -#srv-host=_ldap._tcp.example.com,ldapserver.example.com,389 - -# A SRV record sending LDAP for the example.com domain to -# ldapserver.example.com port 389 (using domain=) -#domain=example.com -#srv-host=_ldap._tcp,ldapserver.example.com,389 - -# Two SRV records for LDAP, each with different priorities -#srv-host=_ldap._tcp.example.com,ldapserver.example.com,389,1 -#srv-host=_ldap._tcp.example.com,ldapserver.example.com,389,2 - -# A SRV record indicating that there is no LDAP server for the domain -# example.com -#srv-host=_ldap._tcp.example.com - -# The following line shows how to make dnsmasq serve an arbitrary PTR -# record. This is useful for DNS-SD. (Note that the -# domain-name expansion done for SRV records _does_not -# occur for PTR records.) -#ptr-record=_http._tcp.dns-sd-services,"New Employee Page._http._tcp.dns-sd-services" - -# Change the following lines to enable dnsmasq to serve TXT records. -# These are used for things like SPF and zeroconf. (Note that the -# domain-name expansion done for SRV records _does_not -# occur for TXT records.) - -#Example SPF. -#txt-record=example.com,"v=spf1 a -all" - -#Example zeroconf -#txt-record=_http._tcp.example.com,name=value,paper=A4 - -# Provide an alias for a "local" DNS name. Note that this _only_ works -# for targets which are names from DHCP or /etc/hosts. Give host -# "bert" another name, bertrand -#cname=bertand,bert - -# For debugging purposes, log each DNS query as it passes through -# dnsmasq. -#log-queries - -# Log lots of extra information about DHCP transactions. -#log-dhcp - -# Include another lot of configuration options. -#conf-file=/etc/dnsmasq.more.conf -#conf-dir=/etc/dnsmasq.d - -# Include all the files in a directory except those ending in .bak -#conf-dir=/etc/dnsmasq.d,.bak - -# Include all files in a directory which end in .conf -#conf-dir=/etc/dnsmasq.d/*.conf diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index d9e22e08..7d899940 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1137,83 +1137,31 @@ installDefaultBlocklists() { echo "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" >> "${adlistFile}" } -# Check if /etc/dnsmasq.conf is from pi-hole. If so replace with an original and install new in .d directory -version_check_dnsmasq() { +remove_old_dnsmasq_ftl_configs() { # Local, named variables local dnsmasq_conf="/etc/dnsmasq.conf" - local dnsmasq_conf_orig="/etc/dnsmasq.conf.orig" - local dnsmasq_pihole_id_string="addn-hosts=/etc/pihole/gravity.list" - local dnsmasq_pihole_id_string2="# Dnsmasq config for Pi-hole's FTLDNS" - local dnsmasq_original_config="${PI_HOLE_LOCAL_REPO}/advanced/dnsmasq.conf.original" - local dnsmasq_pihole_01_source="${PI_HOLE_LOCAL_REPO}/advanced/01-pihole.conf" - local dnsmasq_pihole_01_target="/etc/dnsmasq.d/01-pihole.conf" - local dnsmasq_rfc6761_06_source="${PI_HOLE_LOCAL_REPO}/advanced/06-rfc6761.conf" - local dnsmasq_rfc6761_06_target="/etc/dnsmasq.d/06-rfc6761.conf" + local pihole_01="/etc/dnsmasq.d/01-pihole.conf" + local rfc6761_06="/etc/dnsmasq.d/06-rfc6761.conf" + local pihole_dhcp_02="/etc/dnsmasq.d/02-pihole-dhcp.conf" + + # pihole-FTL does some fancy stuff with config these days, and so we can remove some old config files + if [[ -f "${pihole_01}" ]]; then + rm "${pihole_01}" + fi + + if [[ -f "${rfc6761_06}" ]]; then + rm "${rfc6761_06}" + fi + + if [[ -f "${pihole_dhcp_02}" ]]; then + rm "${pihole_dhcp_02}" + fi # If the dnsmasq config file exists if [[ -f "${dnsmasq_conf}" ]]; then - printf " %b Existing dnsmasq.conf found..." "${INFO}" - # If a specific string is found within this file, we presume it's from older versions on Pi-hole, - if grep -q "${dnsmasq_pihole_id_string}" "${dnsmasq_conf}" || - grep -q "${dnsmasq_pihole_id_string2}" "${dnsmasq_conf}"; then - printf " it is from a previous Pi-hole install.\\n" - printf " %b Backing up dnsmasq.conf to dnsmasq.conf.orig..." "${INFO}" - # so backup the original file, - mv -f "${dnsmasq_conf}" "${dnsmasq_conf_orig}" - printf "%b %b Backing up dnsmasq.conf to dnsmasq.conf.orig...\\n" "${OVER}" "${TICK}" - printf " %b Restoring default dnsmasq.conf..." "${INFO}" - # and replace it with the default - install -D -m 644 -T "${dnsmasq_original_config}" "${dnsmasq_conf}" - printf "%b %b Restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" - else - # Otherwise, don't to anything - printf " it is not a Pi-hole file, leaving alone!\\n" - fi - else - # If a file cannot be found, - printf " %b No dnsmasq.conf found... restoring default dnsmasq.conf..." "${INFO}" - # restore the default one - install -D -m 644 -T "${dnsmasq_original_config}" "${dnsmasq_conf}" - printf "%b %b No dnsmasq.conf found... restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" - fi - - printf " %b Installing %s..." "${INFO}" "${dnsmasq_pihole_01_target}" - # Check to see if dnsmasq directory exists (it may not due to being a fresh install and dnsmasq no longer being a dependency) - if [[ ! -d "/etc/dnsmasq.d" ]];then - install -d -m 755 "/etc/dnsmasq.d" - fi - # Copy the new Pi-hole DNS config file into the dnsmasq.d directory - install -D -m 644 -T "${dnsmasq_pihole_01_source}" "${dnsmasq_pihole_01_target}" - printf "%b %b Installed %s\n" "${OVER}" "${TICK}" "${dnsmasq_pihole_01_target}" - # Add settings with the GLOBAL DNS variables that we populated earlier - # First, set the interface to listen on - addOrEditKeyValPair "${dnsmasq_pihole_01_target}" "interface" "$PIHOLE_INTERFACE" - if [[ "${PIHOLE_DNS_1}" != "" ]]; then - # then add in the primary DNS server. - addOrEditKeyValPair "${dnsmasq_pihole_01_target}" "server" "$PIHOLE_DNS_1" - fi - # Ditto if DNS2 is not empty - if [[ "${PIHOLE_DNS_2}" != "" ]]; then - addKey "${dnsmasq_pihole_01_target}" "server=$PIHOLE_DNS_2" - fi - - # Set the cache size - addOrEditKeyValPair "${dnsmasq_pihole_01_target}" "cache-size" "$CACHE_SIZE" - - sed -i 's/^#conf-dir=\/etc\/dnsmasq.d$/conf-dir=\/etc\/dnsmasq.d/' "${dnsmasq_conf}" - - # If the user does not want to enable logging, - if [[ "${QUERY_LOGGING}" == false ]] ; then - # remove itfrom the DNS config file - removeKey "${dnsmasq_pihole_01_target}" "log-queries" - else - # Otherwise, enable it by adding the directive to the DNS config file - addKey "${dnsmasq_pihole_01_target}" "log-queries" + # Back it up - we will need to add a symlink to /etc/pihole/dnsmasq.conf later + mv "${dnsmasq_conf}" "${dnsmasq_conf}.old" fi - - printf " %b Installing %s..." "${INFO}" "${dnsmasq_rfc6761_06_source}" - install -D -m 644 -T "${dnsmasq_rfc6761_06_source}" "${dnsmasq_rfc6761_06_target}" - printf "%b %b Installed %s\n" "${OVER}" "${TICK}" "${dnsmasq_rfc6761_06_target}" } # Clean an existing installation to prepare for upgrade/reinstall @@ -1272,7 +1220,7 @@ installScripts() { installConfigs() { printf "\\n %b Installing configs from %s...\\n" "${INFO}" "${PI_HOLE_LOCAL_REPO}" # Make sure Pi-hole's config files are in place - version_check_dnsmasq + remove_old_dnsmasq_ftl_configs # Install list of DNS servers # Format: Name;Primary IPv4;Secondary IPv4;Primary IPv6;Secondary IPv6 @@ -2077,28 +2025,6 @@ FTLinstall() { fi } -disable_dnsmasq() { - # dnsmasq can now be stopped and disabled if it exists - if is_command dnsmasq; then - if check_service_active "dnsmasq";then - printf " %b FTL can now resolve DNS Queries without dnsmasq running separately\\n" "${INFO}" - stop_service dnsmasq - disable_service dnsmasq - fi - fi - - # Backup existing /etc/dnsmasq.conf if present and ensure that - # /etc/dnsmasq.conf contains only "conf-dir=/etc/dnsmasq.d" - local conffile="/etc/dnsmasq.conf" - if [[ -f "${conffile}" ]]; then - printf " %b Backing up %s to %s.old\\n" "${INFO}" "${conffile}" "${conffile}" - mv "${conffile}" "${conffile}.old" - fi - # Create /etc/dnsmasq.conf - echo "conf-dir=/etc/dnsmasq.d" > "${conffile}" - chmod 644 "${conffile}" -} - get_binary_name() { # This gives the machine architecture which may be different from the OS architecture... local machine From 0e558e4c367132f2c525a27ec7bf19c2ffe10e6c Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 13 Jan 2023 21:30:01 +0000 Subject: [PATCH 006/200] Remove some code that installs the blockpage/sets up web directories Remove some code that disables dnsmasq if it is running Create dnsmasq.d directory if it does not exist --- automated install/basic-install.sh | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 7d899940..9d9eb8be 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1162,6 +1162,11 @@ remove_old_dnsmasq_ftl_configs() { # Back it up - we will need to add a symlink to /etc/pihole/dnsmasq.conf later mv "${dnsmasq_conf}" "${dnsmasq_conf}.old" fi + + # Create /etc/dnsmasq.d if it doesn't exist + if [[ ! -d "/etc/dnsmasq.d" ]]; then + mkdir "/etc/dnsmasq.d" + fi } # Clean an existing installation to prepare for upgrade/reinstall @@ -1380,7 +1385,7 @@ check_service_active() { fi } -# Systemd-resolved's DNSStubListener and dnsmasq can't share port 53. +# Systemd-resolved's DNSStubListener and ftl can't share port 53. disable_resolved_stublistener() { printf " %b Testing if systemd-resolved is enabled\\n" "${INFO}" # Check if Systemd-resolved's DNSStubListener is enabled and active on port 53 @@ -1389,7 +1394,7 @@ disable_resolved_stublistener() { printf " %b %b Testing if systemd-resolved DNSStub-Listener is active" "${OVER}" "${INFO}" if ( grep -E '#?DNSStubListener=yes' /etc/systemd/resolved.conf &> /dev/null ); then # Disable the DNSStubListener to unbind it from port 53 - # Note that this breaks dns functionality on host until dnsmasq/ftl are up and running + # Note that this breaks dns functionality on host until ftl are up and running printf "%b %b Disabling systemd-resolved DNSStubListener" "${OVER}" "${TICK}" # Make a backup of the original /etc/systemd/resolved.conf # (This will need to be restored on uninstallation) @@ -1695,21 +1700,13 @@ installPihole() { printf " %b Failure in dependent config copy function.\\n" "${CROSS}" exit 1 fi - # If the user wants to install the dashboard, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - # do so - installPiholeWeb - fi + # Install the cron file installCron # Install the logrotate file installLogrotate || true - # Check if dnsmasq is present. If so, disable it and back up any possible - # config file - disable_dnsmasq - # install a man page entry for pihole install_manpage @@ -2134,13 +2131,6 @@ FTLcheckUpdate() { local remoteSha1 local localSha1 - # if dnsmasq exists and is running at this point, force reinstall of FTL Binary - if is_command dnsmasq; then - if check_service_active "dnsmasq";then - return 0 - fi - fi - if [[ ! "${ftlBranch}" == "master" ]]; then #Check whether or not the binary for this FTL branch actually exists. If not, then there is no update! local path @@ -2397,7 +2387,7 @@ main() { # Check for and disable systemd-resolved-DNSStubListener before reloading resolved # DNSStubListener needs to remain in place for installer to download needed files, # so this change needs to be made after installation is complete, - # but before starting or resarting the dnsmasq or ftl services + # but before starting or resarting the ftl service disable_resolved_stublistener printf " %b Restarting services...\\n" "${INFO}" From e3ffec57622c45ed01e1a9bf8e1ee31159a35871 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 13 Jan 2023 22:23:33 +0000 Subject: [PATCH 007/200] Change when old configs are removed Signed-off-by: Adam Warner --- automated install/basic-install.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9d9eb8be..764503f2 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1224,8 +1224,7 @@ installScripts() { # Install the configs from PI_HOLE_LOCAL_REPO to their various locations installConfigs() { printf "\\n %b Installing configs from %s...\\n" "${INFO}" "${PI_HOLE_LOCAL_REPO}" - # Make sure Pi-hole's config files are in place - remove_old_dnsmasq_ftl_configs + # Install list of DNS servers # Format: Name;Primary IPv4;Secondary IPv4;Primary IPv6;Secondary IPv6 @@ -1695,6 +1694,8 @@ installPihole() { exit 1 fi + remove_old_dnsmasq_ftl_configs + # Install config files if ! installConfigs; then printf " %b Failure in dependent config copy function.\\n" "${CROSS}" From 632aead691f747036c8d78a1096d705cf26820e5 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 14 Jan 2023 23:11:42 +0000 Subject: [PATCH 008/200] add wrappers to utils.sh for setting FTL config & add tests Signed-off-by: Adam Warner --- advanced/Scripts/utils.sh | 23 ++++++++++++++++ test/test_any_utils.py | 55 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index f655e56c..c97ad0cf 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -145,3 +145,26 @@ getFTLPID() { FTL_PID=${FTL_PID:=-1} echo "${FTL_PID}" } + +####################### +# returns value from FTLs config file using pihole-FTL --config +# +# Takes one argument: key +# Example getFTLConfigValue dns.piholePTR +####################### +getFTLConfigValue(){ + pihole-FTL --config "${1}" +} + +####################### +# sets value in FTLs config file using pihole-FTL --config +# +# Takes two arguments: key and value +# Example setFTLConfigValue dns.piholePTR PI.HOLE +# +# Note, for complex values such as dnsmasq.upstreams, you should wrap the value in single quotes: +# setFTLConfigValue dnsmasq.upstreams '[ "8.8.8.8" , "8.8.4.4" ]' +####################### +setFTLConfigValue(){ + pihole-FTL --config "${1}" "${2}" +} diff --git a/test/test_any_utils.py b/test/test_any_utils.py index b3fabe6c..741b1127 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -170,3 +170,58 @@ def test_getFTLPIDFile_and_getFTLPID_custom(host): ) expected_stdout = "1234\n" assert expected_stdout == output.stdout + + +def test_setFTLConfigValue_getFTLConfigValue(host): + """ + Confirms setFTLConfigValue works + Requires FTL to be installed, so we do that first (taken from test_FTL_binary_installed_and_responsive_no_errors) + """ + host.run( + """ + source /opt/pihole/basic-install.sh + create_pihole_user + funcOutput=$(get_binary_name) + echo "new/http" > /etc/pihole/ftlbranch + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" + """ + ) + + output = host.run( + """ + source /opt/pihole/utils.sh + setFTLConfigValue "dnsmasq.upstreams" '["1.1.1.1"]' + """ + ) + + assert '[ "1.1.1.1" ]' in output.stdout + + +def test_getFTLConfigValue_getFTLConfigValue(host): + """ + Confirms getFTLConfigValue works (also assumes setFTLConfigValue works) + Requires FTL to be installed, so we do that first (taken from test_FTL_binary_installed_and_responsive_no_errors) + """ + host.run( + """ + source /opt/pihole/basic-install.sh + create_pihole_user + funcOutput=$(get_binary_name) + echo "new/http" > /etc/pihole/ftlbranch + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" + """ + ) + + output = host.run( + """ + source /opt/pihole/utils.sh + setFTLConfigValue "dnsmasq.upstreams" '["9.9.9.9"]' > /dev/null + getFTLConfigValue "dnsmasq.upstreams" + """ + ) + + assert '[ "9.9.9.9" ]' in output.stdout From 019be067d9ddaf5cf1e1304ddb53c69ce2237a5e Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 14 Jan 2023 23:15:40 +0000 Subject: [PATCH 009/200] Convert SetWebPassword to use new FTL config Signed-off-by: Adam Warner --- advanced/Scripts/webpage.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 7680590f..1282b1ee 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -77,7 +77,7 @@ deleteFTLsetting() { } changeFTLsetting() { - addOrEditKeyValPair "${FTLconf}" "${1}" "${2}" + pihole-FTL --config "${1}" "${2}" } add_dnsmasq_setting() { @@ -165,7 +165,7 @@ SetWebPassword() { echo "" if [ "${PASSWORD}" == "" ]; then - addOrEditKeyValPair "${setupVars}" "WEBPASSWORD" "" + setFTLConfigValue "api.pwhash" "" echo -e " ${TICK} Password Removed" exit 0 fi @@ -178,7 +178,7 @@ SetWebPassword() { # We do not wrap this in brackets, otherwise BASH will expand any appropriate syntax hash=$(HashPassword "$PASSWORD") # Save hash to file - addOrEditKeyValPair "${setupVars}" "WEBPASSWORD" "${hash}" + setFTLConfigValue "api.pwhash" "${hash}" echo -e " ${TICK} New password set" else echo -e " ${CROSS} Passwords don't match. Your password has not been changed" From b8acccde90502731e1a1226a45ff8e4c903d0421 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 15 Jan 2023 11:33:53 +0000 Subject: [PATCH 010/200] Update comments on the move of /etc/dnsmasq.old -> /etc/dnsmasq.old to better reflect why we are doing it Signed-off-by: Adam Warner --- automated install/basic-install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 764503f2..87ff6d83 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1159,7 +1159,8 @@ remove_old_dnsmasq_ftl_configs() { # If the dnsmasq config file exists if [[ -f "${dnsmasq_conf}" ]]; then - # Back it up - we will need to add a symlink to /etc/pihole/dnsmasq.conf later + # There should not be anything custom in here for Pi-hole users + # It is no longer needed, but we'll back it up instead of deleting it just in case mv "${dnsmasq_conf}" "${dnsmasq_conf}.old" fi From 8f7c82840728ac742e21eb0fd260243c6f60416f Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 15 Jan 2023 13:47:38 +0000 Subject: [PATCH 011/200] No need for test_installPiholeWeb_fresh_install_no_errors Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index c1288287..e15839f2 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -118,21 +118,6 @@ def test_selinux_not_detected(host): assert check_selinux.rc == 0 -def test_installPiholeWeb_fresh_install_no_errors(host): - """ - confirms all web page assets from Core repo are installed on a fresh build - """ - installWeb = host.run( - """ - umask 0027 - source /opt/pihole/basic-install.sh - installPiholeWeb - """ - ) - expected_stdout = tick_box + " Installing sudoer file" - assert expected_stdout in installWeb.stdout - - def get_directories_recursive(host, directory): if directory is None: return directory From 5e431210fd5a312939a4e204f0d4e32df199b744 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 15 Jan 2023 13:49:25 +0000 Subject: [PATCH 012/200] [REVISIT] Remove test_setupVars_saved_to_file for now - it may no longer be needed as we move towards getting rid of the file. Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 34 ------------------------------ 1 file changed, 34 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index e15839f2..059e3a8b 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -68,40 +68,6 @@ def test_setupVars_are_sourced_to_global_scope(host): assert "{}={}".format(k, v) in output -def test_setupVars_saved_to_file(host): - """ - confirm saved settings are written to a file for future updates to re-use - """ - # dedent works better with this and padding matching script below - set_setup_vars = "\n" - for k, v in SETUPVARS.items(): - set_setup_vars += " {}={}\n".format(k, v) - host.run(set_setup_vars) - - script = dedent( - """\ - set -e - echo start - TERM=xterm - source /opt/pihole/basic-install.sh - source /opt/pihole/utils.sh - {} - mkdir -p /etc/dnsmasq.d - version_check_dnsmasq - echo "" > /etc/pihole/pihole-FTL.conf - finalExports - cat /etc/pihole/setupVars.conf - """.format( - set_setup_vars - ) - ) - - output = run_script(host, script).stdout - - for k, v in SETUPVARS.items(): - assert "{}={}".format(k, v) in output - - def test_selinux_not_detected(host): """ confirms installer continues when SELinux configuration file does not exist From 414df5b3724d3a5f5f6cb54d140bb737b27034a5 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 15 Jan 2023 13:54:40 +0000 Subject: [PATCH 013/200] Remove no longer required checks in test_installPihole_fresh_install_readableFiles Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 43 ------------------------------ 1 file changed, 43 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 059e3a8b..ef99d57b 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -105,9 +105,6 @@ def test_installPihole_fresh_install_readableFiles(host): mock_command_2( "systemctl", { - "enable lighttpd": ("", "0"), - "restart lighttpd": ("", "0"), - "start lighttpd": ("", "0"), "enable pihole-FTL": ("", "0"), "restart pihole-FTL": ("", "0"), "start pihole-FTL": ("", "0"), @@ -123,7 +120,6 @@ def test_installPihole_fresh_install_readableFiles(host): setup_var_file = "cat < /etc/pihole/setupVars.conf\n" for k, v in SETUPVARS.items(): setup_var_file += "{}={}\n".format(k, v) - setup_var_file += "INSTALL_WEB_SERVER=true\n" setup_var_file += "INSTALL_WEB_INTERFACE=true\n" setup_var_file += "EOF\n" host.run(setup_var_file) @@ -194,23 +190,6 @@ def test_installPihole_fresh_install_readableFiles(host): check_setup = test_cmd.format("r", "/etc/pihole/setupVars.conf", piholeuser) actual_rc = host.run(check_setup).rc assert exit_status_success == actual_rc - # check dnsmasq files - # readable /etc/dnsmasq.conf - check_dnsmasqconf = test_cmd.format("r", "/etc/dnsmasq.conf", piholeuser) - actual_rc = host.run(check_dnsmasqconf).rc - assert exit_status_success == actual_rc - # readable /etc/dnsmasq.d/01-pihole.conf - check_dnsmasqconf = test_cmd.format("r", "/etc/dnsmasq.d", piholeuser) - actual_rc = host.run(check_dnsmasqconf).rc - assert exit_status_success == actual_rc - check_dnsmasqconf = test_cmd.format("x", "/etc/dnsmasq.d", piholeuser) - actual_rc = host.run(check_dnsmasqconf).rc - assert exit_status_success == actual_rc - check_dnsmasqconf = test_cmd.format( - "r", "/etc/dnsmasq.d/01-pihole.conf", piholeuser - ) - actual_rc = host.run(check_dnsmasqconf).rc - assert exit_status_success == actual_rc # check readable and executable /etc/init.d/pihole-FTL check_init = test_cmd.format("x", "/etc/init.d/pihole-FTL", piholeuser) actual_rc = host.run(check_init).rc @@ -218,28 +197,6 @@ def test_installPihole_fresh_install_readableFiles(host): check_init = test_cmd.format("r", "/etc/init.d/pihole-FTL", piholeuser) actual_rc = host.run(check_init).rc assert exit_status_success == actual_rc - # check readable /etc/lighttpd/lighttpd.conf - check_lighttpd = test_cmd.format("r", "/etc/lighttpd/lighttpd.conf", piholeuser) - actual_rc = host.run(check_lighttpd).rc - assert exit_status_success == actual_rc - # check readable /etc/lighttpd/conf*/pihole-admin.conf - check_lighttpd = test_cmd.format("r", "/etc/lighttpd/conf.d", piholeuser) - if host.run(check_lighttpd).rc == exit_status_success: - check_lighttpd = test_cmd.format( - "r", "/etc/lighttpd/conf.d/pihole-admin.conf", piholeuser - ) - actual_rc = host.run(check_lighttpd).rc - assert exit_status_success == actual_rc - else: - check_lighttpd = test_cmd.format( - "r", "/etc/lighttpd/conf-available", piholeuser - ) - if host.run(check_lighttpd).rc == exit_status_success: - check_lighttpd = test_cmd.format( - "r", "/etc/lighttpd/conf-available/15-pihole-admin.conf", piholeuser - ) - actual_rc = host.run(check_lighttpd).rc - assert exit_status_success == actual_rc # check readable and executable manpages if maninstalled is True: check_man = test_cmd.format("x", "/usr/local/share/man", piholeuser) From 9bf372ef43f314cfee4770640725376d40cc6681 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 15 Jan 2023 13:58:05 +0000 Subject: [PATCH 014/200] [REVISIT] Remove test_installPihole_fresh_install_readableBlockpage for now. We may be able to recylcle it later, but I have my doubts Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 225 ----------------------------- 1 file changed, 225 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index ef99d57b..66d00814 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -255,231 +255,6 @@ def test_installPihole_fresh_install_readableFiles(host): actual_rc = host.run(check_pihole).rc -@pytest.mark.parametrize("test_webpage", [True]) -def test_installPihole_fresh_install_readableBlockpage(host, test_webpage): - """ - confirms all web page assets from Core repo are readable - by $LIGHTTPD_USER on a fresh build - """ - piholeWebpage = [ - "127.0.0.1", - # "pi.hole" - ] - # dialog returns Cancel for user prompt - mock_command("dialog", {"*": ("", "0")}, host) - - # mock git pull - mock_command_passthrough("git", {"pull": ("", "0")}, host) - # mock systemctl to start lighttpd and FTL - ligthttpdcommand = dedent( - r'''\"\" - echo 'starting lighttpd with {}' - if [ command -v "apt-get" >/dev/null 2>&1 ]; then - LIGHTTPD_USER="www-data" - LIGHTTPD_GROUP="www-data" - else - LIGHTTPD_USER="lighttpd" - LIGHTTPD_GROUP="lighttpd" - fi - mkdir -p "{run}" - chown {usergroup} "{run}" - mkdir -p "{cache}" - chown {usergroup} "/var/cache" - chown {usergroup} "{cache}" - mkdir -p "{compress}" - chown {usergroup} "{compress}" - mkdir -p "{uploads}" - chown {usergroup} "{uploads}" - chmod 0777 /var - chmod 0777 /var/cache - chmod 0777 "{cache}" - find "{run}" -type d -exec chmod 0777 {chmodarg} \;; - find "{run}" -type f -exec chmod 0666 {chmodarg} \;; - find "{compress}" -type d -exec chmod 0777 {chmodarg} \;; - find "{compress}" -type f -exec chmod 0666 {chmodarg} \;; - find "{uploads}" -type d -exec chmod 0777 {chmodarg} \;; - find "{uploads}" -type f -exec chmod 0666 {chmodarg} \;; - /usr/sbin/lighttpd -tt -f '{config}' - /usr/sbin/lighttpd -f '{config}' - echo \"\"'''.format( - "{}", - usergroup="${{LIGHTTPD_USER}}:${{LIGHTTPD_GROUP}}", - chmodarg="{{}}", - config="/etc/lighttpd/lighttpd.conf", - run="/run/lighttpd", - cache="/var/cache/lighttpd", - uploads="/var/cache/lighttpd/uploads", - compress="/var/cache/lighttpd/compress", - ) - ) - FTLcommand = dedent( - '''\"\" - set -x - /etc/init.d/pihole-FTL restart - echo \"\"''' - ) - mock_command_run( - "systemctl", - { - "enable lighttpd": ("", "0"), - "restart lighttpd": (ligthttpdcommand.format("restart"), "0"), - "start lighttpd": (ligthttpdcommand.format("start"), "0"), - "enable pihole-FTL": ("", "0"), - "restart pihole-FTL": (FTLcommand, "0"), - "start pihole-FTL": (FTLcommand, "0"), - "*": ('echo "systemctl call with $@"', "0"), - }, - host, - ) - # create configuration file - setup_var_file = "cat < /etc/pihole/setupVars.conf\n" - for k, v in SETUPVARS.items(): - setup_var_file += "{}={}\n".format(k, v) - setup_var_file += "INSTALL_WEB_SERVER=true\n" - setup_var_file += "INSTALL_WEB_INTERFACE=true\n" - setup_var_file += "EOF\n" - host.run(setup_var_file) - installWeb = host.run( - """ - export TERM=xterm - export DEBIAN_FRONTEND=noninteractive - umask 0027 - runUnattended=true - useUpdateVars=true - source /opt/pihole/basic-install.sh > /dev/null - runUnattended=true - useUpdateVars=true - main - echo "LIGHTTPD_USER=${LIGHTTPD_USER}" - echo "webroot=${webroot}" - echo "INSTALL_WEB_INTERFACE=${INSTALL_WEB_INTERFACE}" - echo "INSTALL_WEB_SERVER=${INSTALL_WEB_SERVER}" - """ - ) - assert 0 == installWeb.rc - piholeuser = "pihole" - webuser = "" - user = re.findall(r"^\s*LIGHTTPD_USER=.*$", installWeb.stdout, re.MULTILINE) - for match in user: - webuser = match.replace("LIGHTTPD_USER=", "").strip() - webroot = "" - user = re.findall(r"^\s*webroot=.*$", installWeb.stdout, re.MULTILINE) - for match in user: - webroot = match.replace("webroot=", "").strip() - if not webroot.strip(): - webroot = "/var/www/html" - installWebInterface = True - interface = re.findall( - r"^\s*INSTALL_WEB_INTERFACE=.*$", installWeb.stdout, re.MULTILINE - ) - for match in interface: - testvalue = match.replace("INSTALL_WEB_INTERFACE=", "").strip().lower() - if not testvalue.strip(): - installWebInterface = testvalue == "true" - installWebServer = True - server = re.findall(r"^\s*INSTALL_WEB_SERVER=.*$", installWeb.stdout, re.MULTILINE) - for match in server: - testvalue = match.replace("INSTALL_WEB_SERVER=", "").strip().lower() - if not testvalue.strip(): - installWebServer = testvalue == "true" - # if webserver install was not requested - # at least pihole must be able to read files - if installWebServer is False: - webuser = piholeuser - exit_status_success = 0 - test_cmd = 'su --shell /bin/bash --command "test -{0} {1}" -p {2}' - # check files that need a running FTL to be created - # readable and writeable pihole-FTL.db - check_FTLconf = test_cmd.format("r", "/etc/pihole/pihole-FTL.db", piholeuser) - actual_rc = host.run(check_FTLconf).rc - assert exit_status_success == actual_rc - check_FTLconf = test_cmd.format("w", "/etc/pihole/pihole-FTL.db", piholeuser) - actual_rc = host.run(check_FTLconf).rc - assert exit_status_success == actual_rc - # check directories above $webroot for read and execute permission - check_var = test_cmd.format("r", "/var", webuser) - actual_rc = host.run(check_var).rc - assert exit_status_success == actual_rc - check_var = test_cmd.format("x", "/var", webuser) - actual_rc = host.run(check_var).rc - assert exit_status_success == actual_rc - check_www = test_cmd.format("r", "/var/www", webuser) - actual_rc = host.run(check_www).rc - assert exit_status_success == actual_rc - check_www = test_cmd.format("x", "/var/www", webuser) - actual_rc = host.run(check_www).rc - assert exit_status_success == actual_rc - check_html = test_cmd.format("r", "/var/www/html", webuser) - actual_rc = host.run(check_html).rc - assert exit_status_success == actual_rc - check_html = test_cmd.format("x", "/var/www/html", webuser) - actual_rc = host.run(check_html).rc - assert exit_status_success == actual_rc - # check directories below $webroot for read and execute permission - check_admin = test_cmd.format("r", webroot + "/admin", webuser) - actual_rc = host.run(check_admin).rc - assert exit_status_success == actual_rc - check_admin = test_cmd.format("x", webroot + "/admin", webuser) - actual_rc = host.run(check_admin).rc - assert exit_status_success == actual_rc - directories = get_directories_recursive(host, webroot + "/admin/") - for directory in directories: - check_pihole = test_cmd.format("r", directory, webuser) - actual_rc = host.run(check_pihole).rc - check_pihole = test_cmd.format("x", directory, webuser) - actual_rc = host.run(check_pihole).rc - findfiles = 'find "{}" -maxdepth 1 -type f -exec echo {{}} \\;;' - filelist = host.run(findfiles.format(directory)) - files = list(filter(bool, filelist.stdout.splitlines())) - for file in files: - check_pihole = test_cmd.format("r", file, webuser) - actual_rc = host.run(check_pihole).rc - # check web interface files - # change nameserver to pi-hole - # setting nameserver in /etc/resolv.conf to pi-hole does - # not work here because of the way docker uses this file - ns = host.run(r"sed -i 's/nameserver.*/nameserver 127.0.0.1/' /etc/resolv.conf") - pihole_is_ns = ns.rc == 0 - - def is_ip(address): - m = re.match(r"(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})", address) - return bool(m) - - if installWebInterface is True: - if test_webpage is True: - # check webpage for unreadable files - noPHPfopen = re.compile( - ( - r"PHP Error(%d+):\s+fopen([^)]+):\s+" - + r"failed to open stream: " - + r"Permission denied in" - ), - re.I, - ) - # using cURL option --dns-servers is not possible - status = ( - 'curl -s --head "{}" | ' - + "head -n 1 | " - + 'grep "HTTP/1.[01] [23].." > /dev/null' - ) - digcommand = r"dig A +short {} @127.0.0.1 | head -n 1" - pagecontent = 'curl --verbose -L "{}"' - for page in piholeWebpage: - testpage = "http://" + page + "/admin/" - resolvesuccess = True - if is_ip(page) is False: - dig = host.run(digcommand.format(page)) - testpage = "http://" + dig.stdout.strip() + "/admin/" - resolvesuccess = dig.rc == 0 - if resolvesuccess or pihole_is_ns: - # check HTTP status of blockpage - actual_rc = host.run(status.format(testpage)) - assert exit_status_success == actual_rc.rc - # check for PHP error - actual_output = host.run(pagecontent.format(testpage)) - assert noPHPfopen.match(actual_output.stdout) is None - - def test_update_package_cache_success_no_errors(host): """ confirms package cache was updated without any errors From bf16fe4a376c30a84f842ec569327ef5dc28b41d Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 15 Jan 2023 15:47:10 +0000 Subject: [PATCH 015/200] Remove the EPEL related tests on centos/fedora - no longer neccasery Signed-off-by: Adam Warner --- test/test_centos_common_support.py | 27 --------------------------- test/test_fedora_support.py | 15 --------------- test/tox.centos_8.ini | 2 +- test/tox.centos_9.ini | 2 +- test/tox.fedora_36.ini | 2 +- test/tox.fedora_37.ini | 2 +- 6 files changed, 4 insertions(+), 46 deletions(-) delete mode 100644 test/test_centos_common_support.py delete mode 100644 test/test_fedora_support.py diff --git a/test/test_centos_common_support.py b/test/test_centos_common_support.py deleted file mode 100644 index 871fee29..00000000 --- a/test/test_centos_common_support.py +++ /dev/null @@ -1,27 +0,0 @@ -import pytest -from .conftest import ( - tick_box, - info_box, - cross_box, - mock_command, -) - - -def test_enable_epel_repository_centos(host): - """ - confirms the EPEL package repository is enabled when installed on CentOS - """ - package_manager_detect = host.run( - """ - source /opt/pihole/basic-install.sh - package_manager_detect - """ - ) - expected_stdout = info_box + ( - " Enabling EPEL package repository " "(https://fedoraproject.org/wiki/EPEL)" - ) - assert expected_stdout in package_manager_detect.stdout - expected_stdout = tick_box + " Installed" - assert expected_stdout in package_manager_detect.stdout - epel_package = host.package("epel-release") - assert epel_package.is_installed diff --git a/test/test_fedora_support.py b/test/test_fedora_support.py deleted file mode 100644 index e7d31a5d..00000000 --- a/test/test_fedora_support.py +++ /dev/null @@ -1,15 +0,0 @@ -def test_epel_and_remi_not_installed_fedora(host): - """ - confirms installer does not attempt to install EPEL/REMI repositories - on Fedora - """ - package_manager_detect = host.run( - """ - source /opt/pihole/basic-install.sh - package_manager_detect - """ - ) - assert package_manager_detect.stdout == "" - - epel_package = host.package("epel-release") - assert not epel_package.is_installed diff --git a/test/tox.centos_8.ini b/test/tox.centos_8.ini index dca77c93..85ae1ffb 100644 --- a/test/tox.centos_8.ini +++ b/test/tox.centos_8.ini @@ -5,4 +5,4 @@ envlist = py3 allowlist_externals = docker deps = -rrequirements.txt commands = docker buildx build --load --progress plain -f _centos_8.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py diff --git a/test/tox.centos_9.ini b/test/tox.centos_9.ini index a69c336a..4c51aefa 100644 --- a/test/tox.centos_9.ini +++ b/test/tox.centos_9.ini @@ -5,4 +5,4 @@ envlist = py3 allowlist_externals = docker deps = -rrequirements.txt commands = docker buildx build --load --progress plain -f _centos_9.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py diff --git a/test/tox.fedora_36.ini b/test/tox.fedora_36.ini index 515487ed..1896a45f 100644 --- a/test/tox.fedora_36.ini +++ b/test/tox.fedora_36.ini @@ -5,4 +5,4 @@ envlist = py3 allowlist_externals = docker deps = -rrequirements.txt commands = docker buildx build --load --progress plain -f _fedora_36.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py diff --git a/test/tox.fedora_37.ini b/test/tox.fedora_37.ini index 2a8ef398..9c8752cc 100644 --- a/test/tox.fedora_37.ini +++ b/test/tox.fedora_37.ini @@ -5,4 +5,4 @@ envlist = py3 allowlist_externals = docker deps = -rrequirements.txt commands = docker buildx build --load --progress plain -f _fedora_37.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py From 36956103004a99f59aa2258ff0880745d535f354 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 24 Jan 2023 19:51:01 +0100 Subject: [PATCH 016/200] Allow running pihole -g without root (run as user pihole) Signed-off-by: DL6ER --- gravity.sh | 2 +- pihole | 20 ++++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index fe2097dd..5b950b28 100755 --- a/gravity.sh +++ b/gravity.sh @@ -912,7 +912,7 @@ gravity_ShowCount # Determine if DNS has been restarted by this instance of gravity if [[ -z "${dnsWasOffline:-}" ]]; then - "${PIHOLE_COMMAND}" restartdns reload + "${PIHOLE_COMMAND}" reloaddns fi gravity_Cleanup diff --git a/pihole b/pihole index 1d9ad82c..6796acc6 100755 --- a/pihole +++ b/pihole @@ -513,13 +513,13 @@ if [[ $# = 0 ]]; then fi # functions that do not require sudo power +need_root=1 case "${1}" in "-h" | "help" | "--help" ) helpFunc;; "-v" | "version" ) versionFunc "$@";; "-c" | "chronometer" ) chronometerFunc "$@";; "-q" | "query" ) queryFunc "$@";; "status" ) statusFunc "$2";; - "tricorder" ) tricorderFunc;; # we need to add all arguments that require sudo power to not trigger the * argument @@ -532,13 +532,14 @@ case "${1}" in "-f" | "flush" ) ;; "-up" | "updatePihole" ) ;; "-r" | "reconfigure" ) ;; - "-g" | "updateGravity" ) ;; "-l" | "logging" ) ;; "uninstall" ) ;; "enable" ) ;; "disable" ) ;; "-d" | "debug" ) ;; "restartdns" ) ;; + "-g" | "updateGravity" ) need_root=0;; + "reloaddns" ) need_root=0;; "-a" | "admin" ) ;; "checkout" ) ;; "updatechecker" ) ;; @@ -547,8 +548,8 @@ case "${1}" in * ) helpFunc;; esac -# Must be root to use this tool -if [[ ! $EUID -eq 0 ]];then +# Must be root to use this tool for most functions +if [[ ! $EUID -eq 0 && need_root -eq 1 ]];then if [[ -x "$(command -v sudo)" ]]; then exec sudo bash "$0" "$@" exit $? @@ -557,6 +558,16 @@ if [[ ! $EUID -eq 0 ]];then exit 1 fi fi +# Can also be user pihole for other functions +if [[ ${USER} != "pihole" && need_root -eq 0 ]];then + if [[ -x "$(command -v sudo)" ]]; then + exec sudo -u pihole bash "$0" "$@" + exit $? + else + echo -e " ${CROSS} sudo is needed to run pihole commands. Please run this script as root or install sudo." + exit 1 + fi +fi # Handle redirecting to specific functions based on arguments case "${1}" in @@ -576,6 +587,7 @@ case "${1}" in "enable" ) piholeEnable 1;; "disable" ) piholeEnable 0 "$2";; "restartdns" ) restartDNS "$2";; + "reloaddns" ) restartDNS "reload";; "-a" | "admin" ) webpageFunc "$@";; "checkout" ) piholeCheckoutFunc "$@";; "updatechecker" ) shift; updateCheckFunc "$@";; From 3cb3adc5ca5796c6b4f79b943eb5563c1c0972a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 3 Feb 2023 19:55:55 +0100 Subject: [PATCH 017/200] Fix setting webpassword via pihole -a -p MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 1282b1ee..53f36a20 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -165,7 +165,7 @@ SetWebPassword() { echo "" if [ "${PASSWORD}" == "" ]; then - setFTLConfigValue "api.pwhash" "" + setFTLConfigValue "webserver.api.pwhash" "" >/dev/null echo -e " ${TICK} Password Removed" exit 0 fi @@ -178,7 +178,7 @@ SetWebPassword() { # We do not wrap this in brackets, otherwise BASH will expand any appropriate syntax hash=$(HashPassword "$PASSWORD") # Save hash to file - setFTLConfigValue "api.pwhash" "${hash}" + setFTLConfigValue "webserver.api.pwhash" "${hash}" >/dev/null echo -e " ${TICK} New password set" else echo -e " ${CROSS} Passwords don't match. Your password has not been changed" From 0568a69d8346d31372a1f35acc9f246369064001 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 13:15:47 +0000 Subject: [PATCH 018/200] Use WEBPORT instead of PRIVACY_LEVEL to set the web port... Signed-off-by: Adam Warner --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 87ff6d83..bce7aa91 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1625,7 +1625,7 @@ finalExports() { addOrEditKeyValPair "${FTL_CONFIG_FILE}" "PRIVACYLEVEL" "${PRIVACY_LEVEL}" # Set the web port - addOrEditKeyValPair "${FTL_CONFIG_FILE}" "WEBPORT" "${PRIVACY_LEVEL}" + addOrEditKeyValPair "${FTL_CONFIG_FILE}" "WEBPORT" "${WEBPORT}" # Bring in the current settings and the functions to manipulate them source "${setupVars}" From dbc6b814b2381e9db3d0361dff7cc7b80ef0236f Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 15:15:55 +0000 Subject: [PATCH 019/200] Should not need to reloadDNS manually any more as FTL will detect that it needs to reload by itself Signed-off-by: Adam Warner --- gravity.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index 5b950b28..e914732a 100755 --- a/gravity.sh +++ b/gravity.sh @@ -910,11 +910,6 @@ chmod g+w "${piholeDir}" "${gravityDBfile}" # Compute numbers to be displayed gravity_ShowCount -# Determine if DNS has been restarted by this instance of gravity -if [[ -z "${dnsWasOffline:-}" ]]; then - "${PIHOLE_COMMAND}" reloaddns -fi - gravity_Cleanup echo "" From 2784b267ec6424ae1235bf4d0a52d07fe7ab8c11 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 16:40:38 +0000 Subject: [PATCH 020/200] pihole command: read values from pihole-FTL instead of setupvars.conf Signed-off-by: Adam Warner --- advanced/Scripts/utils.sh | 4 ++-- pihole | 33 ++++++++++----------------------- 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index c97ad0cf..8bab396a 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -153,7 +153,7 @@ getFTLPID() { # Example getFTLConfigValue dns.piholePTR ####################### getFTLConfigValue(){ - pihole-FTL --config "${1}" + pihole-FTL --config -q "${1}" } ####################### @@ -166,5 +166,5 @@ getFTLConfigValue(){ # setFTLConfigValue dnsmasq.upstreams '[ "8.8.8.8" , "8.8.4.4" ]' ####################### setFTLConfigValue(){ - pihole-FTL --config "${1}" "${2}" + pihole-FTL --config "${1}" "${2}" >/dev/null } diff --git a/pihole b/pihole index 6796acc6..0c91df35 100755 --- a/pihole +++ b/pihole @@ -11,10 +11,9 @@ readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" -# setupVars and PI_HOLE_BIN_DIR are not readonly here because in some functions (checkout), +# PI_HOLE_BIN_DIR is not readonly here because in some functions (checkout), # they might get set again when the installer is sourced. This causes an # error due to modifying a readonly variable. -setupVars="/etc/pihole/setupVars.conf" PI_HOLE_BIN_DIR="/usr/local/bin" readonly colfile="${PI_HOLE_SCRIPT_DIR}/COL_TABLE" @@ -193,7 +192,7 @@ Time: elif [[ "${1}" == "0" ]]; then # Disable Pi-hole - if grep -cq "BLOCKING_ENABLED=false" "${setupVars}"; then + if ! getFTLConfigValue dns.blocking.active; then echo -e " ${INFO} Blocking already disabled, nothing to do" exit 0 fi @@ -233,19 +232,19 @@ Time: fi local str="Pi-hole Disabled" - addOrEditKeyValPair "${setupVars}" "BLOCKING_ENABLED" "false" + setFTLConfigValue dns.blocking.active false fi else # Enable Pi-hole killall -q pihole-reenable - if grep -cq "BLOCKING_ENABLED=true" "${setupVars}"; then + if getFTLConfigValue dns.blocking.active; then echo -e " ${INFO} Blocking already enabled, nothing to do" exit 0 fi echo -e " ${INFO} Enabling blocking" local str="Pi-hole Enabled" - addOrEditKeyValPair "${setupVars}" "BLOCKING_ENABLED" "true" + setFTLConfigValue dns.blocking.active true fi restartDNS reload-lists @@ -267,8 +266,7 @@ Options: exit 0 elif [[ "${1}" == "off" ]]; then # Disable logging - removeKey /etc/dnsmasq.d/01-pihole.conf "log-queries" - addOrEditKeyValPair "${setupVars}" "QUERY_LOGGING" "false" + setFTLConfigValue dns.queryLogging false if [[ "${2}" != "noflush" ]]; then # Flush logs "${PI_HOLE_BIN_DIR}"/pihole -f @@ -277,8 +275,7 @@ Options: local str="Logging has been disabled!" elif [[ "${1}" == "on" ]]; then # Enable logging - addKey /etc/dnsmasq.d/01-pihole.conf "log-queries" - addOrEditKeyValPair "${setupVars}" "QUERY_LOGGING" "true" + setFTLConfigValue dns.queryLogging true echo -e " ${INFO} Enabling logging..." local str="Logging has been enabled!" else @@ -354,26 +351,16 @@ statusFunc() { fi # Determine if Pi-hole's blocking is enabled - if grep -q "BLOCKING_ENABLED=false" /etc/pihole/setupVars.conf; then - # A config is commented out - case "${1}" in - "web") echo 0;; - *) echo -e " ${CROSS} Pi-hole blocking is disabled";; - esac - elif grep -q "BLOCKING_ENABLED=true" /etc/pihole/setupVars.conf; then - # Configs are set + if getFTLConfigValue dns.blocking.active; then case "${1}" in "web") echo "$port";; *) echo -e " ${TICK} Pi-hole blocking is enabled";; esac else - # No configs were found case "${1}" in - "web") echo -2;; - *) echo -e " ${INFO} Pi-hole blocking will be enabled";; + "web") echo 0;; + *) echo -e " ${CROSS} Pi-hole blocking is disabled";; esac - # Enable blocking - "${PI_HOLE_BIN_DIR}"/pihole enable fi exit 0 } From cd17040f959551fbc3250a4565e5fe547f595073 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 17:20:52 +0000 Subject: [PATCH 021/200] setupVars.conf not existing is no longer a critical error Signed-off-by: Adam Warner --- gravity.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index e914732a..515927e8 100755 --- a/gravity.sh +++ b/gravity.sh @@ -46,10 +46,6 @@ curl_connect_timeout=10 setupVars="${piholeDir}/setupVars.conf" if [[ -f "${setupVars}" ]];then source "${setupVars}" -else - echo -e " ${COL_LIGHT_RED}Installation Failure: ${setupVars} does not exist! ${COL_NC} - Please run 'pihole -r', and choose the 'reconfigure' option to fix." - exit 1 fi # Set up tmp dir variable in case it's not configured From 0e8f285f4f670d07be8c8b40d15ebdad3c4afa75 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 18:39:48 +0000 Subject: [PATCH 022/200] Always assume that the web interface will be installed. We could revist this again in future if we decide that there should still be a choice to install the interface or not Signed-off-by: Adam Warner --- advanced/Scripts/piholeCheckout.sh | 27 +++---- advanced/Scripts/piholeDebug.sh | 15 +--- advanced/Scripts/update.sh | 26 +++---- advanced/Scripts/updatecheck.sh | 28 +++---- advanced/Scripts/version.sh | 24 ++---- automated install/basic-install.sh | 115 ++++++++--------------------- automated install/uninstall.sh | 4 - test/test_any_automated_install.py | 1 - 8 files changed, 72 insertions(+), 168 deletions(-) diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index cf57800c..5a2eebd7 100755 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -61,12 +61,11 @@ checkout() { echo -e " Please re-run install script from https://github.com/pi-hole/pi-hole${COL_NC}" exit 1; fi - if [[ "${INSTALL_WEB_INTERFACE}" == "true" ]]; then - if ! is_repo "${webInterfaceDir}" ; then - echo -e " ${COL_LIGHT_RED}Error: Web Admin repo is missing from system!" - echo -e " Please re-run install script from https://github.com/pi-hole/pi-hole${COL_NC}" - exit 1; - fi + + if ! is_repo "${webInterfaceDir}" ; then + echo -e " ${COL_LIGHT_RED}Error: Web Admin repo is missing from system!" + echo -e " Please re-run install script from https://github.com/pi-hole/pi-hole${COL_NC}" + exit 1; fi if [[ -z "${1}" ]]; then @@ -85,11 +84,9 @@ checkout() { echo "" echo -e " ${INFO} Pi-hole Core" fetch_checkout_pull_branch "${PI_HOLE_FILES_DIR}" "development" || { echo " ${CROSS} Unable to pull Core development branch"; exit 1; } - if [[ "${INSTALL_WEB_INTERFACE}" == "true" ]]; then - echo "" - echo -e " ${INFO} Web interface" - fetch_checkout_pull_branch "${webInterfaceDir}" "devel" || { echo " ${CROSS} Unable to pull Web development branch"; exit 1; } - fi + echo "" + echo -e " ${INFO} Web interface" + fetch_checkout_pull_branch "${webInterfaceDir}" "devel" || { echo " ${CROSS} Unable to pull Web development branch"; exit 1; } #echo -e " ${TICK} Pi-hole Core" local path @@ -101,10 +98,8 @@ checkout() { echo -e " ${INFO} Shortcut \"master\" detected - checking out master branches..." echo -e " ${INFO} Pi-hole core" fetch_checkout_pull_branch "${PI_HOLE_FILES_DIR}" "master" || { echo " ${CROSS} Unable to pull Core master branch"; exit 1; } - if [[ ${INSTALL_WEB_INTERFACE} == "true" ]]; then - echo -e " ${INFO} Web interface" - fetch_checkout_pull_branch "${webInterfaceDir}" "master" || { echo " ${CROSS} Unable to pull Web master branch"; exit 1; } - fi + echo -e " ${INFO} Web interface" + fetch_checkout_pull_branch "${webInterfaceDir}" "master" || { echo " ${CROSS} Unable to pull Web master branch"; exit 1; } #echo -e " ${TICK} Web Interface" local path path="master/${binary}" @@ -137,7 +132,7 @@ checkout() { exit 1 fi checkout_pull_branch "${PI_HOLE_FILES_DIR}" "${2}" - elif [[ "${1}" == "web" ]] && [[ "${INSTALL_WEB_INTERFACE}" == "true" ]] ; then + elif [[ "${1}" == "web" ]] ; then str="Fetching branches from ${webInterfaceGitUrl}" echo -ne " ${INFO} $str" if ! fully_fetch_repo "${webInterfaceDir}" ; then diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 9f895aab..e7c383b5 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -291,17 +291,10 @@ compare_local_version_to_git_version() { return 1 fi else - # There is no git directory so check if the web interface was disabled - local setup_vars_web_interface - setup_vars_web_interface=$(< ${PIHOLE_SETUP_VARS_FILE} grep ^INSTALL_WEB_INTERFACE | cut -d '=' -f2) - if [[ "${pihole_component}" == "Web" ]] && [[ "${setup_vars_web_interface}" == "false" ]]; then - log_write "${INFO} ${pihole_component}: Disabled in setupVars.conf via INSTALL_WEB_INTERFACE=false" - else - # Return an error message - log_write "${COL_RED}Directory ${git_dir} doesn't exist${COL_NC}" - # and exit with a non zero code - return 1 - fi + # Return an error message + log_write "${COL_RED}Directory ${git_dir} doesn't exist${COL_NC}" + # and exit with a non zero code + return 1 fi } diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index c41c9232..6185d94a 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -128,20 +128,18 @@ main() { echo -e " ${INFO} Pi-hole Core:\\t${COL_LIGHT_GREEN}up to date${COL_NC}" fi - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - if ! is_repo "${ADMIN_INTERFACE_DIR}" ; then - echo -e "\\n ${COL_LIGHT_RED}Error: Web Admin repo is missing from system!" - echo -e " Please re-run install script from https://pi-hole.net${COL_NC}" - exit 1; - fi - - if GitCheckUpdateAvail "${ADMIN_INTERFACE_DIR}" ; then - web_update=true - echo -e " ${INFO} Web Interface:\\t${COL_YELLOW}update available${COL_NC}" - else - web_update=false - echo -e " ${INFO} Web Interface:\\t${COL_LIGHT_GREEN}up to date${COL_NC}" - fi + if ! is_repo "${ADMIN_INTERFACE_DIR}" ; then + echo -e "\\n ${COL_LIGHT_RED}Error: Web Admin repo is missing from system!" + echo -e " Please re-run install script from https://pi-hole.net${COL_NC}" + exit 1; + fi + + if GitCheckUpdateAvail "${ADMIN_INTERFACE_DIR}" ; then + web_update=true + echo -e " ${INFO} Web Interface:\\t${COL_YELLOW}update available${COL_NC}" + else + web_update=false + echo -e " ${INFO} Web Interface:\\t${COL_LIGHT_GREEN}up to date${COL_NC}" fi local funcOutput diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 0cc65218..7d7103d2 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -34,10 +34,6 @@ function get_remote_hash(){ git ls-remote "https://github.com/pi-hole/${1}" --tags "${2}" | awk '{print substr($0, 0,8);}' || return 1 } -# Source the setupvars config file -# shellcheck disable=SC1091 -. /etc/pihole/setupVars.conf - # Source the utils file for addOrEditKeyValPair() # shellcheck disable=SC1091 . /opt/pihole/utils.sh @@ -86,24 +82,20 @@ addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_CORE_HASH" "${GITHUB_CORE_HASH}" # get Web versions -if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - - WEB_VERSION="$(get_local_version /var/www/html/admin)" - addOrEditKeyValPair "${VERSION_FILE}" "WEB_VERSION" "${WEB_VERSION}" +WEB_VERSION="$(get_local_version /var/www/html/admin)" +addOrEditKeyValPair "${VERSION_FILE}" "WEB_VERSION" "${WEB_VERSION}" - WEB_BRANCH="$(get_local_branch /var/www/html/admin)" - addOrEditKeyValPair "${VERSION_FILE}" "WEB_BRANCH" "${WEB_BRANCH}" +WEB_BRANCH="$(get_local_branch /var/www/html/admin)" +addOrEditKeyValPair "${VERSION_FILE}" "WEB_BRANCH" "${WEB_BRANCH}" - WEB_HASH="$(get_local_hash /var/www/html/admin)" - addOrEditKeyValPair "${VERSION_FILE}" "WEB_HASH" "${WEB_HASH}" +WEB_HASH="$(get_local_hash /var/www/html/admin)" +addOrEditKeyValPair "${VERSION_FILE}" "WEB_HASH" "${WEB_HASH}" - GITHUB_WEB_VERSION="$(get_remote_version AdminLTE)" - addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_VERSION" "${GITHUB_WEB_VERSION}" +GITHUB_WEB_VERSION="$(get_remote_version AdminLTE)" +addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_VERSION" "${GITHUB_WEB_VERSION}" - GITHUB_WEB_HASH="$(get_remote_hash AdminLTE "${WEB_BRANCH}")" - addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_HASH" "${GITHUB_WEB_HASH}" - -fi +GITHUB_WEB_HASH="$(get_remote_hash AdminLTE "${WEB_BRANCH}")" +addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_HASH" "${GITHUB_WEB_HASH}" # get FTL versions diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index 946c69fe..af86b045 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -8,10 +8,6 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. -# Source the setupvars config file -# shellcheck disable=SC1091 -. /etc/pihole/setupVars.conf - # Source the versions file poupulated by updatechecker.sh cachedVersions="/etc/pihole/versions" @@ -28,7 +24,7 @@ fi getLocalVersion() { case ${1} in "Pi-hole" ) echo "${CORE_VERSION:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_VERSION:=N/A}";; + "AdminLTE" ) echo "${WEB_VERSION:=N/A}";; "FTL" ) echo "${FTL_VERSION:=N/A}";; esac } @@ -36,7 +32,7 @@ getLocalVersion() { getLocalHash() { case ${1} in "Pi-hole" ) echo "${CORE_HASH:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_HASH:=N/A}";; + "AdminLTE" ) echo "${WEB_HASH:=N/A}";; "FTL" ) echo "${FTL_HASH:=N/A}";; esac } @@ -44,7 +40,7 @@ getLocalHash() { getRemoteHash(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_HASH:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${GITHUB_WEB_HASH:=N/A}";; + "AdminLTE" ) echo "${GITHUB_WEB_HASH:=N/A}";; "FTL" ) echo "${GITHUB_FTL_HASH:=N/A}";; esac } @@ -52,7 +48,7 @@ getRemoteHash(){ getRemoteVersion(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_VERSION:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${GITHUB_WEB_VERSION:=N/A}";; + "AdminLTE" ) echo "${GITHUB_WEB_VERSION:=N/A}";; "FTL" ) echo "${GITHUB_FTL_VERSION:=N/A}";; esac } @@ -60,16 +56,12 @@ getRemoteVersion(){ getLocalBranch(){ case ${1} in "Pi-hole" ) echo "${CORE_BRANCH:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_BRANCH:=N/A}";; + "AdminLTE" ) echo "${WEB_BRANCH:=N/A}";; "FTL" ) echo "${FTL_BRANCH:=N/A}";; esac } versionOutput() { - if [ "$1" = "AdminLTE" ] && [ "${INSTALL_WEB_INTERFACE}" != true ]; then - echo " WebAdmin not installed" - return 1 - fi [ "$2" = "-c" ] || [ "$2" = "--current" ] || [ -z "$2" ] && current=$(getLocalVersion "${1}") && branch=$(getLocalBranch "${1}") [ "$2" = "-l" ] || [ "$2" = "--latest" ] || [ -z "$2" ] && latest=$(getRemoteVersion "${1}") @@ -115,11 +107,7 @@ errorOutput() { defaultOutput() { versionOutput "Pi-hole" "$@" - - if [ "${INSTALL_WEB_INTERFACE}" = true ]; then - versionOutput "AdminLTE" "$@" - fi - + versionOutput "AdminLTE" "$@" versionOutput "FTL" "$@" } diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index bce7aa91..9e5a9963 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -91,7 +91,6 @@ IPV4_ADDRESS=${IPV4_ADDRESS} IPV6_ADDRESS=${IPV6_ADDRESS} # Give settings their default values. These may be changed by prompts later in the script. QUERY_LOGGING=true -INSTALL_WEB_INTERFACE=true WEBPORT=8080 PRIVACY_LEVEL=0 CACHE_SIZE=10000 @@ -1048,44 +1047,6 @@ setPrivacyLevel() { esac } -# Function to ask the user if they want to install the dashboard -setAdminFlag() { - # Similar to the logging function, ask what the user wants - dialog --no-shadow --keep-tite \ - --backtitle "Pihole Installation" \ - --title "Admin Web Interface" \ - --yesno "\\n\\nDo you want to install the Admin Web Interface?" \ - "${r}" "${c}" && result=0 || result=$? - - case ${result} in - "${DIALOG_OK}") - # If they chose yes, - printf " %b Installing Admin Web Interface\\n" "${INFO}" - # Set the flag to install the web interface - INSTALL_WEB_INTERFACE=true - - # Web port TODO: Below whiptail copy pasted from a previous go at this. needs converting to dialog - # Ask for the IPv4 address - WEBPORT=$(whiptail --backtitle "Setting web interface port" --title "Web Port" --inputbox "By default, pihole-FTL listens for http traffic on port 8080. If you wish to change the port, you may do so now. You can also do it later by editing /etc/pihole/pihole-FTL.conf" "${r}" "${c}" "${WEBPORT}" 3>&1 1>&2 2>&3) || \ - # Canceling IPv4 settings window - { echo -e " ${COL_LIGHT_RED}Cancel was selected, exiting installer${COL_NC}"; exit 1; } - printf " %b The Web interface will be accessible on port: %s\\n" "${INFO}" "${WEBPORT}" - - ;; - "${DIALOG_CANCEL}") - # If they chose no, - printf " %b Not installing Admin Web Interface\\n" "${INFO}" - # Set the flag to not install the web interface - INSTALL_WEB_INTERFACE=false - ;; - "${DIALOG_ESC}") - # User pressed - printf " %b Escape pressed, exiting installer at Admin Web Interface choice.%b\\n" "${COL_LIGHT_RED}" "${COL_NC}" - exit 1 - ;; - esac -} - # A function to display a list of example blocklists for users to select chooseBlocklists() { # Back up any existing adlist file, on the off chance that it exists. Useful in case of a reconfigure. @@ -1613,7 +1574,6 @@ finalExports() { addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_1" "${PIHOLE_DNS_1}" addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_2" "${PIHOLE_DNS_2}" addOrEditKeyValPair "${setupVars}" "QUERY_LOGGING" "${QUERY_LOGGING}" - addOrEditKeyValPair "${setupVars}" "INSTALL_WEB_INTERFACE" "${INSTALL_WEB_INTERFACE}" addOrEditKeyValPair "${setupVars}" "CACHE_SIZE" "${CACHE_SIZE}" addOrEditKeyValPair "${setupVars}" "DNS_FQDN_REQUIRED" "${DNS_FQDN_REQUIRED:-true}" addOrEditKeyValPair "${setupVars}" "DNS_BOGUS_PRIV" "${DNS_BOGUS_PRIV:-true}" @@ -1775,11 +1735,10 @@ displayFinalMessage() { # Else, inform the user that there is no set password. pwstring="NOT SET" fi - # If the user wants to install the dashboard, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - # Store a message in a variable and display it - additional="View the web interface at http://pi.hole/admin:${WEBPORT} or http://${IPV4_ADDRESS%/*}:${WEBPORT}/admin\\n\\nYour Admin Webpage login password is ${pwstring}" - fi + + # Store a message in a variable and display it + additional="View the web interface at http://pi.hole/admin:${WEBPORT} or http://${IPV4_ADDRESS%/*}:${WEBPORT}/admin\\n\\nYour Admin Webpage login password is ${pwstring}" + # Final completion message to user dialog --no-shadow --keep-tite \ @@ -1928,14 +1887,11 @@ clone_or_update_repos() { { printf " %b Unable to reset %s, exiting installer%b\\n" "${COL_LIGHT_RED}" "${PI_HOLE_LOCAL_REPO}" "${COL_NC}"; \ exit 1; \ } - # If the Web interface was installed, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - # reset it's repo - resetRepo ${webInterfaceDir} || \ - { printf " %b Unable to reset %s, exiting installer%b\\n" "${COL_LIGHT_RED}" "${webInterfaceDir}" "${COL_NC}"; \ - exit 1; \ - } - fi + # Reset the Web repo + resetRepo ${webInterfaceDir} || \ + { printf " %b Unable to reset %s, exiting installer%b\\n" "${COL_LIGHT_RED}" "${webInterfaceDir}" "${COL_NC}"; \ + exit 1; \ + } # Otherwise, a repair is happening else # so get git files for Core @@ -1943,14 +1899,11 @@ clone_or_update_repos() { { printf " %b Unable to clone %s into %s, unable to continue%b\\n" "${COL_LIGHT_RED}" "${piholeGitUrl}" "${PI_HOLE_LOCAL_REPO}" "${COL_NC}"; \ exit 1; \ } - # If the Web interface was installed, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - # get the Web git files - getGitFiles ${webInterfaceDir} ${webInterfaceGitUrl} || \ - { printf " %b Unable to clone %s into ${webInterfaceDir}, exiting installer%b\\n" "${COL_LIGHT_RED}" "${webInterfaceGitUrl}" "${COL_NC}"; \ - exit 1; \ - } - fi + # get the Web git files + getGitFiles ${webInterfaceDir} ${webInterfaceGitUrl} || \ + { printf " %b Unable to clone %s into ${webInterfaceDir}, exiting installer%b\\n" "${COL_LIGHT_RED}" "${webInterfaceGitUrl}" "${COL_NC}"; \ + exit 1; \ + } fi } @@ -2253,7 +2206,7 @@ main() { # when run via curl piping if [[ "$0" == "bash" ]]; then # Download the install script and run it with admin rights - exec curl -sSL https://raw.githubusercontent.com/pi-hole/pi-hole/master/automated%20install/basic-install.sh | sudo bash "$@" + exec curl -sSL https://install.pi-hole.net | sudo bash "$@" else # when run via calling local bash script exec sudo bash "$0" "$@" @@ -2319,8 +2272,6 @@ main() { setDNS # Give the user a choice of blocklists to include in their install. Or not. chooseBlocklists - # Let the user decide if they want the web interface to be installed automatically - setAdminFlag # Let the user decide if they want query logging enabled... setLogging # Let the user decide the FTL privacy level @@ -2373,17 +2324,13 @@ main() { # Copy the temp log file into final log location for storage copy_to_install_log - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - # Add password to web UI if there is none - pw="" - # If no password is set, - if [[ $(grep 'WEBPASSWORD' -c "${setupVars}") == 0 ]] ; then - # generate a random password - pw=$(tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c 8) - # shellcheck disable=SC1091 - . /opt/pihole/webpage.sh - echo "WEBPASSWORD=$(HashPassword "${pw}")" >> "${setupVars}" - fi + # Add password to web UI if there is none + pw="" + # If no password is set, + if [[ $(pihole-FTL --config webserver.api.pwhash) == "${pw}" ]] ; then + # generate a random password + pw=$(tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c 8) + pihole -a -p "${pw}" fi # Check for and disable systemd-resolved-DNSStubListener before reloading resolved @@ -2441,21 +2388,17 @@ main() { displayFinalMessage "${pw}" fi - # If the Web interface was installed, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - # If there is a password, - if (( ${#pw} > 0 )) ; then - # display the password - printf " %b Web Interface password: %b%s%b\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${pw}" "${COL_NC}" - printf " %b This can be changed using 'pihole -a -p'\\n\\n" "${INFO}" - fi + # If there is a password + if (( ${#pw} > 0 )) ; then + # display the password + printf " %b Web Interface password: %b%s%b\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${pw}" "${COL_NC}" + printf " %b This can be changed using 'pihole -a -p'\\n\\n" "${INFO}" fi if [[ "${useUpdateVars}" == false ]]; then # If the Web interface was installed, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - printf " %b View the web interface at http://pi.hole:${WEBPORT}/admin or http://%s/admin\\n\\n" "${INFO}" "${IPV4_ADDRESS%/*}:${WEBPORT}" - fi + printf " %b View the web interface at http://pi.hole:${WEBPORT}/admin or http://%s/admin\\n\\n" "${INFO}" "${IPV4_ADDRESS%/*}:${WEBPORT}" + # Explain to the user how to use Pi-hole as their DNS server printf " %b You may now configure your devices to use the Pi-hole as their DNS server\\n" "${INFO}" [[ -n "${IPV4_ADDRESS%/*}" ]] && printf " %b Pi-hole DNS (IPv4): %s\\n" "${INFO}" "${IPV4_ADDRESS%/*}" diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 7a1a290d..8cffae20 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -46,10 +46,6 @@ package_manager_detect # Uninstall packages used by the Pi-hole DEPS=("${INSTALLER_DEPS[@]}" "${PIHOLE_DEPS[@]}" "${OS_CHECK_DEPS[@]}") -if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - # Install the Web dependencies - DEPS+=("${PIHOLE_WEB_DEPS[@]}") -fi # Compatibility if [ -x "$(command -v apt-get)" ]; then diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 66d00814..42c94e63 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -120,7 +120,6 @@ def test_installPihole_fresh_install_readableFiles(host): setup_var_file = "cat < /etc/pihole/setupVars.conf\n" for k, v in SETUPVARS.items(): setup_var_file += "{}={}\n".format(k, v) - setup_var_file += "INSTALL_WEB_INTERFACE=true\n" setup_var_file += "EOF\n" host.run(setup_var_file) install = host.run( From 7cbe713873d38eb4b4952ea7e24a59f4c0084ed7 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 18:59:38 +0000 Subject: [PATCH 023/200] REVISIT: Don't rely on existence of setupVars.conf Signed-off-by: Adam Warner --- advanced/Scripts/update.sh | 3 -- advanced/Scripts/utils.sh | 4 +- automated install/basic-install.sh | 65 ++++++------------------------ test/conftest.py | 7 ---- test/test_any_automated_install.py | 47 --------------------- 5 files changed, 15 insertions(+), 111 deletions(-) diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index 6185d94a..b6153293 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -104,9 +104,6 @@ main() { web_update=false FTL_update=false - # shellcheck disable=1090,2154 - source "${setupVars}" - # Install packages used by this installation script (necessary if users have removed e.g. git from their systems) package_manager_detect install_dependent_packages "${INSTALLER_DEPS[@]}" diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 8bab396a..205edd90 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -162,8 +162,8 @@ getFTLConfigValue(){ # Takes two arguments: key and value # Example setFTLConfigValue dns.piholePTR PI.HOLE # -# Note, for complex values such as dnsmasq.upstreams, you should wrap the value in single quotes: -# setFTLConfigValue dnsmasq.upstreams '[ "8.8.8.8" , "8.8.4.4" ]' +# Note, for complex values such as dns.upstreams, you should wrap the value in single quotes: +# setFTLConfigValue dns.upstreams '[ "8.8.8.8" , "8.8.4.4" ]' ####################### setFTLConfigValue(){ pihole-FTL --config "${1}" "${2}" >/dev/null diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9e5a9963..984fe0c0 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -56,8 +56,6 @@ EOM # Location for final installation log storage installLogLoc="/etc/pihole/install.log" -# This is an important file as it contains information specific to the machine it's being installed on -setupVars="/etc/pihole/setupVars.conf" # This is a file used for the colorized output coltable="/opt/pihole/COL_TABLE" @@ -1566,29 +1564,7 @@ create_pihole_user() { fi } -# This function saves any changes to the setup variables into the setupvars.conf file for future runs finalExports() { - # set or update the variables in the file - - addOrEditKeyValPair "${setupVars}" "PIHOLE_INTERFACE" "${PIHOLE_INTERFACE}" - addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_1" "${PIHOLE_DNS_1}" - addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_2" "${PIHOLE_DNS_2}" - addOrEditKeyValPair "${setupVars}" "QUERY_LOGGING" "${QUERY_LOGGING}" - addOrEditKeyValPair "${setupVars}" "CACHE_SIZE" "${CACHE_SIZE}" - addOrEditKeyValPair "${setupVars}" "DNS_FQDN_REQUIRED" "${DNS_FQDN_REQUIRED:-true}" - addOrEditKeyValPair "${setupVars}" "DNS_BOGUS_PRIV" "${DNS_BOGUS_PRIV:-true}" - addOrEditKeyValPair "${setupVars}" "DNSMASQ_LISTENING" "${DNSMASQ_LISTENING:-local}" - - chmod 644 "${setupVars}" - - # Set the privacy level - addOrEditKeyValPair "${FTL_CONFIG_FILE}" "PRIVACYLEVEL" "${PRIVACY_LEVEL}" - - # Set the web port - addOrEditKeyValPair "${FTL_CONFIG_FILE}" "WEBPORT" "${WEBPORT}" - - # Bring in the current settings and the functions to manipulate them - source "${setupVars}" # shellcheck source=advanced/Scripts/webpage.sh source "${PI_HOLE_LOCAL_REPO}/advanced/Scripts/webpage.sh" @@ -1672,7 +1648,6 @@ installPihole() { # install a man page entry for pihole install_manpage - # Update setupvars.conf with any variables that may or may not have been changed during the install finalExports } @@ -1724,11 +1699,12 @@ checkSelinux() { # Installation complete message with instructions for the user displayFinalMessage() { + # TODO: COME BACK TO THIS, WHAT IS GOING ON? # If the number of arguments is > 0, if [[ "${#1}" -gt 0 ]] ; then # set the password to the first argument. pwstring="$1" - elif [[ $(grep 'WEBPASSWORD' -c "${setupVars}") -gt 0 ]]; then + elif [[ $(pihole-FTL --config webserver.api.pwhash) == '""' ]] ; then # Else if the password exists from previous setup, we'll load it later pwstring="unchanged" else @@ -2242,19 +2218,16 @@ main() { printf " %b Checking for / installing Required dependencies for this install script...\\n" "${INFO}" install_dependent_packages "${INSTALLER_DEPS[@]}" - # If the setup variable file exists, - if [[ -f "${setupVars}" ]]; then - # if it's running unattended, - if [[ "${runUnattended}" == true ]]; then - printf " %b Performing unattended setup, no dialogs will be displayed\\n" "${INFO}" - # Use the setup variables - useUpdateVars=true - # also disable debconf-apt-progress dialogs - export DEBIAN_FRONTEND="noninteractive" - else - # If running attended, show the available options (repair/reconfigure) - update_dialogs - fi + # if it's running unattended, + if [[ "${runUnattended}" == true ]]; then + printf " %b Performing unattended setup, no dialogs will be displayed\\n" "${INFO}" + # Use the setup variables + useUpdateVars=true + # also disable debconf-apt-progress dialogs + export DEBIAN_FRONTEND="noninteractive" + else + # If running attended, show the available options (repair/reconfigure) + update_dialogs fi if [[ "${useUpdateVars}" == false ]]; then @@ -2279,18 +2252,6 @@ main() { else # Setup adlist file if not exists installDefaultBlocklists - - # Source ${setupVars} to use predefined user variables in the functions - source "${setupVars}" - - # Get the privacy level if it exists (default is 0) - if [[ -f "${FTL_CONFIG_FILE}" ]]; then - # get the value from $FTL_CONFIG_FILE (and ignoring all commented lines) - PRIVACY_LEVEL=$(sed -e '/^[[:blank:]]*#/d' "${FTL_CONFIG_FILE}" | grep "PRIVACYLEVEL" | awk -F "=" 'NR==1{printf$2}') - - # If no setting was found, default to 0 - PRIVACY_LEVEL="${PRIVACY_LEVEL:-0}" - fi fi # Download or update the scripts by updating the appropriate git repos clone_or_update_repos @@ -2327,7 +2288,7 @@ main() { # Add password to web UI if there is none pw="" # If no password is set, - if [[ $(pihole-FTL --config webserver.api.pwhash) == "${pw}" ]] ; then + if [[ $(pihole-FTL --config webserver.api.pwhash) == '""' ]] ; then # generate a random password pw=$(tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c 8) pihole -a -p "${pw}" diff --git a/test/conftest.py b/test/conftest.py index e395ec27..164e8de5 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -4,13 +4,6 @@ import testinfra.backend.docker import subprocess from textwrap import dedent - -SETUPVARS = { - "PIHOLE_INTERFACE": "eth99", - "PIHOLE_DNS_1": "4.2.2.1", - "PIHOLE_DNS_2": "4.2.2.2", -} - IMAGE = "pytest_pihole:test_container" tick_box = "[\x1b[1;32m\u2713\x1b[0m]" diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 42c94e63..45cb01c5 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -2,7 +2,6 @@ import pytest from textwrap import dedent import re from .conftest import ( - SETUPVARS, tick_box, info_box, cross_box, @@ -32,42 +31,6 @@ def test_supported_package_manager(host): # assert package_manager_detect.rc == 1 -def test_setupVars_are_sourced_to_global_scope(host): - """ - currently update_dialogs sources setupVars with a dot, - then various other functions use the variables. - This confirms the sourced variables are in scope between functions - """ - setup_var_file = "cat < /etc/pihole/setupVars.conf\n" - for k, v in SETUPVARS.items(): - setup_var_file += "{}={}\n".format(k, v) - setup_var_file += "EOF\n" - host.run(setup_var_file) - - script = dedent( - """\ - set -e - printSetupVars() { - # Currently debug test function only - echo "Outputting sourced variables" - echo "PIHOLE_INTERFACE=${PIHOLE_INTERFACE}" - echo "PIHOLE_DNS_1=${PIHOLE_DNS_1}" - echo "PIHOLE_DNS_2=${PIHOLE_DNS_2}" - } - update_dialogs() { - . /etc/pihole/setupVars.conf - } - update_dialogs - printSetupVars - """ - ) - - output = run_script(host, script).stdout - - for k, v in SETUPVARS.items(): - assert "{}={}".format(k, v) in output - - def test_selinux_not_detected(host): """ confirms installer continues when SELinux configuration file does not exist @@ -116,12 +79,6 @@ def test_installPihole_fresh_install_readableFiles(host): host.run("command -v apt-get > /dev/null && apt-get install -qq man") host.run("command -v dnf > /dev/null && dnf install -y man") host.run("command -v yum > /dev/null && yum install -y man") - # create configuration file - setup_var_file = "cat < /etc/pihole/setupVars.conf\n" - for k, v in SETUPVARS.items(): - setup_var_file += "{}={}\n".format(k, v) - setup_var_file += "EOF\n" - host.run(setup_var_file) install = host.run( """ export TERM=xterm @@ -185,10 +142,6 @@ def test_installPihole_fresh_install_readableFiles(host): check_FTLconf = test_cmd.format("w", "/etc/pihole/pihole-FTL.conf", piholeuser) actual_rc = host.run(check_FTLconf).rc assert exit_status_success == actual_rc - # readable setupVars.conf - check_setup = test_cmd.format("r", "/etc/pihole/setupVars.conf", piholeuser) - actual_rc = host.run(check_setup).rc - assert exit_status_success == actual_rc # check readable and executable /etc/init.d/pihole-FTL check_init = test_cmd.format("x", "/etc/init.d/pihole-FTL", piholeuser) actual_rc = host.run(check_init).rc From 6b1c8a7fff92ae08a2b42d2e827249e6e2d345ca Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 21:30:31 +0000 Subject: [PATCH 024/200] @DL6ER changed the config key names... Remove superfluous test Signed-off-by: Adam Warner --- test/test_any_utils.py | 31 ++----------------------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 741b1127..8dc3f1a5 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -172,33 +172,6 @@ def test_getFTLPIDFile_and_getFTLPID_custom(host): assert expected_stdout == output.stdout -def test_setFTLConfigValue_getFTLConfigValue(host): - """ - Confirms setFTLConfigValue works - Requires FTL to be installed, so we do that first (taken from test_FTL_binary_installed_and_responsive_no_errors) - """ - host.run( - """ - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - echo "new/http" > /etc/pihole/ftlbranch - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - - output = host.run( - """ - source /opt/pihole/utils.sh - setFTLConfigValue "dnsmasq.upstreams" '["1.1.1.1"]' - """ - ) - - assert '[ "1.1.1.1" ]' in output.stdout - - def test_getFTLConfigValue_getFTLConfigValue(host): """ Confirms getFTLConfigValue works (also assumes setFTLConfigValue works) @@ -219,8 +192,8 @@ def test_getFTLConfigValue_getFTLConfigValue(host): output = host.run( """ source /opt/pihole/utils.sh - setFTLConfigValue "dnsmasq.upstreams" '["9.9.9.9"]' > /dev/null - getFTLConfigValue "dnsmasq.upstreams" + setFTLConfigValue "dns.upstreams" '["9.9.9.9"]' > /dev/null + getFTLConfigValue "dns.upstreams" """ ) From a91eb48d48f18026a87067e5ba0ce86a4d91a125 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 11 Feb 2023 12:34:12 +0000 Subject: [PATCH 025/200] Remove some code from gravity that gets values from setupVars.conf / pihole-FTL.conf - use getFTLConfigValue from utils.sh instead Signed-off-by: Adam Warner --- gravity.sh | 39 ++++++++++++--------------------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/gravity.sh b/gravity.sh index 515927e8..66a90449 100755 --- a/gravity.sh +++ b/gravity.sh @@ -13,10 +13,17 @@ export LC_ALL=C -coltable="/opt/pihole/COL_TABLE" -source "${coltable}" +PI_HOLE_SCRIPT_DIR="/opt/pihole" +# Source utils.sh for GetFTLConfigValue +utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" +# shellcheck disable=SC1090 +. "${utilsfile}" + +coltable="${PI_HOLE_SCRIPT_DIR}/COL_TABLE" +# shellcheck disable=SC1090 +. "${coltable}" # shellcheck disable=SC1091 -source "/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh" +. "/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh" basename="pihole" PIHOLE_COMMAND="/usr/local/bin/${basename}" @@ -33,20 +40,13 @@ localList="${piholeDir}/local.list" VPNList="/etc/openvpn/ipp.txt" piholeGitDir="/etc/.pihole" -gravityDBfile_default="${piholeDir}/gravity.db" -# GRAVITYDB may be overwritten by source pihole-FTL.conf below -GRAVITYDB="${gravityDBfile_default}" +GRAVITYDB=$(getFTLConfigValue files.gravity) gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql" gravityDBcopy="${piholeGitDir}/advanced/Templates/gravity_copy.sql" domainsExtension="domains" curl_connect_timeout=10 -# Source setupVars from install script -setupVars="${piholeDir}/setupVars.conf" -if [[ -f "${setupVars}" ]];then - source "${setupVars}" -fi # Set up tmp dir variable in case it's not configured : "${GRAVITY_TMPDIR:=/tmp}" @@ -56,12 +56,6 @@ if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then GRAVITY_TMPDIR="/tmp" fi -# Source pihole-FTL from install script -pihole_FTL="${piholeDir}/pihole-FTL.conf" -if [[ -f "${pihole_FTL}" ]]; then - source "${pihole_FTL}" -fi - # Set this only after sourcing pihole-FTL.conf as the gravity database path may # have changed gravityDBfile="${GRAVITYDB}" @@ -69,15 +63,6 @@ gravityTEMPfile="${GRAVITYDB}_temp" gravityDIR="$(dirname -- "${gravityDBfile}")" gravityOLDfile="${gravityDIR}/gravity_old.db" -if [[ -z "${BLOCKINGMODE}" ]] ; then - BLOCKINGMODE="NULL" -fi - -# Determine if superseded pihole.conf exists -if [[ -r "${piholeDir}/pihole.conf" ]]; then - echo -e " ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}" -fi - # Generate new SQLite3 file from schema template generate_gravity_database() { if ! pihole-FTL sqlite3 "${gravityDBfile}" < "${gravityDBschema}"; then @@ -519,7 +504,7 @@ gravity_DownloadBlocklistFromUrl() { str="Status:" echo -ne " ${INFO} ${str} Pending..." blocked=false - case $BLOCKINGMODE in + case $(getFTLConfigValue dns.blocking.mode) in "IP-NODATA-AAAA"|"IP") # Get IP address of this domain ip="$(dig "${domain}" +short)" From c0e352094da78f1d888e34bbc6dc511e3e069bb0 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 11 Feb 2023 12:51:55 +0000 Subject: [PATCH 026/200] REVISIT: Remove some references to setupVars (some files can probably go entirely...) Signed-off-by: Adam Warner --- advanced/Scripts/chronometer.sh | 8 -------- advanced/Scripts/piholeCheckout.sh | 3 --- advanced/Scripts/piholeDebug.sh | 2 +- advanced/Scripts/utils.sh | 1 + advanced/Scripts/webpage.sh | 8 ++++---- 5 files changed, 6 insertions(+), 16 deletions(-) diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh index d69a56d3..99cff230 100755 --- a/advanced/Scripts/chronometer.sh +++ b/advanced/Scripts/chronometer.sh @@ -210,11 +210,6 @@ get_init_stats() { else temp_file="" fi - - # Test existence of setupVars config - if [[ -f "/etc/pihole/setupVars.conf" ]]; then - setupVars="/etc/pihole/setupVars.conf" - fi } get_sys_stats() { @@ -226,9 +221,6 @@ get_sys_stats() { # Update every 12 refreshes (Def: every 60s) count=$((count+1)) if [[ "$count" == "1" ]] || (( "$count" % 12 == 0 )); then - # Do not source setupVars if file does not exist - [[ -n "$setupVars" ]] && source "$setupVars" - mapfile -t ph_ver_raw < <(pihole -v -c 2> /dev/null | sed -n 's/^.* v/v/p') if [[ -n "${ph_ver_raw[0]}" ]]; then ph_core_ver="${ph_ver_raw[0]}" diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index 5a2eebd7..39d39b1c 100755 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -16,15 +16,12 @@ source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh" # webInterfaceDir set in basic-install.sh # piholeGitURL set in basic-install.sh # is_repo() sourced from basic-install.sh -# setupVars set in basic-install.sh # check_download_exists sourced from basic-install.sh # fully_fetch_repo sourced from basic-install.sh # get_available_branches sourced from basic-install.sh # fetch_checkout_pull_branch sourced from basic-install.sh # checkout_pull_branch sourced from basic-install.sh -source "${setupVars}" - warning1() { echo " Please note that changing branches severely alters your Pi-hole subsystems" echo " Features that work on the master branch, may not on a development branch" diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index e7c383b5..ef1d8558 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -77,7 +77,6 @@ PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" -PIHOLE_SETUP_VARS_FILE="${PIHOLE_DIRECTORY}/setupVars.conf" PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole-FTL.conf" PIHOLE_CUSTOM_HOSTS_FILE="${PIHOLE_DIRECTORY}/custom.list" PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" @@ -1465,6 +1464,7 @@ upload_to_tricorder() { # Run through all the functions we made make_temporary_log initialize_debug +# TODO: Address the reliance on setupVars.conf here. Should debug read pihole.toml directly, or rely on pihole-FTL --config? # setupVars.conf needs to be sourced before the networking so the values are # available to the other functions source_setup_variables diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 205edd90..8f33d678 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -25,6 +25,7 @@ # # Example usage: # addOrEditKeyValPair "/etc/pihole/setupVars.conf" "BLOCKING_ENABLED" "true" +# TODO: We miight not actually need this function in v6 ####################### addOrEditKeyValPair() { local file="${1}" diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 53f36a20..8fd0253b 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -12,6 +12,8 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. +# TODO - this entire file might be able to be removed in v6 + readonly dnsmasqconfig="/etc/dnsmasq.d/01-pihole.conf" readonly dhcpconfig="/etc/dnsmasq.d/02-pihole-dhcp.conf" readonly FTLconf="/etc/pihole/pihole-FTL.conf" @@ -175,10 +177,8 @@ SetWebPassword() { fi if [ "${PASSWORD}" == "${CONFIRM}" ] ; then - # We do not wrap this in brackets, otherwise BASH will expand any appropriate syntax - hash=$(HashPassword "$PASSWORD") - # Save hash to file - setFTLConfigValue "webserver.api.pwhash" "${hash}" >/dev/null + # pihole-FTL will automatically hash the password + setFTLConfigValue "webserver.api.password" "${PASSWORD}" >/dev/null echo -e " ${TICK} New password set" else echo -e " ${CROSS} Passwords don't match. Your password has not been changed" From ea748822ef5d6e0962a25005b32ea5357a85614f Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 12 Feb 2023 17:39:37 +0000 Subject: [PATCH 027/200] REVISIT: I don't _think_ this line is nessacery. Why should gravity restart FTL if it is offline? It might be offline on purpose. Signed-off-by: Adam Warner --- gravity.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gravity.sh b/gravity.sh index 66a90449..3b08ff2b 100755 --- a/gravity.sh +++ b/gravity.sh @@ -720,11 +720,11 @@ gravity_Cleanup() { echo -e "${OVER} ${TICK} ${str}" - # Only restart DNS service if offline - if ! pgrep pihole-FTL &> /dev/null; then - "${PIHOLE_COMMAND}" restartdns - dnsWasOffline=true - fi + # # Only restart DNS service if offline + # if ! pgrep pihole-FTL &> /dev/null; then + # "${PIHOLE_COMMAND}" restartdns + # dnsWasOffline=true + # fi # Print Pi-hole status if an error occurred if [[ -n "${error}" ]]; then @@ -894,4 +894,4 @@ gravity_ShowCount gravity_Cleanup echo "" -"${PIHOLE_COMMAND}" status +# "${PIHOLE_COMMAND}" status From 44bfb8ebf095caa2f68d68e89e73fb194c4034c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 24 Feb 2023 21:11:19 +0100 Subject: [PATCH 028/200] Remove the ability to reboot/shutdown via webpage.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 8fd0253b..4c321ec2 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -401,14 +401,6 @@ SetExcludeClients() { addOrEditKeyValPair "${setupVars}" "API_EXCLUDE_CLIENTS" "${args[2]}" } -Poweroff(){ - nohup bash -c "sleep 5; poweroff" &> /dev/null /dev/null Date: Fri, 24 Feb 2023 21:31:35 +0100 Subject: [PATCH 029/200] Remove getFTLAPIPort() function and fix pihole status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/chronometer.sh | 2 +- advanced/Scripts/utils.sh | 23 ----------------- pihole | 7 +++-- test/test_any_utils.py | 46 --------------------------------- 4 files changed, 4 insertions(+), 74 deletions(-) diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh index 99cff230..49de6efd 100755 --- a/advanced/Scripts/chronometer.sh +++ b/advanced/Scripts/chronometer.sh @@ -16,7 +16,7 @@ pihole-FTL() { local ftl_port LINE # shellcheck disable=SC1091 . /opt/pihole/utils.sh - ftl_port=$(getFTLAPIPort) + ftl_port=$(getFTLConfigValue dns.port) if [[ -n "$ftl_port" ]]; then # Open connection to FTL exec 3<>"/dev/tcp/127.0.0.1/$ftl_port" diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 8f33d678..3f9b7031 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -81,29 +81,6 @@ removeKey() { sed -i "/^${key}/d" "${file}" } - -####################### -# returns FTL's current telnet API port based on the setting in /etc/pihole-FTL.conf -######################## -getFTLAPIPort(){ - local FTLCONFFILE="/etc/pihole/pihole-FTL.conf" - local DEFAULT_FTL_PORT=4711 - local ftl_api_port - - if [ -s "$FTLCONFFILE" ]; then - # if FTLPORT is not set in pihole-FTL.conf, use the default port - ftl_api_port="$({ grep '^FTLPORT=' "${FTLCONFFILE}" || echo "${DEFAULT_FTL_PORT}"; } | cut -d'=' -f2-)" - # Exploit prevention: set the port to the default port if there is malicious (non-numeric) - # content set in pihole-FTL.conf - expr "${ftl_api_port}" : "[^[:digit:]]" > /dev/null && ftl_api_port="${DEFAULT_FTL_PORT}" - else - # if there is no pihole-FTL.conf, use the default port - ftl_api_port="${DEFAULT_FTL_PORT}" - fi - - echo "${ftl_api_port}" -} - ####################### # returns path of FTL's PID file ####################### diff --git a/pihole b/pihole index 0c91df35..0be995e4 100755 --- a/pihole +++ b/pihole @@ -320,13 +320,12 @@ analyze_ports() { statusFunc() { # Determine if there is pihole-FTL service is listening - local pid port ftl_api_port ftl_pid_file + local pid port ftl_pid_file ftl_pid_file="$(getFTLPIDFile)" pid="$(getFTLPID ${ftl_pid_file})" - ftl_api_port="$(getFTLAPIPort)" if [[ "$pid" -eq "-1" ]]; then case "${1}" in "web") echo "-1";; @@ -334,8 +333,8 @@ statusFunc() { esac return 0 else - #get the DNS port pihole-FTL is listening on by using FTL's telnet API - port="$(echo ">dns-port >quit" | nc 127.0.0.1 "$ftl_api_port")" + # get the DNS port pihole-FTL is listening on + port="$(getFTLConfigValue dns.port)" if [[ "${port}" == "0" ]]; then case "${1}" in "web") echo "-1";; diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 8dc3f1a5..efab3760 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -82,52 +82,6 @@ def test_key_removal_works(host): assert expected_stdout == output.stdout -def test_getFTLAPIPort_default(host): - """Confirms getFTLAPIPort returns the default API port""" - output = host.run( - """ - source /opt/pihole/utils.sh - getFTLAPIPort - """ - ) - expected_stdout = "4711\n" - assert expected_stdout == output.stdout - - -def test_getFTLAPIPort_custom(host): - """Confirms getFTLAPIPort returns a custom API port""" - host.run( - """ - echo "FTLPORT=1234" > /etc/pihole/pihole-FTL.conf - """ - ) - output = host.run( - """ - source /opt/pihole/utils.sh - getFTLAPIPort - """ - ) - expected_stdout = "1234\n" - assert expected_stdout == output.stdout - - -def test_getFTLAPIPort_malicious(host): - """Confirms getFTLAPIPort returns 4711 if the setting in pihole-FTL.conf contains non-digits""" - host.run( - """ - echo "FTLPORT=*$ssdfsd" > /etc/pihole/pihole-FTL.conf - """ - ) - output = host.run( - """ - source /opt/pihole/utils.sh - getFTLAPIPort - """ - ) - expected_stdout = "4711\n" - assert expected_stdout == output.stdout - - def test_getFTLPIDFile_default(host): """Confirms getFTLPIDFile returns the default PID file path""" output = host.run( From 137e6dc1843c2b9ccfd1e66632f9b476a4d5fd0d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 28 May 2023 08:57:14 +0200 Subject: [PATCH 030/200] Remove all the undocumented now useless stuff from pihole -a Signed-off-by: DL6ER --- advanced/Scripts/webpage.sh | 790 +----------------------------------- 1 file changed, 5 insertions(+), 785 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 4c321ec2..67cbe766 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -43,126 +43,23 @@ fi helpFunc() { echo "Usage: pihole -a [options] Example: pihole -a -p password -Set options for the Admin Console +Set options for the API/Web interface Options: - -p, password Set Admin Console password - -c, celsius Set Celsius as preferred temperature unit - -f, fahrenheit Set Fahrenheit as preferred temperature unit - -k, kelvin Set Kelvin as preferred temperature unit - -h, --help Show this help dialog - -i, interface Specify dnsmasq's interface listening behavior - -l, privacylevel Set privacy level (0 = lowest, 3 = highest) - -t, teleporter Backup configuration as an archive - -t, teleporter myname.tar.gz Backup configuration to archive with name myname.tar.gz as specified" + -p, password Set API/Web interface password + -h, --help Show this help dialog" exit 0 } -add_setting() { - addOrEditKeyValPair "${setupVars}" "${1}" "${2}" -} - -delete_setting() { - removeKey "${setupVars}" "${1}" -} - -change_setting() { - addOrEditKeyValPair "${setupVars}" "${1}" "${2}" -} - -addFTLsetting() { - addOrEditKeyValPair "${FTLconf}" "${1}" "${2}" -} - -deleteFTLsetting() { - removeKey "${FTLconf}" "${1}" -} - -changeFTLsetting() { - pihole-FTL --config "${1}" "${2}" -} - -add_dnsmasq_setting() { - addOrEditKeyValPair "${dnsmasqconfig}" "${1}" "${2}" -} - -delete_dnsmasq_setting() { - removeKey "${dnsmasqconfig}" "${1}" -} - -SetTemperatureUnit() { - addOrEditKeyValPair "${setupVars}" "TEMPERATUREUNIT" "${unit}" - echo -e " ${TICK} Set temperature unit to ${unit}" -} - -HashPassword() { - # Compute password hash twice to avoid rainbow table vulnerability - return=$(echo -n "${1}" | sha256sum | sed 's/\s.*$//') - return=$(echo -n "${return}" | sha256sum | sed 's/\s.*$//') - echo "${return}" -} - -# Check an IP address to see if it is a valid one -valid_ip() { - # Local, named variables - local ip=${1} - local stat=1 - - # Regex matching one IPv4 component, i.e. an integer from 0 to 255. - # See https://tools.ietf.org/html/rfc1340 - local ipv4elem="(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]?|0)"; - # Regex matching an optional port (starting with '#') range of 1-65536 - local portelem="(#(6553[0-5]|655[0-2][0-9]|65[0-4][0-9]{2}|6[0-4][0-9]{3}|[1-5][0-9]{4}|[1-9][0-9]{0,3}|0))?"; - # Build a full IPv4 regex from the above subexpressions - local regex="^${ipv4elem}\\.${ipv4elem}\\.${ipv4elem}\\.${ipv4elem}${portelem}$" - - # Evaluate the regex, and return the result - [[ $ip =~ ${regex} ]] - - stat=$? - return "${stat}" -} - -valid_ip6() { - local ip=${1} - local stat=1 - - # Regex matching one IPv6 element, i.e. a hex value from 0000 to FFFF - local ipv6elem="[0-9a-fA-F]{1,4}" - # Regex matching an IPv6 CIDR, i.e. 1 to 128 - local v6cidr="(\\/([1-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])){0,1}" - # Regex matching an optional port (starting with '#') range of 1-65536 - local portelem="(#(6553[0-5]|655[0-2][0-9]|65[0-4][0-9]{2}|6[0-4][0-9]{3}|[1-5][0-9]{4}|[1-9][0-9]{0,3}|0))?"; - # Build a full IPv6 regex from the above subexpressions - local regex="^(((${ipv6elem}))*((:${ipv6elem}))*::((${ipv6elem}))*((:${ipv6elem}))*|((${ipv6elem}))((:${ipv6elem})){7})${v6cidr}${portelem}$" - - # Evaluate the regex, and return the result - [[ ${ip} =~ ${regex} ]] - - stat=$? - return "${stat}" -} - +# TODO: We can probably remove the reliance on this function too, just tell people to pihole-FTL --config webserver.api.password "password" SetWebPassword() { - if [ "${SUDO_USER}" == "www-data" ]; then - echo "Security measure: user www-data is not allowed to change webUI password!" - echo "Exiting" - exit 1 - fi - - if [ "${SUDO_USER}" == "lighttpd" ]; then - echo "Security measure: user lighttpd is not allowed to change webUI password!" - echo "Exiting" - exit 1 - fi - if (( ${#args[2]} > 0 )) ; then readonly PASSWORD="${args[2]}" readonly CONFIRM="${PASSWORD}" else # Prevents a bug if the user presses Ctrl+C and it continues to hide the text typed. # So we reset the terminal via stty if the user does press Ctrl+C - trap '{ echo -e "\nNo password will be set" ; stty sane ; exit 1; }' INT + trap '{ echo -e "\nNot changed" ; stty sane ; exit 1; }' INT read -s -r -p "Enter New Password (Blank for no password): " PASSWORD echo "" @@ -186,689 +83,12 @@ SetWebPassword() { fi } -ProcessDNSSettings() { - source "${setupVars}" - - removeKey "${dnsmasqconfig}" "server" - - COUNTER=1 - while true ; do - var=PIHOLE_DNS_${COUNTER} - if [ -z "${!var}" ]; then - break; - fi - addKey "${dnsmasqconfig}" "server=${!var}" - (( COUNTER++ )) - done - - # The option LOCAL_DNS_PORT is deprecated - # We apply it once more, and then convert it into the current format - if [ -n "${LOCAL_DNS_PORT}" ]; then - addOrEditKeyValPair "${dnsmasqconfig}" "server" "127.0.0.1#${LOCAL_DNS_PORT}" - addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_${COUNTER}" "127.0.0.1#${LOCAL_DNS_PORT}" - removeKey "${setupVars}" "LOCAL_DNS_PORT" - fi - - removeKey "${dnsmasqconfig}" "domain-needed" - removeKey "${dnsmasqconfig}" "expand-hosts" - - if [[ "${DNS_FQDN_REQUIRED}" == true ]]; then - addKey "${dnsmasqconfig}" "domain-needed" - addKey "${dnsmasqconfig}" "expand-hosts" - fi - - removeKey "${dnsmasqconfig}" "bogus-priv" - - if [[ "${DNS_BOGUS_PRIV}" == true ]]; then - addKey "${dnsmasqconfig}" "bogus-priv" - fi - - removeKey "${dnsmasqconfig}" "dnssec" - removeKey "${dnsmasqconfig}" "trust-anchor" - - if [[ "${DNSSEC}" == true ]]; then - echo "dnssec -trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC683457104237C7F8EC8D -" >> "${dnsmasqconfig}" - fi - - removeKey "${dnsmasqconfig}" "host-record" - - if [ -n "${HOSTRECORD}" ]; then - addOrEditKeyValPair "${dnsmasqconfig}" "host-record" "${HOSTRECORD}" - fi - - # Setup interface listening behavior of dnsmasq - removeKey "${dnsmasqconfig}" "interface" - removeKey "${dnsmasqconfig}" "local-service" - removeKey "${dnsmasqconfig}" "except-interface" - removeKey "${dnsmasqconfig}" "bind-interfaces" - - if [[ "${DNSMASQ_LISTENING}" == "all" ]]; then - # Listen on all interfaces, permit all origins - addOrEditKeyValPair "${dnsmasqconfig}" "except-interface" "nonexisting" - elif [[ "${DNSMASQ_LISTENING}" == "local" ]]; then - # Listen only on all interfaces, but only local subnets - addKey "${dnsmasqconfig}" "local-service" - else - # Options "bind" and "single" - # Listen only on one interface - # Use eth0 as fallback interface if interface is missing in setupVars.conf - if [ -z "${PIHOLE_INTERFACE}" ]; then - PIHOLE_INTERFACE="eth0" - fi - - addOrEditKeyValPair "${dnsmasqconfig}" "interface" "${PIHOLE_INTERFACE}" - - if [[ "${DNSMASQ_LISTENING}" == "bind" ]]; then - # Really bind to interface - addKey "${dnsmasqconfig}" "bind-interfaces" - fi - fi - - if [[ "${CONDITIONAL_FORWARDING}" == true ]]; then - # Convert legacy "conditional forwarding" to rev-server configuration - # Remove any existing REV_SERVER settings - removeKey "${setupVars}" "REV_SERVER" - removeKey "${setupVars}" "REV_SERVER_DOMAIN" - removeKey "${setupVars}" "REV_SERVER_TARGET" - removeKey "${setupVars}" "REV_SERVER_CIDR" - - REV_SERVER=true - addOrEditKeyValPair "${setupVars}" "REV_SERVER" "true" - - REV_SERVER_DOMAIN="${CONDITIONAL_FORWARDING_DOMAIN}" - addOrEditKeyValPair "${setupVars}" "REV_SERVER_DOMAIN" "${REV_SERVER_DOMAIN}" - - REV_SERVER_TARGET="${CONDITIONAL_FORWARDING_IP}" - addOrEditKeyValPair "${setupVars}" "REV_SERVER_TARGET" "${REV_SERVER_TARGET}" - - #Convert CONDITIONAL_FORWARDING_REVERSE if necessary e.g: - # 1.1.168.192.in-addr.arpa to 192.168.1.1/32 - # 1.168.192.in-addr.arpa to 192.168.1.0/24 - # 168.192.in-addr.arpa to 192.168.0.0/16 - # 192.in-addr.arpa to 192.0.0.0/8 - if [[ "${CONDITIONAL_FORWARDING_REVERSE}" == *"in-addr.arpa" ]];then - arrRev=("${CONDITIONAL_FORWARDING_REVERSE//./ }") - case ${#arrRev[@]} in - 6 ) REV_SERVER_CIDR="${arrRev[3]}.${arrRev[2]}.${arrRev[1]}.${arrRev[0]}/32";; - 5 ) REV_SERVER_CIDR="${arrRev[2]}.${arrRev[1]}.${arrRev[0]}.0/24";; - 4 ) REV_SERVER_CIDR="${arrRev[1]}.${arrRev[0]}.0.0/16";; - 3 ) REV_SERVER_CIDR="${arrRev[0]}.0.0.0/8";; - esac - else - # Set REV_SERVER_CIDR to whatever value it was set to - REV_SERVER_CIDR="${CONDITIONAL_FORWARDING_REVERSE}" - fi - - # If REV_SERVER_CIDR is not converted by the above, then use the REV_SERVER_TARGET variable to derive it - if [ -z "${REV_SERVER_CIDR}" ]; then - # Convert existing input to /24 subnet (preserves legacy behavior) - # This sed converts "192.168.1.2" to "192.168.1.0/24" - # shellcheck disable=2001 - REV_SERVER_CIDR="$(sed "s+\\.[0-9]*$+\\.0/24+" <<< "${REV_SERVER_TARGET}")" - fi - addOrEditKeyValPair "${setupVars}" "REV_SERVER_CIDR" "${REV_SERVER_CIDR}" - - # Remove obsolete settings from setupVars.conf - removeKey "${setupVars}" "CONDITIONAL_FORWARDING" - removeKey "${setupVars}" "CONDITIONAL_FORWARDING_REVERSE" - removeKey "${setupVars}" "CONDITIONAL_FORWARDING_DOMAIN" - removeKey "${setupVars}" "CONDITIONAL_FORWARDING_IP" - fi - - removeKey "${dnsmasqconfig}" "rev-server" - - if [[ "${REV_SERVER}" == true ]]; then - addKey "${dnsmasqconfig}" "rev-server=${REV_SERVER_CIDR},${REV_SERVER_TARGET}" - if [ -n "${REV_SERVER_DOMAIN}" ]; then - # Forward local domain names to the CF target, too - addKey "${dnsmasqconfig}" "server=/${REV_SERVER_DOMAIN}/${REV_SERVER_TARGET}" - fi - - if [[ "${DNS_FQDN_REQUIRED}" != true ]]; then - # Forward unqualified names to the CF target only when the "never - # forward non-FQDN" option is unticked - addKey "${dnsmasqconfig}" "server=//${REV_SERVER_TARGET}" - fi - - fi - - # We need to process DHCP settings here as well to account for possible - # changes in the non-FQDN forwarding. This cannot be done in 01-pihole.conf - # as we don't want to delete all local=/.../ lines so it's much safer to - # simply rewrite the entire corresponding config file (which is what the - # DHCP settings subroutine is doing) - ProcessDHCPSettings -} - -SetDNSServers() { - # Save setting to file - removeKey "${setupVars}" "PIHOLE_DNS" - IFS=',' read -r -a array <<< "${args[2]}" - for index in "${!array[@]}" - do - # Replace possible "\#" by "#". This fixes AdminLTE#1427 - local ip - ip="${array[index]//\\#/#}" - - if valid_ip "${ip}" || valid_ip6 "${ip}" ; then - addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_$((index+1))" "${ip}" - else - echo -e " ${CROSS} Invalid IP has been passed" - exit 1 - fi - done - - if [[ "${args[3]}" == "domain-needed" ]]; then - addOrEditKeyValPair "${setupVars}" "DNS_FQDN_REQUIRED" "true" - else - addOrEditKeyValPair "${setupVars}" "DNS_FQDN_REQUIRED" "false" - fi - - if [[ "${args[4]}" == "bogus-priv" ]]; then - addOrEditKeyValPair "${setupVars}" "DNS_BOGUS_PRIV" "true" - else - addOrEditKeyValPair "${setupVars}" "DNS_BOGUS_PRIV" "false" - fi - - if [[ "${args[5]}" == "dnssec" ]]; then - addOrEditKeyValPair "${setupVars}" "DNSSEC" "true" - else - addOrEditKeyValPair "${setupVars}" "DNSSEC" "false" - fi - - if [[ "${args[6]}" == "rev-server" ]]; then - addOrEditKeyValPair "${setupVars}" "REV_SERVER" "true" - addOrEditKeyValPair "${setupVars}" "REV_SERVER_CIDR" "${args[7]}" - addOrEditKeyValPair "${setupVars}" "REV_SERVER_TARGET" "${args[8]}" - addOrEditKeyValPair "${setupVars}" "REV_SERVER_DOMAIN" "${args[9]}" - else - addOrEditKeyValPair "${setupVars}" "REV_SERVER" "false" - fi - - ProcessDNSSettings - - # Restart dnsmasq to load new configuration - RestartDNS -} - -SetExcludeDomains() { - addOrEditKeyValPair "${setupVars}" "API_EXCLUDE_DOMAINS" "${args[2]}" -} - -SetExcludeClients() { - addOrEditKeyValPair "${setupVars}" "API_EXCLUDE_CLIENTS" "${args[2]}" -} - -RestartDNS() { - "${PI_HOLE_BIN_DIR}"/pihole restartdns -} - -SetQueryLogOptions() { - addOrEditKeyValPair "${setupVars}" "API_QUERY_LOG_SHOW" "${args[2]}" -} - -ProcessDHCPSettings() { - source "${setupVars}" - - if [[ "${DHCP_ACTIVE}" == "true" ]]; then - interface="${PIHOLE_INTERFACE}" - - # Use eth0 as fallback interface - if [ -z ${interface} ]; then - interface="eth0" - fi - - if [[ "${PIHOLE_DOMAIN}" == "" ]]; then - PIHOLE_DOMAIN="lan" - addOrEditKeyValPair "${setupVars}" "PIHOLE_DOMAIN" "${PIHOLE_DOMAIN}" - fi - - if [[ "${DHCP_LEASETIME}" == "0" ]]; then - leasetime="infinite" - elif [[ "${DHCP_LEASETIME}" == "" ]]; then - leasetime="24h" - addOrEditKeyValPair "${setupVars}" "DHCP_LEASETIME" "24" - else - leasetime="${DHCP_LEASETIME}h" - fi - - # Write settings to file - echo "############################################################################### -# DHCP SERVER CONFIG FILE AUTOMATICALLY POPULATED BY PI-HOLE WEB INTERFACE. # -# ANY CHANGES MADE TO THIS FILE WILL BE LOST ON CHANGE # -############################################################################### -dhcp-authoritative -dhcp-range=${DHCP_START},${DHCP_END},${leasetime} -dhcp-option=option:router,${DHCP_ROUTER} -dhcp-leasefile=/etc/pihole/dhcp.leases -#quiet-dhcp -" > "${dhcpconfig}" - chmod 644 "${dhcpconfig}" - - if [[ "${PIHOLE_DOMAIN}" != "none" ]]; then - echo "domain=${PIHOLE_DOMAIN}" >> "${dhcpconfig}" - - # When there is a Pi-hole domain set and "Never forward non-FQDNs" is - # ticked, we add `local=/domain/` to tell FTL that this domain is purely - # local and FTL may answer queries from /etc/hosts or DHCP but should - # never forward queries on that domain to any upstream servers - if [[ "${DNS_FQDN_REQUIRED}" == true ]]; then - echo "local=/${PIHOLE_DOMAIN}/" >> "${dhcpconfig}" - fi - fi - - # Sourced from setupVars - # shellcheck disable=SC2154 - if [[ "${DHCP_rapid_commit}" == "true" ]]; then - echo "dhcp-rapid-commit" >> "${dhcpconfig}" - fi - - if [[ "${DHCP_IPv6}" == "true" ]]; then - echo "#quiet-dhcp6 -#enable-ra -dhcp-option=option6:dns-server,[::] -dhcp-range=::,constructor:${interface},ra-names,ra-stateless,64 - -" >> "${dhcpconfig}" - fi - - else - if [[ -f "${dhcpconfig}" ]]; then - rm "${dhcpconfig}" &> /dev/null - fi - fi -} - -EnableDHCP() { - addOrEditKeyValPair "${setupVars}" "DHCP_ACTIVE" "true" - addOrEditKeyValPair "${setupVars}" "DHCP_START" "${args[2]}" - addOrEditKeyValPair "${setupVars}" "DHCP_END" "${args[3]}" - addOrEditKeyValPair "${setupVars}" "DHCP_ROUTER" "${args[4]}" - addOrEditKeyValPair "${setupVars}" "DHCP_LEASETIME" "${args[5]}" - addOrEditKeyValPair "${setupVars}" "PIHOLE_DOMAIN" "${args[6]}" - addOrEditKeyValPair "${setupVars}" "DHCP_IPv6" "${args[7]}" - addOrEditKeyValPair "${setupVars}" "DHCP_rapid_commit" "${args[8]}" - - # Remove possible old setting from file - removeKey "${dnsmasqconfig}" "dhcp-" - removeKey "${dnsmasqconfig}" "quiet-dhcp" - - # If a DHCP client claims that its name is "wpad", ignore that. - # This fixes a security hole. see CERT Vulnerability VU#598349 - # We also ignore "localhost" as Windows behaves strangely if a - # device claims this host name - addKey "${dnsmasqconfig}" "dhcp-name-match=set:hostname-ignore,wpad -dhcp-name-match=set:hostname-ignore,localhost -dhcp-ignore-names=tag:hostname-ignore" - - ProcessDHCPSettings - - RestartDNS -} - -DisableDHCP() { - addOrEditKeyValPair "${setupVars}" "DHCP_ACTIVE" "false" - - # Remove possible old setting from file - removeKey "${dnsmasqconfig}" "dhcp-" - removeKey "${dnsmasqconfig}" "quiet-dhcp" - - ProcessDHCPSettings - - RestartDNS -} - -SetWebUILayout() { - addOrEditKeyValPair "${setupVars}" "WEBUIBOXEDLAYOUT" "${args[2]}" -} - -SetWebUITheme() { - addOrEditKeyValPair "${setupVars}" "WEBTHEME" "${args[2]}" -} - -CheckUrl(){ - local regex check_url - # Check for characters NOT allowed in URLs - regex="[^a-zA-Z0-9:/?&%=~._()-;]" - - # this will remove first @ that is after schema and before domain - # \1 is optional schema, \2 is userinfo - check_url="$( sed -re 's#([^:/]*://)?([^/]+)@#\1\2#' <<< "$1" )" - - if [[ "${check_url}" =~ ${regex} ]]; then - return 1 - else - return 0 - fi -} - -CustomizeAdLists() { - local address - address="${args[3]}" - local comment - comment="${args[4]}" - - if CheckUrl "${address}"; then - if [[ "${args[2]}" == "enable" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'" - elif [[ "${args[2]}" == "disable" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'" - elif [[ "${args[2]}" == "add" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address, comment) VALUES ('${address}', '${comment}')" - elif [[ "${args[2]}" == "del" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'" - else - echo "Not permitted" - return 1 - fi - else - echo "Invalid Url" - return 1 - fi -} - -AddDHCPStaticAddress() { - mac="${args[2]}" - ip="${args[3]}" - host="${args[4]}" - - if [[ "${ip}" == "noip" ]]; then - # Static host name - echo "dhcp-host=${mac},${host}" >> "${dhcpstaticconfig}" - elif [[ "${host}" == "nohost" ]]; then - # Static IP - echo "dhcp-host=${mac},${ip}" >> "${dhcpstaticconfig}" - else - # Full info given - echo "dhcp-host=${mac},${ip},${host}" >> "${dhcpstaticconfig}" - fi -} - -RemoveDHCPStaticAddress() { - mac="${args[2]}" - if [[ "$mac" =~ ^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$ ]]; then - sed -i "/dhcp-host=${mac}.*/d" "${dhcpstaticconfig}" - else - echo " ${CROSS} Invalid Mac Passed!" - exit 1 - fi - -} - -SetListeningMode() { - source "${setupVars}" - - if [[ "$3" == "-h" ]] || [[ "$3" == "--help" ]]; then - echo "Usage: pihole -a -i [interface] -Example: 'pihole -a -i local' -Specify dnsmasq's network interface listening behavior - -Interfaces: - local Only respond to queries from devices that - are at most one hop away (local devices) - single Respond only on interface ${PIHOLE_INTERFACE} - bind Bind only on interface ${PIHOLE_INTERFACE} - all Listen on all interfaces, permit all origins" - exit 0 - fi - - if [[ "${args[2]}" == "all" ]]; then - echo -e " ${INFO} Listening on all interfaces, permitting all origins. Please use a firewall!" - addOrEditKeyValPair "${setupVars}" "DNSMASQ_LISTENING" "all" - elif [[ "${args[2]}" == "local" ]]; then - echo -e " ${INFO} Listening on all interfaces, permitting origins from one hop away (LAN)" - addOrEditKeyValPair "${setupVars}" "DNSMASQ_LISTENING" "local" - elif [[ "${args[2]}" == "bind" ]]; then - echo -e " ${INFO} Binding on interface ${PIHOLE_INTERFACE}" - addOrEditKeyValPair "${setupVars}" "DNSMASQ_LISTENING" "bind" - else - echo -e " ${INFO} Listening only on interface ${PIHOLE_INTERFACE}" - addOrEditKeyValPair "${setupVars}" "DNSMASQ_LISTENING" "single" - fi - - # Don't restart DNS server yet because other settings - # will be applied afterwards if "-web" is set - if [[ "${args[3]}" != "-web" ]]; then - ProcessDNSSettings - # Restart dnsmasq to load new configuration - RestartDNS - fi -} - -Teleporter() { - local filename - filename="${args[2]}" - if [[ -z "${filename}" ]]; then - local datetimestamp - local host - datetimestamp=$(date "+%Y-%m-%d_%H-%M-%S") - host=$(hostname) - host="${host//./_}" - filename="pi-hole-${host:-noname}-teleporter_${datetimestamp}.tar.gz" - fi - php "${webroot}/admin/scripts/pi-hole/php/teleporter.php" > "${filename}" -} - -checkDomain() -{ - local domain validDomain - # Convert to lowercase - domain="${1,,}" - validDomain=$(grep -P "^((-|_)*[a-z0-9]((-|_)*[a-z0-9)*(-|_)*)(\\.(-|_)*([a-z0-9]((-|_)*[a-z0-9])*))*$" <<< "${domain}") # Valid chars check - validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label - echo "${validDomain}" -} - -escapeDots() -{ - # SC suggest bashism ${variable//search/replace} - # shellcheck disable=SC2001 - escaped=$(echo "$1" | sed 's/\./\\./g') - echo "${escaped}" -} - -addAudit() -{ - shift # skip "-a" - shift # skip "audit" - local domains validDomain - domains="" - for domain in "$@" - do - # Check domain to be added. Only continue if it is valid - validDomain="$(checkDomain "${domain}")" - if [[ -n "${validDomain}" ]]; then - # Put comma in between domains when there is - # more than one domains to be added - # SQL INSERT allows adding multiple rows at once using the format - ## INSERT INTO table (domain) VALUES ('abc.de'),('fgh.ij'),('klm.no'),('pqr.st'); - if [[ -n "${domains}" ]]; then - domains="${domains}," - fi - domains="${domains}('${domain}')" - fi - done - # Insert only the domain here. The date_added field will be - # filled with its default value (date_added = current timestamp) - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};" -} - -clearAudit() -{ - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domain_audit;" -} - -SetPrivacyLevel() { - # Set privacy level. Minimum is 0, maximum is 3 - if [ "${args[2]}" -ge 0 ] && [ "${args[2]}" -le 3 ]; then - addOrEditKeyValPair "${FTLconf}" "PRIVACYLEVEL" "${args[2]}" - pihole restartdns reload-lists - fi -} - -AddCustomDNSAddress() { - echo -e " ${TICK} Adding custom DNS entry..." - - ip="${args[2]}" - host="${args[3]}" - reload="${args[4]}" - - validHost="$(checkDomain "${host}")" - if [[ -n "${validHost}" ]]; then - if valid_ip "${ip}" || valid_ip6 "${ip}" ; then - echo "${ip} ${validHost}" >> "${dnscustomfile}" - else - echo -e " ${CROSS} Invalid IP has been passed" - exit 1 - fi - else - echo " ${CROSS} Invalid Domain passed!" - exit 1 - fi - - # Restart dnsmasq to load new custom DNS entries only if $reload not false - if [[ ! $reload == "false" ]]; then - RestartDNS - fi -} - -RemoveCustomDNSAddress() { - echo -e " ${TICK} Removing custom DNS entry..." - - ip="${args[2]}" - host="${args[3]}" - reload="${args[4]}" - - validHost="$(checkDomain "${host}")" - if [[ -n "${validHost}" ]]; then - if valid_ip "${ip}" || valid_ip6 "${ip}" ; then - validHost=$(escapeDots "${validHost}") - sed -i "/^${ip} ${validHost}$/Id" "${dnscustomfile}" - else - echo -e " ${CROSS} Invalid IP has been passed" - exit 1 - fi - else - echo " ${CROSS} Invalid Domain passed!" - exit 1 - fi - - # Restart dnsmasq to load new custom DNS entries only if reload is not false - if [[ ! $reload == "false" ]]; then - RestartDNS - fi -} - -AddCustomCNAMERecord() { - echo -e " ${TICK} Adding custom CNAME record..." - - domain="${args[2]}" - target="${args[3]}" - reload="${args[4]}" - - validDomain="$(checkDomain "${domain}")" - if [[ -n "${validDomain}" ]]; then - validTarget="$(checkDomain "${target}")" - if [[ -n "${validTarget}" ]]; then - if [ "${validDomain}" = "${validTarget}" ]; then - echo " ${CROSS} Domain and target are the same. This would cause a DNS loop." - exit 1 - else - echo "cname=${validDomain},${validTarget}" >> "${dnscustomcnamefile}" - fi - else - echo " ${CROSS} Invalid Target Passed!" - exit 1 - fi - else - echo " ${CROSS} Invalid Domain passed!" - exit 1 - fi - # Restart dnsmasq to load new custom CNAME records only if reload is not false - if [[ ! $reload == "false" ]]; then - RestartDNS - fi -} - -RemoveCustomCNAMERecord() { - echo -e " ${TICK} Removing custom CNAME record..." - - domain="${args[2]}" - target="${args[3]}" - reload="${args[4]}" - - validDomain="$(checkDomain "${domain}")" - if [[ -n "${validDomain}" ]]; then - validTarget="$(checkDomain "${target}")" - if [[ -n "${validTarget}" ]]; then - validDomain=$(escapeDots "${validDomain}") - validTarget=$(escapeDots "${validTarget}") - sed -i "/^cname=${validDomain},${validTarget}$/Id" "${dnscustomcnamefile}" - else - echo " ${CROSS} Invalid Target Passed!" - exit 1 - fi - else - echo " ${CROSS} Invalid Domain passed!" - exit 1 - fi - - # Restart dnsmasq to update removed custom CNAME records only if $reload not false - if [[ ! $reload == "false" ]]; then - RestartDNS - fi -} - -SetRateLimit() { - local rate_limit_count rate_limit_interval reload - rate_limit_count="${args[2]}" - rate_limit_interval="${args[3]}" - reload="${args[4]}" - - # Set rate-limit setting inf valid - if [ "${rate_limit_count}" -ge 0 ] && [ "${rate_limit_interval}" -ge 0 ]; then - addOrEditKeyValPair "${FTLconf}" "RATE_LIMIT" "${rate_limit_count}/${rate_limit_interval}" - fi - - # Restart FTL to update rate-limit settings only if $reload not false - if [[ ! $reload == "false" ]]; then - RestartDNS - fi -} - main() { args=("$@") case "${args[1]}" in "-p" | "password" ) SetWebPassword;; - "-c" | "celsius" ) unit="C"; SetTemperatureUnit;; - "-f" | "fahrenheit" ) unit="F"; SetTemperatureUnit;; - "-k" | "kelvin" ) unit="K"; SetTemperatureUnit;; - "setdns" ) SetDNSServers;; - "setexcludedomains" ) SetExcludeDomains;; - "setexcludeclients" ) SetExcludeClients;; - "restartdns" ) RestartDNS;; - "setquerylog" ) SetQueryLogOptions;; - "enabledhcp" ) EnableDHCP;; - "disabledhcp" ) DisableDHCP;; - "layout" ) SetWebUILayout;; - "theme" ) SetWebUITheme;; "-h" | "--help" ) helpFunc;; - "addstaticdhcp" ) AddDHCPStaticAddress;; - "removestaticdhcp" ) RemoveDHCPStaticAddress;; - "-i" | "interface" ) SetListeningMode "$@";; - "-t" | "teleporter" ) Teleporter;; - "adlist" ) CustomizeAdLists;; - "audit" ) addAudit "$@";; - "clearaudit" ) clearAudit;; - "-l" | "privacylevel" ) SetPrivacyLevel;; - "addcustomdns" ) AddCustomDNSAddress;; - "removecustomdns" ) RemoveCustomDNSAddress;; - "addcustomcname" ) AddCustomCNAMERecord;; - "removecustomcname" ) RemoveCustomCNAMERecord;; - "ratelimit" ) SetRateLimit;; * ) helpFunc;; esac From f7b9d70054758137d60318e6df32ebb0d92987fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 30 May 2023 21:39:10 +0200 Subject: [PATCH 031/200] Remove man page for pihole-FTL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 8 +- manpages/pihole-FTL.8 | 154 ----------------------------- test/test_any_automated_install.py | 5 - 3 files changed, 5 insertions(+), 162 deletions(-) delete mode 100644 manpages/pihole-FTL.8 diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 984fe0c0..28813f1e 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1253,12 +1253,14 @@ install_manpage() { fi # Testing complete, copy the files & update the man db install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/manpages/pihole.8 /usr/local/share/man/man8/pihole.8 - install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.8 /usr/local/share/man/man8/pihole-FTL.8 - # remove previously installed "pihole-FTL.conf.5" man page + # remove previously installed man pages if [[ -f "/usr/local/share/man/man5/pihole-FTL.conf.5" ]]; then rm /usr/local/share/man/man5/pihole-FTL.conf.5 fi + if [[ -f "/usr/local/share/man/man5/pihole-FTL.8" ]]; then + rm /usr/local/share/man/man5/pihole-FTL.8 + fi if mandb -q &>/dev/null; then # Updated successfully @@ -1267,7 +1269,7 @@ install_manpage() { else # Something is wrong with the system's man installation, clean up # our files, (leave everything how we found it). - rm /usr/local/share/man/man8/pihole.8 /usr/local/share/man/man8/pihole-FTL.8 + rm /usr/local/share/man/man8/pihole.8 printf "%b %b man page db not updated, man pages not installed\\n" "${OVER}" "${CROSS}" fi } diff --git a/manpages/pihole-FTL.8 b/manpages/pihole-FTL.8 deleted file mode 100644 index c1b7550f..00000000 --- a/manpages/pihole-FTL.8 +++ /dev/null @@ -1,154 +0,0 @@ -.TH "Pihole-FTL" "8" "pihole-FTL" "Pi-hole" "November 2020" -.SH "NAME" -pihole-FTL - Pi-hole : The Faster-Than-Light (FTL) Engine -.br -.SH "SYNOPSIS" -\fBservice pihole-FTL \fR(\fBstart\fR|\fBstop\fR|\fBrestart\fR) -.br - -\fBpihole-FTL debug\fR -.br -\fBpihole-FTL test\fR -.br -\fBpihole-FTL -v|-vv\fR -.br -\fBpihole-FTL -t\fR -.br -\fBpihole-FTL -b\fR -.br -\fBpihole-FTL -f\fR -.br -\fBpihole-FTL -h\fR -.br -\fBpihole-FTL dnsmasq-test\fR -.br -\fBpihole-FTL regex-test str\fR -.br -\fBpihole-FTL regex-test str rgx\fR -.br -\fBpihole-FTL lua\fR -.br -\fBpihole-FTL luac\fR -.br -\fBpihole-FTL dhcp-discover\fR -.br -\fBpihole-FTL --\fR (\fBoptions\fR) -.br - -.SH "DESCRIPTION" -Pi-hole : The Faster-Than-Light (FTL) Engine is a lightweight, purpose-built daemon used to provide statistics needed for the Pi-hole Web Interface, and its API can be easily integrated into your own projects. Although it is an optional component of the Pi-hole ecosystem, it will be installed by default to provide statistics. As the name implies, FTL does its work \fIvery\fR \fIquickly\fR! -.br - -Usage -.br - -\fBservice pihole-FTL start\fR -.br - Start the pihole-FTL daemon -.br - -\fBservice pihole-FTL stop\fR -.br - Stop the pihole-FTL daemon -.br - -\fBservice pihole-FTL restart\fR -.br - If the pihole-FTP daemon is running, stop and then start, otherwise start. -.br - -Command line arguments -.br - -\fBdebug\fR -.br - Don't go into daemon mode (stay in foreground) + more verbose logging -.br - -\fBtest\fR -.br - Start FTL and process everything, but shut down immediately afterwards -.br - -\fB-v, version\fR -.br - Don't start FTL, show only version -.br - -\fB-vv\fR -.br - Don't start FTL, show verbose version information of embedded applications -.br - -\fB-t, tag\fR -.br - Don't start FTL, show only git tag -.br - -\fB-b, branch\fR -.br - Don't start FTL, show only git branch FTL was compiled from -.br - -\fB-f, no-daemon\fR -.br - Don't go into background (daemon mode) -.br - -\fB-h, help\fR -.br - Don't start FTL, show help -.br - -\fBdnsmasq-test\fR -.br - Test resolver config file syntax -.br - -\fBregex-test str\fR -.br - Test str against all regular expressions in the database -.br - -\fBregex-test str rgx\fR -.br - Test str against regular expression given by rgx -.br - -\fBlua\fR -.br - Start the embedded Lua interpreter -.br - -\fBluac\fR -.br - Execute the embedded Lua compiler -.br - -\fBdhcp-discover\fR -.br - Discover DHCP servers in the local network -.br - -\fB--\fR (options) -.br - Pass options to internal dnsmasq resolver -.br -.SH "EXAMPLE" -Command line arguments can be arbitrarily combined, e.g: -.br - -\fBpihole-FTL debug test\fR -.br - -Start ftl in foreground with more verbose logging, process everything and shutdown immediately -.br -.SH "SEE ALSO" -\fBpihole\fR(8) -.br -\fBFor FTL's config options please see https://docs.pi-hole.net/ftldns/configfile/\fR -.br -.SH "COLOPHON" - -Get sucked into the latest news and community activity by entering Pi-hole's orbit. Information about Pi-hole, and the latest version of the software can be found at https://pi-hole.net -.br diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 45cb01c5..3626ce6f 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -174,11 +174,6 @@ def test_installPihole_fresh_install_readableFiles(host): ) actual_rc = host.run(check_man).rc assert exit_status_success == actual_rc - check_man = test_cmd.format( - "r", "/usr/local/share/man/man8/pihole-FTL.8", piholeuser - ) - actual_rc = host.run(check_man).rc - assert exit_status_success == actual_rc # check not readable sudoers file check_sudo = test_cmd.format("r", "/etc/sudoers.d/pihole", piholeuser) actual_rc = host.run(check_sudo).rc From 792b0d419b20875dab023e311d532318f80ab2ff Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Thu, 1 Jun 2023 19:29:47 +0100 Subject: [PATCH 032/200] Test fixes Signed-off-by: Adam Warner --- automated install/basic-install.sh | 13 ------------- test/test_any_automated_install.py | 27 --------------------------- test/test_any_utils.py | 2 +- 3 files changed, 1 insertion(+), 41 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a6f6b865..6663bbad 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1566,17 +1566,6 @@ create_pihole_user() { fi } -finalExports() { - # shellcheck source=advanced/Scripts/webpage.sh - source "${PI_HOLE_LOCAL_REPO}/advanced/Scripts/webpage.sh" - - # Look for DNS server settings which would have to be reapplied - ProcessDNSSettings - - # Look for DHCP server settings which would have to be reapplied - ProcessDHCPSettings -} - # Install the logrotate script installLogrotate() { local str="Installing latest logrotate script" @@ -1649,8 +1638,6 @@ installPihole() { # install a man page entry for pihole install_manpage - - finalExports } # SELinux diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 07a18100..3626ce6f 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -781,30 +781,3 @@ def test_package_manager_has_web_deps(host): assert "No package" not in output.stdout assert output.rc == 0 - - -def test_webpage_sh_valid_domain(host): - """Confirms checkDomain function in webpage.sh works as expected""" - check1 = host.run( - """ - source /opt/pihole/webpage.sh - checkDomain "pi-hole.net" - """ - ) - check2 = host.run( - """ - source /opt/pihole/webpage.sh - checkDomain "ab.pi-hole.net" - """ - ) - - check3 = host.run( - """ - source /opt/pihole/webpage.sh - checkDomain "abc.pi-hole.net" - """ - ) - - assert "pi-hole.net" in check1.stdout - assert "ab.pi-hole.net" in check2.stdout - assert "abc.pi-hole.net" in check3.stdout diff --git a/test/test_any_utils.py b/test/test_any_utils.py index efab3760..1efd77c3 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -151,4 +151,4 @@ def test_getFTLConfigValue_getFTLConfigValue(host): """ ) - assert '[ "9.9.9.9" ]' in output.stdout + assert '[ 9.9.9.9 ]' in output.stdout From 001024b4da4be7de189c0ae3b7dde2cbe6aa1ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 1 Jun 2023 20:41:54 +0200 Subject: [PATCH 033/200] Fix Tests No2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/test_any_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 1efd77c3..154b5e16 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -151,4 +151,4 @@ def test_getFTLConfigValue_getFTLConfigValue(host): """ ) - assert '[ 9.9.9.9 ]' in output.stdout + assert "[ 9.9.9.9 ]" in output.stdout From a4322c624f5047123c5b8806f7e9222bfa89568f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 1 Jun 2023 22:00:40 +0200 Subject: [PATCH 034/200] Fix gravity in tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/test_any_automated_install.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 3626ce6f..c1ef8af3 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -79,6 +79,11 @@ def test_installPihole_fresh_install_readableFiles(host): host.run("command -v apt-get > /dev/null && apt-get install -qq man") host.run("command -v dnf > /dev/null && dnf install -y man") host.run("command -v yum > /dev/null && yum install -y man") + # Workaround to get FTLv6 installed until it reaches master branch + host.run(""" + echo "new/http" > /etc/pihole/ftlbranch + """ + ) install = host.run( """ export TERM=xterm From 0becc7615a74e50630a153030e4d0fa7aba87e9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 1 Jun 2023 23:51:03 +0200 Subject: [PATCH 035/200] Run prestart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/test_any_automated_install.py | 7 ++++--- test/test_any_utils.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index c1ef8af3..df9c4721 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -64,7 +64,7 @@ def test_installPihole_fresh_install_readableFiles(host): mock_command("dialog", {"*": ("", "0")}, host) # mock git pull mock_command_passthrough("git", {"pull": ("", "0")}, host) - # mock systemctl to not start lighttpd and FTL + # mock systemctl to not start FTL mock_command_2( "systemctl", { @@ -95,6 +95,7 @@ def test_installPihole_fresh_install_readableFiles(host): runUnattended=true useUpdateVars=true main + /opt/pihole/pihole-FTL-prestart.sh """ ) assert 0 == install.rc @@ -507,9 +508,9 @@ def test_FTL_download_aarch64_no_errors(host): assert "error" not in download_binary.stdout.lower() -def test_FTL_binary_installed_and_responsive_no_errors(host): +def test_FTL_development_binary_installed_and_responsive_no_errors(host): """ - confirms FTL binary is copied and functional in installed location + confirms FTL development binary is copied and functional in installed location """ host.run( """ diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 154b5e16..8c1ea521 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -129,7 +129,7 @@ def test_getFTLPIDFile_and_getFTLPID_custom(host): def test_getFTLConfigValue_getFTLConfigValue(host): """ Confirms getFTLConfigValue works (also assumes setFTLConfigValue works) - Requires FTL to be installed, so we do that first (taken from test_FTL_binary_installed_and_responsive_no_errors) + Requires FTL to be installed, so we do that first (taken from test_FTL_development_binary_installed_and_responsive_no_errors) """ host.run( """ From 7da57c6acddf6d77610b934f7902e134ee9ef377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 2 Jun 2023 00:03:22 +0200 Subject: [PATCH 036/200] Don't check and install old FTL config file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 10 ---------- test/test_any_automated_install.py | 12 +++--------- test/test_any_utils.py | 3 ++- 3 files changed, 5 insertions(+), 20 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 6663bbad..37e97e90 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -78,7 +78,6 @@ PI_HOLE_FILES=(chronometer list piholeDebug piholeLogFlush setupLCD update versi PI_HOLE_INSTALL_DIR="/opt/pihole" PI_HOLE_CONFIG_DIR="/etc/pihole" PI_HOLE_BIN_DIR="/usr/local/bin" -FTL_CONFIG_FILE="${PI_HOLE_CONFIG_DIR}/pihole-FTL.conf" if [ -z "$useUpdateVars" ]; then useUpdateVars=false fi @@ -1192,15 +1191,6 @@ installConfigs() { echo "${DNS_SERVERS}" > "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" chmod 644 "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" - # Install template file if it does not exist - if [[ ! -r "${FTL_CONFIG_FILE}" ]]; then - install -d -m 0755 ${PI_HOLE_CONFIG_DIR} - if ! install -T -o pihole -m 664 "${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole-FTL.conf" "${FTL_CONFIG_FILE}" &>/dev/null; then - printf " %b Error: Unable to initialize configuration file %s/pihole-FTL.conf\\n" "${COL_LIGHT_RED}" "${PI_HOLE_CONFIG_DIR}" - return 1 - fi - fi - # Install empty custom.list file if it does not exist if [[ ! -r "${PI_HOLE_CONFIG_DIR}/custom.list" ]]; then if ! install -o root -m 644 /dev/null "${PI_HOLE_CONFIG_DIR}/custom.list" &>/dev/null; then diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index df9c4721..d5f76ba3 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -80,9 +80,10 @@ def test_installPihole_fresh_install_readableFiles(host): host.run("command -v dnf > /dev/null && dnf install -y man") host.run("command -v yum > /dev/null && yum install -y man") # Workaround to get FTLv6 installed until it reaches master branch - host.run(""" - echo "new/http" > /etc/pihole/ftlbranch + host.run( """ + echo "new/http" > /etc/pihole/ftlbranch + """ ) install = host.run( """ @@ -141,13 +142,6 @@ def test_installPihole_fresh_install_readableFiles(host): check_macvendor = test_cmd.format("r", "/etc/pihole/macvendor.db", piholeuser) actual_rc = host.run(check_macvendor).rc assert exit_status_success == actual_rc - # readable and writeable pihole-FTL.conf - check_FTLconf = test_cmd.format("r", "/etc/pihole/pihole-FTL.conf", piholeuser) - actual_rc = host.run(check_FTLconf).rc - assert exit_status_success == actual_rc - check_FTLconf = test_cmd.format("w", "/etc/pihole/pihole-FTL.conf", piholeuser) - actual_rc = host.run(check_FTLconf).rc - assert exit_status_success == actual_rc # check readable and executable /etc/init.d/pihole-FTL check_init = test_cmd.format("x", "/etc/init.d/pihole-FTL", piholeuser) actual_rc = host.run(check_init).rc diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 8c1ea521..0f300457 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -129,7 +129,8 @@ def test_getFTLPIDFile_and_getFTLPID_custom(host): def test_getFTLConfigValue_getFTLConfigValue(host): """ Confirms getFTLConfigValue works (also assumes setFTLConfigValue works) - Requires FTL to be installed, so we do that first (taken from test_FTL_development_binary_installed_and_responsive_no_errors) + Requires FTL to be installed, so we do that first + (taken from test_FTL_development_binary_installed_and_responsive_no_errors) """ host.run( """ From ec82aec55fab2b0e751b3ff181d8aab8b653feab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 2 Jun 2023 00:13:57 +0200 Subject: [PATCH 037/200] centos_common_support.py was removed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/tox.fedora_38.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tox.fedora_38.ini b/test/tox.fedora_38.ini index 0aa7612e..d596092c 100644 --- a/test/tox.fedora_38.ini +++ b/test/tox.fedora_38.ini @@ -5,4 +5,4 @@ envlist = py3 allowlist_externals = docker deps = -rrequirements.txt commands = docker buildx build --load --progress plain -f _fedora_38.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py From 667418c71d0b29c927eb0c16eea4c4dadfd974a8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 31 May 2023 02:22:53 +0200 Subject: [PATCH 038/200] Use new FTL binary names Signed-off-by: DL6ER --- automated install/basic-install.sh | 46 ++++++++++++++---------------- test/test_any_automated_install.py | 24 +++++----------- 2 files changed, 29 insertions(+), 41 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 37e97e90..681b6798 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1951,39 +1951,29 @@ get_binary_name() { if [[ "${lib}" == "/lib/ld-linux-aarch64.so.1" ]]; then printf "%b %b Detected AArch64 (64 Bit ARM) processor\\n" "${OVER}" "${TICK}" # set the binary to be used - l_binary="pihole-FTL-aarch64-linux-gnu" + l_binary="pihole-FTL-arm64" elif [[ "${lib}" == "/lib/ld-linux-armhf.so.3" ]]; then # Hard-float available: Use gnueabihf binaries # If ARMv8 or higher is found (e.g., BCM2837 as found in Raspberry Pi Model 3B) if [[ "${rev}" -gt 7 ]]; then printf "%b %b Detected ARMv8 (or newer) processor\\n" "${OVER}" "${TICK}" # set the binary to be used - l_binary="pihole-FTL-armv8-linux-gnueabihf" + l_binary="pihole-FTL-armv8" elif [[ "${rev}" -eq 7 ]]; then # Otherwise, if ARMv7 is found (e.g., BCM2836 as found in Raspberry Pi Model 2) printf "%b %b Detected ARMv7 processor (with hard-float support)\\n" "${OVER}" "${TICK}" # set the binary to be used - l_binary="pihole-FTL-armv7-linux-gnueabihf" + l_binary="pihole-FTL-armv7" else # Otherwise, use the ARMv6 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) printf "%b %b Detected ARMv6 processor (with hard-float support)\\n" "${OVER}" "${TICK}" # set the binary to be used - l_binary="pihole-FTL-armv6-linux-gnueabihf" + l_binary="pihole-FTL-armv6" fi else - # No hard-float support found: Use gnueabi binaries - # Use the ARMv4-compliant binary only if we detected an ARMv4T core - if [[ "${rev}" -eq 4 ]]; then - printf "%b %b Detected ARMv4 processor\\n" "${OVER}" "${TICK}" - # set the binary to be used - l_binary="pihole-FTL-armv4-linux-gnueabi" - # Otherwise, use the ARMv5 binary. To date (end of 2020), all modern ARM processors - # are backwards-compatible to the ARMv5 - else - printf "%b %b Detected ARMv5 (or newer) processor\\n" "${OVER}" "${TICK}" - # set the binary to be used - l_binary="pihole-FTL-armv5-linux-gnueabi" - fi + # No hard-float support found + printf "%b %b%b ARM processor without hard-float support detected%b\\n" "${OVER}" "${COL_LIGHT_RED}" "${CROSS}" "${COL_NC}" + l_binary="" fi elif [[ "${machine}" == "x86_64" ]]; then # This gives the processor of packages dpkg installs (for example, "i386") @@ -1996,16 +1986,16 @@ get_binary_name() { # in the past (see https://github.com/pi-hole/pi-hole/pull/2004) if [[ "${dpkgarch}" == "i386" ]]; then printf "%b %b Detected 32bit (i686) processor\\n" "${OVER}" "${TICK}" - l_binary="pihole-FTL-linux-x86_32" + l_binary="pihole-FTL-386" else # 64bit printf "%b %b Detected x86_64 processor\\n" "${OVER}" "${TICK}" # set the binary to be used - l_binary="pihole-FTL-linux-x86_64" + l_binary="pihole-FTL-amd64" fi elif [[ "${machine}" == "riscv64" ]]; then printf "%b %b Detected riscv64 processor\\n" "${OVER}" "${TICK}" - l_binary="pihole-FTL-riscv64-linux-gnu" + l_binary="pihole-FTL-riscv64" else # Something else - we try to use 32bit executable and warn the user if [[ ! "${machine}" == "i686" ]]; then @@ -2015,7 +2005,7 @@ get_binary_name() { else printf "%b %b Detected 32bit (i686) processor\\n" "${OVER}" "${TICK}" fi - l_binary="pihole-FTL-linux-x86_32" + l_binary="pihole-FTL-linux-386" fi # Returning a string value via echo @@ -2180,6 +2170,16 @@ main() { fi fi + # Check if there is a usable FTL binary available on this architecture - do + # this early on as FTL is a hard dependency for Pi-hole + local funcOutput + funcOutput=$(get_binary_name) #Store output of get_binary_name here + # Abort early if this processor is not supported (get_binary_name returnS empty string) + if [[ "${funcOutput}" == "" ]]; then + printf " %b Upgrade/install aborted\\n" "${CROSS}" "${DISTRO_NAME}" + exit 1 + fi + # Check if SELinux is Enforcing and exit before doing anything else checkSelinux @@ -2249,9 +2249,7 @@ main() { # Create the pihole user create_pihole_user - # Check if FTL is installed - do this early on as FTL is a hard dependency for Pi-hole - local funcOutput - funcOutput=$(get_binary_name) #Store output of get_binary_name here + # Download and install FTL local binary binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL) local theRest diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index d5f76ba3..4a86d28e 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -268,9 +268,9 @@ def test_FTL_detect_aarch64_no_errors(host): assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv4t_no_errors(host): +def test_FTL_detect_armv4t_no_install(host): """ - confirms only armv4t package is downloaded for FTL engine + confirms armv4t architecture is not supported """ # mock uname to return armv4t platform mock_command("uname", {"-m": ("armv4t", "0")}, host) @@ -288,22 +288,17 @@ def test_FTL_detect_armv4t_no_errors(host): source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" """ ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv4 processor") - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" + expected_stdout = cross_box + (" ARM processor without hard-float support detected") assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv5te_no_errors(host): +def test_FTL_detect_armv5te_no_install(host): """ - confirms only armv5te package is downloaded for FTL engine + confirms armv5te architecture is not supported """ # mock uname to return armv5te platform mock_command("uname", {"-m": ("armv5te", "0")}, host) @@ -321,16 +316,11 @@ def test_FTL_detect_armv5te_no_errors(host): source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" """ ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv5 (or newer) processor") - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" + expected_stdout = cross_box + (" ARM processor without hard-float support detected") assert expected_stdout in detectPlatform.stdout @@ -375,7 +365,7 @@ def test_FTL_detect_armv7l_no_errors(host): """ # mock uname to return armv7l platform mock_command("uname", {"-m": ("armv7l", "0")}, host) - # mock ldd to respond with ld-linux-armhf shared library + # mock ldd to respond with ld-linux-armhf shared lib rary mock_command( "ldd", { From 366345e87e6dc94294ea746e8cea8f781864a9e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 4 Jun 2023 20:20:18 +0200 Subject: [PATCH 039/200] Fix no_installs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 2 +- test/test_any_automated_install.py | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 681b6798..c4ea4764 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2174,7 +2174,7 @@ main() { # this early on as FTL is a hard dependency for Pi-hole local funcOutput funcOutput=$(get_binary_name) #Store output of get_binary_name here - # Abort early if this processor is not supported (get_binary_name returnS empty string) + # Abort early if this processor is not supported (get_binary_name returns empty string) if [[ "${funcOutput}" == "" ]]; then printf " %b Upgrade/install aborted\\n" "${CROSS}" "${DISTRO_NAME}" exit 1 diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 4a86d28e..5c902c2b 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -286,12 +286,9 @@ def test_FTL_detect_armv4t_no_install(host): detectPlatform = host.run( """ source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) + get_binary_name """ ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout expected_stdout = cross_box + (" ARM processor without hard-float support detected") assert expected_stdout in detectPlatform.stdout @@ -314,12 +311,9 @@ def test_FTL_detect_armv5te_no_install(host): detectPlatform = host.run( """ source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) + get_binary_name """ ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout expected_stdout = cross_box + (" ARM processor without hard-float support detected") assert expected_stdout in detectPlatform.stdout From 53fb7ae0ef1b4eefcbf77ed2b4fbaed9a12c55ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 4 Jun 2023 21:06:44 +0200 Subject: [PATCH 040/200] Use shellcheck in smoke-tests. RIP stickler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fca8bb19..d5037f37 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,6 +23,13 @@ jobs: # If FAIL is 1 then we fail. [[ $FAIL == 1 ]] && exit 1 || echo "Scripts are executable!" + - name: Run shellcheck + uses: ludeeus/action-shellcheck@master + with: + check_together: 'yes' + format: tty + severity: error + - name: Spell-Checking uses: codespell-project/actions-codespell@master with: From ba3e290915a41dbe6a8b885e00ba0a796376fb58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 4 Jun 2023 21:21:48 +0200 Subject: [PATCH 041/200] Fix shellcheck errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeLogFlush.sh | 6 +++--- pihole | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 3473fad5..14542e4b 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -30,10 +30,10 @@ if [ -z "$DBFILE" ]; then DBFILE="/etc/pihole/pihole-FTL.db" fi -if [[ "$@" != *"quiet"* ]]; then +if [[ "$*" != *"quiet"* ]]; then echo -ne " ${INFO} Flushing /var/log/pihole/pihole.log ..." fi -if [[ "$@" == *"once"* ]]; then +if [[ "$*" == *"once"* ]]; then # Nightly logrotation if command -v /usr/sbin/logrotate >/dev/null; then # Logrotate once @@ -69,7 +69,7 @@ else sudo pihole restartdns fi -if [[ "$@" != *"quiet"* ]]; then +if [[ "$*" != *"quiet"* ]]; then echo -e "${OVER} ${TICK} Flushed /var/log/pihole/pihole.log" echo -e " ${TICK} Deleted ${deleted} queries from database" fi diff --git a/pihole b/pihole index 0be995e4..b762c7d4 100755 --- a/pihole +++ b/pihole @@ -196,7 +196,7 @@ Time: echo -e " ${INFO} Blocking already disabled, nothing to do" exit 0 fi - if [[ $# > 1 ]]; then + if [[ $# -gt 1 ]]; then local error=false if [[ "${2}" == *"s" ]]; then tt=${2%"s"} From 088b2c29205c8da2145de664cf2b0978e6fcba79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 1 Jun 2023 15:10:05 +0200 Subject: [PATCH 042/200] Do not remove -all|excat when not sourrounded by space in query.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 1d3b0a29..bfa21247 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -50,7 +50,7 @@ fi # Strip valid options, leaving only the domain and invalid options # This allows users to place the options before or after the domain -options=$(sed -E 's/ ?-(all|exact) ?//g' <<< "${options}") +options=$(sed -E 's/ +-(all|exact) ?//g' <<< "${options}") # Handle remaining options # If $options contain non ASCII characters, convert to punycode From 8bf4ab0cd6ae133661e980a7cd067d23f2bc6b22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 7 Jan 2023 21:41:12 +0100 Subject: [PATCH 043/200] Remove option to set static IP address if DHCPCD is installed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 127 ----------------------------- 1 file changed, 127 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index c4ea4764..d49ae012 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -500,9 +500,6 @@ find_IPv4_information() { # disabled as we intentionally want to split on whitespace and have printf populate # the variable with just the first field. printf -v IPv4bare "$(printf ${route#*src })" - # Get the default gateway IPv4 address (the way to reach the Internet) - # shellcheck disable=SC2059,SC2086 - printf -v IPv4gw "$(printf ${route#*via })" if ! valid_ip "${IPv4bare}" ; then IPv4bare="127.0.0.1" @@ -663,135 +660,11 @@ find_IPv6_information() { # A function to collect IPv4 and IPv6 information of the device collect_v4andv6_information() { find_IPv4_information - # Echo the information to the user printf " %b IPv4 address: %s\\n" "${INFO}" "${IPV4_ADDRESS}" - # if `dhcpcd` is used offer to set this as static IP for the device - if [[ -f "/etc/dhcpcd.conf" ]]; then - # configure networking via dhcpcd - getStaticIPv4Settings - fi find_IPv6_information printf " %b IPv6 address: %s\\n" "${INFO}" "${IPV6_ADDRESS}" } -getStaticIPv4Settings() { - # Local, named variables - local ipSettingsCorrect - local DHCPChoice - # Ask if the user wants to use DHCP settings as their static IP - # This is useful for users that are using DHCP reservations; we can use the information gathered - DHCPChoice=$(dialog --no-shadow --keep-tite --output-fd 1 \ - --cancel-label "Exit" --ok-label "Continue" \ - --backtitle "Calibrating network interface" \ - --title "Static IP Address" \ - --menu "Do you want to use your current network settings as a static address?\\n \ - IP address: ${IPV4_ADDRESS}\\n \ - Gateway: ${IPv4gw}\\n" \ - "${r}" "${c}" 3 \ - "Yes" "Set static IP using current values" \ - "No" "Set static IP using custom values" \ - "Skip" "I will set a static IP later, or have already done so") - - result=$? - case ${result} in - "${DIALOG_CANCEL}" | "${DIALOG_ESC}") - printf " %b Cancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}" - exit 1 - ;; - esac - - case ${DHCPChoice} in - "Skip") - return - ;; - "Yes") - # If they choose yes, let the user know that the IP address will not be available via DHCP and may cause a conflict. - dialog --no-shadow --keep-tite \ - --cancel-label "Exit" \ - --backtitle "IP information" \ - --title "FYI: IP Conflict" \ - --msgbox "\\nIt is possible your router could still try to assign this IP to a device, which would cause a conflict, \ -but in most cases the router is smart enough to not do that.\n\n\ -If you are worried, either manually set the address, or modify the DHCP reservation pool so it does not include the IP you want.\n\n\ -It is also possible to use a DHCP reservation, but if you are going to do that, you might as well set a static address."\ - "${r}" "${c}" && result=0 || result=$? - - case ${result} in - "${DIALOG_CANCEL}" | "${DIALOG_ESC}") - printf " %b Cancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}" - exit 1 - ;; - esac - ;; - - "No") - # Otherwise, we need to ask the user to input their desired settings. - # Start by getting the IPv4 address (pre-filling it with info gathered from DHCP) - # Start a loop to let the user enter their information with the chance to go back and edit it if necessary - ipSettingsCorrect=false - until [[ "${ipSettingsCorrect}" = True ]]; do - - # Ask for the IPv4 address - _staticIPv4Temp=$(dialog --no-shadow --keep-tite --output-fd 1 \ - --cancel-label "Exit" \ - --ok-label "Continue" \ - --backtitle "Calibrating network interface" \ - --title "IPv4 Address" \ - --form "\\nEnter your desired IPv4 address" \ - "${r}" "${c}" 0 \ - "IPv4 Address:" 1 1 "${IPV4_ADDRESS}" 1 15 19 0 \ - "IPv4 Gateway:" 2 1 "${IPv4gw}" 2 15 19 0) - - result=$? - case ${result} in - "${DIALOG_CANCEL}" | "${DIALOG_ESC}") - printf " %b Cancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}" - exit 1 - ;; - esac - - IPV4_ADDRESS=${_staticIPv4Temp%$'\n'*} - IPv4gw=${_staticIPv4Temp#*$'\n'} - - # Give the user a chance to review their settings before moving on - dialog --no-shadow --keep-tite \ - --no-label "Edit IP" \ - --backtitle "Calibrating network interface" \ - --title "Static IP Address" \ - --defaultno \ - --yesno "Are these settings correct? - IP address: ${IPV4_ADDRESS} - Gateway: ${IPv4gw}" \ - "${r}" "${c}" && ipSettingsCorrect=True - done - ;; - esac - setDHCPCD -} - -# Configure networking via dhcpcd -setDHCPCD() { - # Regex for matching a non-commented static ip address setting - local regex="^[ \t]*static ip_address[ \t]*=[ \t]*${IPV4_ADDRESS}" - - # Check if static IP is already set in file - if grep -q "${regex}" /etc/dhcpcd.conf; then - printf " %b Static IP already configured\\n" "${INFO}" - # If it's not, - else - # we can append these lines to dhcpcd.conf to enable a static IP - echo "interface ${PIHOLE_INTERFACE} - static ip_address=${IPV4_ADDRESS} - static routers=${IPv4gw} - static domain_name_servers=${PIHOLE_DNS_1} ${PIHOLE_DNS_2}" | tee -a /etc/dhcpcd.conf >/dev/null - # Then use the ip command to immediately set the new address - ip addr replace dev "${PIHOLE_INTERFACE}" "${IPV4_ADDRESS}" - # Also give a warning that the user may need to reboot their system - printf " %b Set IP address to %s\\n" "${TICK}" "${IPV4_ADDRESS%/*}" - printf " %b You may need to restart after the install is complete\\n" "${INFO}" - fi -} - # Check an IP address to see if it is a valid one valid_ip() { # Local, named variables From 9c27e4766dede7a69b65cc412125ac8186d84852 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 8 Jun 2023 14:10:04 +0200 Subject: [PATCH 044/200] Decide which FTL architecture to pick by by installed architectures rather than by detected processor (this may be wrong in an emulated Docker environment) Signed-off-by: DL6ER --- automated install/basic-install.sh | 74 ++++++++---- test/test_any_automated_install.py | 187 +++++++++++++++++++++-------- 2 files changed, 184 insertions(+), 77 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index d49ae012..b14546d1 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1806,10 +1806,19 @@ FTLinstall() { } get_binary_name() { - # This gives the machine architecture which may be different from the OS architecture... + # Get the OS architecture (we cannot use uname -m as this may return an incorrect architecture when buildx-compiling with QEMU for arm) local machine machine=$(uname -m) + # Get local GLIBC version (leave at "0.0" if no GLIBC, e.g., on musl) + local l_glibc_version="0.0" + if ldd --version 2>&1 | grep -q "GLIBC"; then + l_glibc_version=$(ldd --version | head -n1 | grep -o '[0-9.]*$') + printf "%b %b Detected GLIBC version %s\\n" "${OVER}" "${TICK}" "${l_glibc_version}" + else + printf "%b %b No GLIBC detected\\n" "${OVER}" "${CROSS}" + fi + local l_binary local str="Detecting processor" @@ -1817,36 +1826,51 @@ get_binary_name() { # If the machine is arm or aarch if [[ "${machine}" == "arm"* || "${machine}" == *"aarch"* ]]; then # ARM + # Get supported processor from other binaries installed on the system + local cpu_arch + cpu_arch=$(readelf -A "$(command -v sh)" | grep Tag_CPU_arch | awk '{ print $2 }') + + # Get the revision from the CPU architecture local rev - rev=$(uname -m | sed "s/[^0-9]//g;") - local lib - lib=$(ldd "$(command -v sh)" | grep -E '^\s*/lib' | awk '{ print $1 }') - if [[ "${lib}" == "/lib/ld-linux-aarch64.so.1" ]]; then - printf "%b %b Detected AArch64 (64 Bit ARM) processor\\n" "${OVER}" "${TICK}" + rev=$(echo "${cpu_arch}" | grep -o '[0-9]*') + if [[ "${machine}" == "aarch64" ]]; then + printf "%b %b Detected AArch64 (64 Bit ARM) architecture\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-arm64" - elif [[ "${lib}" == "/lib/ld-linux-armhf.so.3" ]]; then - # Hard-float available: Use gnueabihf binaries + elif [[ "${cpu_arch}" == "armv6KZ" ]]; then + printf "%b %b Detected ARMv6KZ architecture\\n" "${OVER}" "${TICK}" + # set the binary to be used + l_binary="pihole-FTL-armv6" + else # If ARMv8 or higher is found (e.g., BCM2837 as found in Raspberry Pi Model 3B) if [[ "${rev}" -gt 7 ]]; then - printf "%b %b Detected ARMv8 (or newer) processor\\n" "${OVER}" "${TICK}" + printf "%b %b Detected ARMv8 (or newer) architecture\\n" "${OVER}" "${TICK}" # set the binary to be used - l_binary="pihole-FTL-armv8" - elif [[ "${rev}" -eq 7 ]]; then + l_binary="pihole-FTL-armv7" + elif [[ "${rev}" -gt 6 ]]; then # Otherwise, if ARMv7 is found (e.g., BCM2836 as found in Raspberry Pi Model 2) - printf "%b %b Detected ARMv7 processor (with hard-float support)\\n" "${OVER}" "${TICK}" + printf "%b %b Detected ARMv7 architecture\\n" "${OVER}" "${TICK}" # set the binary to be used - l_binary="pihole-FTL-armv7" + l_binary="pihole-FTL-armv6" + elif [[ "${rev}" -gt 5 ]]; then + # Check if the system is using GLIBC 2.29 or higher + if [[ -n "${l_glibc_version}" && "$(printf '%s\n' "2.29" "${l_glibc_version}" | sort -V | head -n1)" == "2.29" ]]; then + # If so, use the ARMv6 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) + printf "%b %b Detected ARMv6 architecture (running GLIBC 2.29 or higher)\\n" "${OVER}" "${TICK}" + # set the binary to be used + l_binary="pihole-FTL-armv5" + else + # Otherwise, use the ARMv5 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) + printf "%b %b Detected ARMv6 architecture (running GLIBC older than 2.29)\\n" "${OVER}" "${TICK}" + # set the binary to be used + l_binary="pihole-FTL-armv4" + fi else - # Otherwise, use the ARMv6 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv6 processor (with hard-float support)\\n" "${OVER}" "${TICK}" + # Otherwise, use the ARMv4 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) + printf "%b %b Detected ARMv4 or ARMv5 architecture\\n" "${OVER}" "${TICK}" # set the binary to be used - l_binary="pihole-FTL-armv6" + l_binary="pihole-FTL-armv4" fi - else - # No hard-float support found - printf "%b %b%b ARM processor without hard-float support detected%b\\n" "${OVER}" "${COL_LIGHT_RED}" "${CROSS}" "${COL_NC}" - l_binary="" fi elif [[ "${machine}" == "x86_64" ]]; then # This gives the processor of packages dpkg installs (for example, "i386") @@ -1858,25 +1882,25 @@ get_binary_name() { # We only check this for Debian-based systems as this has been an issue # in the past (see https://github.com/pi-hole/pi-hole/pull/2004) if [[ "${dpkgarch}" == "i386" ]]; then - printf "%b %b Detected 32bit (i686) processor\\n" "${OVER}" "${TICK}" + printf "%b %b Detected 32bit (i686) architecture\\n" "${OVER}" "${TICK}" l_binary="pihole-FTL-386" else # 64bit - printf "%b %b Detected x86_64 processor\\n" "${OVER}" "${TICK}" + printf "%b %b Detected x86_64 architecture\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-amd64" fi elif [[ "${machine}" == "riscv64" ]]; then - printf "%b %b Detected riscv64 processor\\n" "${OVER}" "${TICK}" + printf "%b %b Detected riscv64 architecture\\n" "${OVER}" "${TICK}" l_binary="pihole-FTL-riscv64" else # Something else - we try to use 32bit executable and warn the user if [[ ! "${machine}" == "i686" ]]; then printf "%b %b %s...\\n" "${OVER}" "${CROSS}" "${str}" - printf " %b %bNot able to detect processor (unknown: %s), trying x86 (32bit) executable%b\\n" "${INFO}" "${COL_LIGHT_RED}" "${machine}" "${COL_NC}" + printf " %b %bNot able to detect architecture (unknown: %s), trying x86 (32bit) executable%b\\n" "${INFO}" "${COL_LIGHT_RED}" "${machine}" "${COL_NC}" printf " %b Contact Pi-hole Support if you experience issues (e.g: FTL not running)\\n" "${INFO}" else - printf "%b %b Detected 32bit (i686) processor\\n" "${OVER}" "${TICK}" + printf "%b %b Detected 32bit (i686) architecture\\n" "${OVER}" "${TICK}" fi l_binary="pihole-FTL-linux-386" fi diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 5c902c2b..4431c565 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -241,15 +241,6 @@ def test_FTL_detect_aarch64_no_errors(host): """ # mock uname to return aarch64 platform mock_command("uname", {"-m": ("aarch64", "0")}, host) - # mock ldd to respond with aarch64 shared library - mock_command( - "ldd", - { - "/bin/sh": ("/lib/ld-linux-aarch64.so.1", "0"), - "/usr/bin/sh": ("/lib/ld-linux-aarch64.so.1", "0"), - }, - host, - ) detectPlatform = host.run( """ source /opt/pihole/basic-install.sh @@ -262,77 +253,138 @@ def test_FTL_detect_aarch64_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected AArch64 (64 Bit ARM) processor" + expected_stdout = tick_box + " Detected AArch64 (64 Bit ARM) architecture" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv4t_no_install(host): +def test_FTL_detect_armv4_no_errors(host): """ - confirms armv4t architecture is not supported + confirms only armv4 package is downloaded for FTL engine """ - # mock uname to return armv4t platform + # mock uname to return armv4 platform mock_command("uname", {"-m": ("armv4t", "0")}, host) - # mock ldd to respond with armv4t shared library - mock_command( - "ldd", + # mock readelf to respond with armv4 CPU architecture + mock_command_2( + "readelf", { - "/bin/sh": ("/lib/ld-linux.so.3", "0"), - "/usr/bin/sh": ("/lib/ld-linux.so.3", "0"), + "-A /bin/sh": ("Tag_CPU_arch: armv4t", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv4t", "0"), }, host, ) detectPlatform = host.run( """ source /opt/pihole/basic-install.sh - get_binary_name + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" """ ) - expected_stdout = cross_box + (" ARM processor without hard-float support detected") + expected_stdout = info_box + " FTL Checks..." + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture" + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv5te_no_install(host): +def test_FTL_detect_armv5_no_errors(host): """ - confirms armv5te architecture is not supported + confirms only armv5 package is downloaded for FTL engine """ # mock uname to return armv5te platform mock_command("uname", {"-m": ("armv5te", "0")}, host) - # mock ldd to respond with ld-linux shared library - mock_command( - "ldd", + # mock readelf to respond with armv5 CPU architecture + mock_command_2( + "readelf", { - "/bin/sh": ("/lib/ld-linux.so.3", "0"), - "/usr/bin/sh": ("/lib/ld-linux.so.3", "0"), + "-A /bin/sh": ("Tag_CPU_arch: armv5te", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv5te", "0"), }, host, ) detectPlatform = host.run( """ source /opt/pihole/basic-install.sh - get_binary_name + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" """ ) - expected_stdout = cross_box + (" ARM processor without hard-float support detected") + expected_stdout = info_box + " FTL Checks..." + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture" + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv6l_no_errors(host): +def test_FTL_detect_armv6_old_no_errors(host): """ - confirms only armv6l package is downloaded for FTL engine + confirms only armv6 package is downloaded for FTL engine """ # mock uname to return armv6l platform mock_command("uname", {"-m": ("armv6l", "0")}, host) - # mock ldd to respond with ld-linux-armhf shared library + # mock readelf to respond with armv6l CPU architecture + mock_command_2( + "readelf", + { + "-A /bin/sh": ("Tag_CPU_arch: armv6l", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv6l", "0"), + }, + host, + ) + # Mock old ldd GLIBC version mock_command( - "ldd", + "ldd", {"--version": ("ldd (Debian GLIBC 2.13-38+deb7u8) 2.13", "0")}, host + ) + + detectPlatform = host.run( + """ + source /opt/pihole/basic-install.sh + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" + """ + ) + expected_stdout = info_box + " FTL Checks..." + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + ( + " Detected ARMv6 architecture (running GLIBC older than 2.29)" + ) + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Downloading and Installing FTL" + assert expected_stdout in detectPlatform.stdout + + +def test_FTL_detect_armv6_recent_no_errors(host): + """ + confirms only armv6 package is downloaded for FTL engine + """ + # mock uname to return armv6l platform + mock_command("uname", {"-m": ("armv6l", "0")}, host) + # mock readelf to respond with armv6l CPU architecture + mock_command_2( + "readelf", { - "/bin/sh": ("/lib/ld-linux-armhf.so.3", "0"), - "/usr/bin/sh": ("/lib/ld-linux-armhf.so.3", "0"), + "-A /bin/sh": ("Tag_CPU_arch: armv6l", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv6l", "0"), }, host, ) + # Mock old ldd GLIBC version + mock_command( + "ldd", {"--version": ("'ldd (Debian GLIBC 2.35-38+deb7u8) 2.35'", "0")}, host + ) + detectPlatform = host.run( """ source /opt/pihole/basic-install.sh @@ -346,25 +398,58 @@ def test_FTL_detect_armv6l_no_errors(host): expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + ( - " Detected ARMv6 processor " "(with hard-float support)" + " Detected ARMv6 architecture (running GLIBC 2.29 or higher)" ) assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout +def test_FTL_detect_armv6KZ_no_errors(host): + """ + confirms only armv6KZ package is downloaded for FTL engine + """ + # mock uname to return armv6KZ platform + mock_command("uname", {"-m": ("armv6KZ", "0")}, host) + # mock readelf to respond with armv6l CPU architecture + mock_command_2( + "readelf", + { + "-A /bin/sh": ("Tag_CPU_arch: armv6KZ", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv6KZ", "0"), + }, + host, + ) + detectPlatform = host.run( + """ + source /opt/pihole/basic-install.sh + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" + """ + ) + expected_stdout = info_box + " FTL Checks..." + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Detected ARMv6KZ architecture" + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Downloading and Installing FTL" + assert expected_stdout in detectPlatform.stdout + + def test_FTL_detect_armv7l_no_errors(host): """ confirms only armv7l package is downloaded for FTL engine """ # mock uname to return armv7l platform mock_command("uname", {"-m": ("armv7l", "0")}, host) - # mock ldd to respond with ld-linux-armhf shared lib rary - mock_command( - "ldd", + # mock readelf to respond with armv7l CPU architecture + mock_command_2( + "readelf", { - "/bin/sh": ("/lib/ld-linux-armhf.so.3", "0"), - "/usr/bin/sh": ("/lib/ld-linux-armhf.so.3", "0"), + "-A /bin/sh": ("Tag_CPU_arch: armv7l", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv7l", "0"), }, host, ) @@ -380,9 +465,7 @@ def test_FTL_detect_armv7l_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + ( - " Detected ARMv7 processor " "(with hard-float support)" - ) + expected_stdout = tick_box + (" Detected ARMv7 architecture") assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -394,12 +477,12 @@ def test_FTL_detect_armv8a_no_errors(host): """ # mock uname to return armv8a platform mock_command("uname", {"-m": ("armv8a", "0")}, host) - # mock ldd to respond with ld-linux-armhf shared library - mock_command( - "ldd", + # mock readelf to respond with armv8a CPU architecture + mock_command_2( + "readelf", { - "/bin/sh": ("/lib/ld-linux-armhf.so.3", "0"), - "/usr/bin/sh": ("/lib/ld-linux-armhf.so.3", "0"), + "-A /bin/sh": ("Tag_CPU_arch: armv8a", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv8a", "0"), }, host, ) @@ -415,7 +498,7 @@ def test_FTL_detect_armv8a_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv8 (or newer) processor" + expected_stdout = tick_box + " Detected ARMv8 (or newer) architecture" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -437,7 +520,7 @@ def test_FTL_detect_x86_64_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected x86_64 processor" + expected_stdout = tick_box + " Detected x86_64 architecture" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -457,7 +540,7 @@ def test_FTL_detect_unknown_no_errors(host): FTLdetect "${binary}" "${theRest}" """ ) - expected_stdout = "Not able to detect processor (unknown: mips)" + expected_stdout = "Not able to detect architecture (unknown: mips)" assert expected_stdout in detectPlatform.stdout From 1c4e58efe3487264291761ec48a0dd0c7d61417e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 9 Jun 2023 15:53:02 +0200 Subject: [PATCH 045/200] Use exact architecture matching binaries where appropriate Signed-off-by: DL6ER --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index b14546d1..ed2d0037 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1843,7 +1843,7 @@ get_binary_name() { l_binary="pihole-FTL-armv6" else # If ARMv8 or higher is found (e.g., BCM2837 as found in Raspberry Pi Model 3B) - if [[ "${rev}" -gt 7 ]]; then + if [[ "${cpu_arch}" == "v7" || "${rev}" -gt 7 ]]; then printf "%b %b Detected ARMv8 (or newer) architecture\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-armv7" @@ -1852,7 +1852,7 @@ get_binary_name() { printf "%b %b Detected ARMv7 architecture\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-armv6" - elif [[ "${rev}" -gt 5 ]]; then + elif [[ "${cpu_arch}" == "v5TE" || "${rev}" -gt 5 ]]; then # Check if the system is using GLIBC 2.29 or higher if [[ -n "${l_glibc_version}" && "$(printf '%s\n' "2.29" "${l_glibc_version}" | sort -V | head -n1)" == "2.29" ]]; then # If so, use the ARMv6 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) From 913be6c3494db826ee0914034bb42d479c4ec8c9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 9 Jun 2023 18:50:31 +0200 Subject: [PATCH 046/200] Print detected CPU arch Signed-off-by: DL6ER --- automated install/basic-install.sh | 10 +++---- test/test_any_automated_install.py | 45 ++++++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ed2d0037..5a6be794 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1844,30 +1844,30 @@ get_binary_name() { else # If ARMv8 or higher is found (e.g., BCM2837 as found in Raspberry Pi Model 3B) if [[ "${cpu_arch}" == "v7" || "${rev}" -gt 7 ]]; then - printf "%b %b Detected ARMv8 (or newer) architecture\\n" "${OVER}" "${TICK}" + printf "%b %b Detected ARMv7 (or newer) architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" # set the binary to be used l_binary="pihole-FTL-armv7" elif [[ "${rev}" -gt 6 ]]; then # Otherwise, if ARMv7 is found (e.g., BCM2836 as found in Raspberry Pi Model 2) - printf "%b %b Detected ARMv7 architecture\\n" "${OVER}" "${TICK}" + printf "%b %b Detected ARMv7 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" # set the binary to be used l_binary="pihole-FTL-armv6" elif [[ "${cpu_arch}" == "v5TE" || "${rev}" -gt 5 ]]; then # Check if the system is using GLIBC 2.29 or higher if [[ -n "${l_glibc_version}" && "$(printf '%s\n' "2.29" "${l_glibc_version}" | sort -V | head -n1)" == "2.29" ]]; then # If so, use the ARMv6 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv6 architecture (running GLIBC 2.29 or higher)\\n" "${OVER}" "${TICK}" + printf "%b %b Detected ARMv6 architecture (running GLIBC 2.29 or higher, %s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" # set the binary to be used l_binary="pihole-FTL-armv5" else # Otherwise, use the ARMv5 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv6 architecture (running GLIBC older than 2.29)\\n" "${OVER}" "${TICK}" + printf "%b %b Detected ARMv6 architecture (running GLIBC older than 2.29, %s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" # set the binary to be used l_binary="pihole-FTL-armv4" fi else # Otherwise, use the ARMv4 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv4 or ARMv5 architecture\\n" "${OVER}" "${TICK}" + printf "%b %b Detected ARMv4 or ARMv5 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" # set the binary to be used l_binary="pihole-FTL-armv4" fi diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 4431c565..bc594304 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -286,7 +286,7 @@ def test_FTL_detect_armv4_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture" + expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture (armv4t)" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -319,7 +319,7 @@ def test_FTL_detect_armv5_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture" + expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture (armv5te)" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -358,7 +358,7 @@ def test_FTL_detect_armv6_old_no_errors(host): expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + ( - " Detected ARMv6 architecture (running GLIBC older than 2.29)" + " Detected ARMv6 architecture (running GLIBC older than 2.29, armv6l)" ) assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" @@ -398,7 +398,7 @@ def test_FTL_detect_armv6_recent_no_errors(host): expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + ( - " Detected ARMv6 architecture (running GLIBC 2.29 or higher)" + " Detected ARMv6 architecture (running GLIBC 2.29 or higher, armv6l)" ) assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" @@ -465,7 +465,40 @@ def test_FTL_detect_armv7l_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv7 architecture") + expected_stdout = tick_box + (" Detected ARMv7 architecture (armv7l)") + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Downloading and Installing FTL" + assert expected_stdout in detectPlatform.stdout + + +def test_FTL_detect_armv7_no_errors(host): + """ + confirms only armv7 package is downloaded for FTL engine + """ + # mock uname to return armv7 platform + mock_command("uname", {"-m": ("armv7", "0")}, host) + # mock readelf to respond with armv7 CPU architecture + mock_command_2( + "readelf", + { + "-A /bin/sh": ("Tag_CPU_arch: armv7", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv7", "0"), + }, + host, + ) + detectPlatform = host.run( + """ + source /opt/pihole/basic-install.sh + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" + """ + ) + expected_stdout = info_box + " FTL Checks..." + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + (" Detected ARMv7 architecture (armv7)") assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -498,7 +531,7 @@ def test_FTL_detect_armv8a_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv8 (or newer) architecture" + expected_stdout = tick_box + " Detected ARMv7 (or newer) architecture (armv8a)" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout From e0d7e5df8519883cf08b45500809ef1d04ee4160 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 9 Jun 2023 19:30:28 +0200 Subject: [PATCH 047/200] Install binutils as installer deps (needed to pick correct architecture for FTL) Signed-off-by: DL6ER --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 5a6be794..699fef10 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -304,7 +304,7 @@ package_manager_detect() { # Packages required to perform the os_check (stored as an array) OS_CHECK_DEPS=(grep dnsutils) # Packages required to run this install script (stored as an array) - INSTALLER_DEPS=(git iproute2 dialog ca-certificates) + INSTALLER_DEPS=(git iproute2 dialog ca-certificates binutils) # Packages required to run Pi-hole (stored as an array) PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) @@ -322,7 +322,7 @@ package_manager_detect() { # CentOS package manager returns 100 when there are packages to update so we need to || true to prevent the script from exiting. PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src|.riscv64)' | wc -l || true" OS_CHECK_DEPS=(grep bind-utils) - INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates) + INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates binutils) PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc libcap nmap-ncat jq) # If neither apt-get or yum/dnf package managers were found From 5459daa03daaaf89e18ac3da64082193c9c2b0c4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 15 Jun 2023 09:28:03 +0200 Subject: [PATCH 048/200] Add abp_entries column (default 0) to the adlist table for newly created gravity databases. This updates the gravity database version to 16 Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity-db.sh | 6 ++++++ .../Scripts/database_migration/gravity/15_to_16.sql | 11 +++++++++++ advanced/Templates/gravity.db.sql | 3 ++- 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 advanced/Scripts/database_migration/gravity/15_to_16.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index a7ba60a9..851ddb0a 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -128,4 +128,10 @@ upgrade_gravityDB(){ pihole-FTL sqlite3 "${database}" < "${scriptPath}/14_to_15.sql" version=15 fi + if [[ "$version" == "15" ]]; then + # Add column abp_entries to adlist table + echo -e " ${INFO} Upgrading gravity database from version 15 to 16" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/15_to_16.sql" + version=16 + fi } diff --git a/advanced/Scripts/database_migration/gravity/15_to_16.sql b/advanced/Scripts/database_migration/gravity/15_to_16.sql new file mode 100644 index 00000000..c6159f40 --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/15_to_16.sql @@ -0,0 +1,11 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +ALTER TABLE adlist ADD COLUMN abp_entries INTEGER NOT NULL DEFAULT 0; + +UPDATE info SET value = 16 WHERE property = 'version'; + +COMMIT; diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 3f696d6d..e0f1de10 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -35,7 +35,8 @@ CREATE TABLE adlist date_updated INTEGER, number INTEGER NOT NULL DEFAULT 0, invalid_domains INTEGER NOT NULL DEFAULT 0, - status INTEGER NOT NULL DEFAULT 0 + status INTEGER NOT NULL DEFAULT 0, + abp_entries INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE adlist_by_group From d5013bfd6c3153113fa8a67b6731c9d91a327e67 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 12 Jun 2023 22:33:50 +0100 Subject: [PATCH 049/200] Add code to remove old lighttpd config files left over from v5. Web config is all dealt with by FTL now Signed-off-by: Adam Warner --- automated install/basic-install.sh | 33 ++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 699fef10..9659a140 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1001,6 +1001,38 @@ remove_old_dnsmasq_ftl_configs() { fi } +remove_old_pihole_lighttpd_configs() { + local lighttpdConfig="/etc/lighttpd/lighttpd.conf" + local condfd="/etc/lighttpd/conf.d/pihole-admin.conf" + local confavailable="/etc/lighttpd/conf-available/15-pihole-admin.conf" + local confenabled="/etc/lighttpd/conf-enabled/15-pihole-admin.conf" + + + if [[ -d "/etc/lighttpd/conf.d" ]]; then + if grep -q -F 'include "/etc/lighttpd/conf.d/pihole-admin.conf"' "${lighttpdConfig}"; then + sed -i '/include "/etc/lighttpd/conf.d/pihole-admin.conf"/d' "${lighttpdConfig}" + fi + + if [[ -f "${condfd}" ]]; then + rm "${condfd}" + fi + + + elif [[ -d "/etc/lighttpd/conf-available" ]]; then + if is_command lighty-disable-mod ; then + lighty-disable-mod pihole-admin > /dev/null || true + fi + + if [[ -f "${confavailable}" ]]; then + rm "${confavailable}" + fi + + if [[ -f "${confenabled}" ]]; then + rm "${confenabled}" + fi + fi +} + # Clean an existing installation to prepare for upgrade/reinstall clean_existing() { # Local, named variables @@ -1486,6 +1518,7 @@ installPihole() { fi remove_old_dnsmasq_ftl_configs + remove_old_pihole_lighttpd_configs # Install config files if ! installConfigs; then From c39cb8cfe066d3520354f3cdee293fa6afe47e0e Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 12 Jun 2023 22:43:39 +0100 Subject: [PATCH 050/200] Escape the sed command for removing a line in fed/centos lighttpd.conf Signed-off-by: Adam Warner --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9659a140..43cd9816 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1010,7 +1010,7 @@ remove_old_pihole_lighttpd_configs() { if [[ -d "/etc/lighttpd/conf.d" ]]; then if grep -q -F 'include "/etc/lighttpd/conf.d/pihole-admin.conf"' "${lighttpdConfig}"; then - sed -i '/include "/etc/lighttpd/conf.d/pihole-admin.conf"/d' "${lighttpdConfig}" + sed -i '/include "\/etc\/lighttpd\/conf.d\/pihole-admin.conf"/d' "${lighttpdConfig}" fi if [[ -f "${condfd}" ]]; then From 001f2012a2b40c200e4c7d6384b8cd2f62c523c5 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 13 Jun 2023 19:08:12 +0100 Subject: [PATCH 051/200] Update automated install/basic-install.sh Co-authored-by: RD WebDesign Signed-off-by: Adam Warner --- automated install/basic-install.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 43cd9816..2edad6e3 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1009,9 +1009,7 @@ remove_old_pihole_lighttpd_configs() { if [[ -d "/etc/lighttpd/conf.d" ]]; then - if grep -q -F 'include "/etc/lighttpd/conf.d/pihole-admin.conf"' "${lighttpdConfig}"; then - sed -i '/include "\/etc\/lighttpd\/conf.d\/pihole-admin.conf"/d' "${lighttpdConfig}" - fi + sed -i '/include "\/etc\/lighttpd\/conf.d\/pihole-admin.conf"/d' "${lighttpdConfig}" if [[ -f "${condfd}" ]]; then rm "${condfd}" From d637d2a7a5f76fdea076d01d091e9e4865ac55f4 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 18 Jun 2023 12:38:02 +0100 Subject: [PATCH 052/200] Simplify nested if statements. Co-authored-by: yubiuser Signed-off-by: Adam Warner --- automated install/basic-install.sh | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 2edad6e3..e3d8ff29 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1008,27 +1008,25 @@ remove_old_pihole_lighttpd_configs() { local confenabled="/etc/lighttpd/conf-enabled/15-pihole-admin.conf" - if [[ -d "/etc/lighttpd/conf.d" ]]; then + if [[ -f "${lighttpdConfig}" ]]; then sed -i '/include "\/etc\/lighttpd\/conf.d\/pihole-admin.conf"/d' "${lighttpdConfig}" + fi - if [[ -f "${condfd}" ]]; then - rm "${condfd}" - fi - + if [[ -f "${condfd}" ]]; then + rm "${condfd}" + fi - elif [[ -d "/etc/lighttpd/conf-available" ]]; then - if is_command lighty-disable-mod ; then - lighty-disable-mod pihole-admin > /dev/null || true - fi + if is_command lighty-disable-mod ; then + lighty-disable-mod pihole-admin > /dev/null || true + fi - if [[ -f "${confavailable}" ]]; then - rm "${confavailable}" - fi + if [[ -f "${confavailable}" ]]; then + rm "${confavailable}" + fi - if [[ -f "${confenabled}" ]]; then - rm "${confenabled}" - fi - fi + if [[ -f "${confenabled}" ]]; then + rm "${confenabled}" + fi } # Clean an existing installation to prepare for upgrade/reinstall From a3bb3872bfbf9520bd29b2069a889d7d6d4f1202 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 18 Jun 2023 16:30:59 +0100 Subject: [PATCH 053/200] adlist table now contains 11 columns, not 10. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Christian König Signed-off-by: Adam Warner --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index abf6700b..ed402a34 100755 --- a/gravity.sh +++ b/gravity.sh @@ -178,7 +178,7 @@ database_table_from_file() { echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" elif [[ "${table}" == "adlist" ]]; then # Adlist table format - echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0" >> "${tmpFile}" + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0,0" >> "${tmpFile}" else # White-, black-, and regexlist table format echo "${rowid},${list_type},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\"" >> "${tmpFile}" From ca66c1ea9cd905b06618dbded58ff49b9b4f45a1 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 18 Jun 2023 18:01:54 +0100 Subject: [PATCH 054/200] Switch to new branch name for FTL v6 development Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 2 +- test/test_any_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index bc594304..a8e0e082 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -82,7 +82,7 @@ def test_installPihole_fresh_install_readableFiles(host): # Workaround to get FTLv6 installed until it reaches master branch host.run( """ - echo "new/http" > /etc/pihole/ftlbranch + echo "development-v6" > /etc/pihole/ftlbranch """ ) install = host.run( diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 0f300457..9eee6885 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -137,7 +137,7 @@ def test_getFTLConfigValue_getFTLConfigValue(host): source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) - echo "new/http" > /etc/pihole/ftlbranch + echo "development-v6" > /etc/pihole/ftlbranch binary="pihole-FTL${funcOutput##*pihole-FTL}" theRest="${funcOutput%pihole-FTL*}" FTLdetect "${binary}" "${theRest}" From c557f29db2d8d222fdd9d38c5b53fcfac110aeb8 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 3 Jul 2023 23:01:27 +0100 Subject: [PATCH 055/200] Set new gravity database version to 16, which was missed in 5459daa Signed-off-by: Adam Warner --- advanced/Templates/gravity.db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index e0f1de10..881cfcc3 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -58,7 +58,7 @@ CREATE TABLE info value TEXT NOT NULL ); -INSERT INTO "info" VALUES('version','15'); +INSERT INTO "info" VALUES('version','16'); CREATE TABLE domain_audit ( From 64319fa96efcd1b8172afb85110ae0742d49185c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 5 Jul 2023 22:11:56 +0100 Subject: [PATCH 056/200] Add /var/log/pihole/webserver.log to the logrotate scripts. While it may be empty for most, it can grow quickly if either API or TLS debugging is enabled (it can quickly reach several hundreds of MB per day when debugging TLS) Signed-off-by: DL6ER --- advanced/Templates/logrotate | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/advanced/Templates/logrotate b/advanced/Templates/logrotate index 9a56b552..5f609e0f 100644 --- a/advanced/Templates/logrotate +++ b/advanced/Templates/logrotate @@ -19,3 +19,14 @@ notifempty nomail } + +/var/log/pihole/webserver.log { + # su # + weekly + copytruncate + rotate 3 + compress + delaycompress + notifempty + nomail +} From e295997d51d50f661e05eee06b290ad57a56f502 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Jul 2023 21:36:02 +0000 Subject: [PATCH 057/200] Bump actions/setup-python from 4.6.1 to 4.7.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.6.1 to 4.7.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.6.1...v4.7.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index be64482d..09462eb4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,7 +75,7 @@ jobs: uses: actions/checkout@v3.5.3 - name: Set up Python 3.10 - uses: actions/setup-python@v4.6.1 + uses: actions/setup-python@v4.7.0 with: python-version: "3.10" From a3955a7ebb33cdf1ec3ab537eb2a507705648c55 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Jul 2023 21:36:15 +0000 Subject: [PATCH 058/200] Bump pytest from 7.3.2 to 7.4.0 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.3.2 to 7.4.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.3.2...7.4.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 44e6c94a..4eb9113e 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ docker-compose == 1.29.2 -pytest == 7.3.2 +pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 tox == 4.6.2 From 8dd8f989abce529ed3176eeecc7174c8ed3a9053 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Jul 2023 21:44:31 +0000 Subject: [PATCH 059/200] Bump tox from 4.6.2 to 4.6.4 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.6.2 to 4.6.4. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.6.2...4.6.4) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 4eb9113e..a4f627fa 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ docker-compose == 1.29.2 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 -tox == 4.6.2 +tox == 4.6.4 From e42c692ec7b364d443c947187f7ab5943f6f6173 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 18 Jul 2023 08:04:33 +0100 Subject: [PATCH 060/200] Drop docker-compose from requirements.txt, it's not used. This was the only thing preventing us from using pyyaml 6.0.1 which fixes issues we were seeing with tests not running (https://github.com/yaml/pyyaml/pull/702) Signed-off-by: Adam Warner --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index a4f627fa..5a551fa7 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,4 +1,4 @@ -docker-compose == 1.29.2 +pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 From 6edd6a4d55af1ca29c799c96ee355f0747df1ea7 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 17 Jul 2023 21:44:13 +0100 Subject: [PATCH 061/200] if ${USER} variable is blank, then populate it with whoami Signed-off-by: Adam Warner --- pihole | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pihole b/pihole index b762c7d4..66771b9b 100755 --- a/pihole +++ b/pihole @@ -544,6 +544,13 @@ if [[ ! $EUID -eq 0 && need_root -eq 1 ]];then exit 1 fi fi + +# In the case of alpine running in a container, the USER variable appears to be blank +# which prevents the next trap from working correctly. Set it by running whoami +if [[ -z ${USER} ]]; then + USER=$(whoami) +fi + # Can also be user pihole for other functions if [[ ${USER} != "pihole" && need_root -eq 0 ]];then if [[ -x "$(command -v sudo)" ]]; then From 00cbb8bc8abb119ebc3c3f0d912b94a334e06128 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 5 Jul 2023 22:24:11 +0200 Subject: [PATCH 062/200] Add antigravity support to gravity Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 7 +++++ .../database_migration/gravity/16_to_17.sql | 13 ++++++++++ advanced/Templates/gravity.db.sql | 16 +++++++++++- gravity.sh | 26 +++++++++++++------ 4 files changed, 53 insertions(+), 9 deletions(-) create mode 100644 advanced/Scripts/database_migration/gravity/16_to_17.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 851ddb0a..e36d9b1e 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -134,4 +134,11 @@ upgrade_gravityDB(){ pihole-FTL sqlite3 "${database}" < "${scriptPath}/15_to_16.sql" version=16 fi + if [[ "$version" == "16" ]]; then + # Add antigravity table + # Add column type to adlist table (to support adlist types) + echo -e " ${INFO} Upgrading gravity database from version 16 to 17" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/16_to_17.sql" + version=17 + fi } diff --git a/advanced/Scripts/database_migration/gravity/16_to_17.sql b/advanced/Scripts/database_migration/gravity/16_to_17.sql new file mode 100644 index 00000000..c7b9049b --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/16_to_17.sql @@ -0,0 +1,13 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +ALTER TABLE adlist ADD COLUMN type INTEGER NOT NULL DEFAULT 0; + +UPDATE adlist SET type = 0; + +UPDATE info SET value = 17 WHERE property = 'version'; + +COMMIT; diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 881cfcc3..17712cf7 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -36,7 +36,8 @@ CREATE TABLE adlist number INTEGER NOT NULL DEFAULT 0, invalid_domains INTEGER NOT NULL DEFAULT 0, status INTEGER NOT NULL DEFAULT 0, - abp_entries INTEGER NOT NULL DEFAULT 0 + abp_entries INTEGER NOT NULL DEFAULT 0, + type INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE adlist_by_group @@ -52,6 +53,12 @@ CREATE TABLE gravity adlist_id INTEGER NOT NULL REFERENCES adlist (id) ); +CREATE TABLE antigravity +( + domain TEXT NOT NULL, + adlist_id INTEGER NOT NULL REFERENCES adlist (id) +); + CREATE TABLE info ( property TEXT PRIMARY KEY, @@ -144,6 +151,13 @@ CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id LEFT JOIN "group" ON "group".id = adlist_by_group.group_id WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1); +CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id + FROM antigravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id + LEFT JOIN adlist ON adlist.id = antigravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; + CREATE VIEW vw_adlist AS SELECT DISTINCT address, id FROM adlist WHERE enabled = 1 diff --git a/gravity.sh b/gravity.sh index ed402a34..d784a2e7 100755 --- a/gravity.sh +++ b/gravity.sh @@ -361,6 +361,7 @@ gravity_DownloadBlocklists() { # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true) mapfile -t sources <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)" + mapfile -t sourceTypes <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2> /dev/null)" # Parse source domains from $sources mapfile -t sourceDomains <<< "$( @@ -382,7 +383,7 @@ gravity_DownloadBlocklists() { unset sources fi - local url domain agent str target compression + local url domain agent str target compression adlist_type echo "" # Prepare new gravity database @@ -394,7 +395,7 @@ gravity_DownloadBlocklists() { if [[ "${status}" -ne 0 ]]; then echo -e "\\n ${CROSS} Unable to create new database ${gravityTEMPfile}\\n ${output}" - gravity_Cleanup "error" + #gravity_Cleanup "error" else echo -e "${OVER} ${TICK} ${str}" fi @@ -433,6 +434,15 @@ gravity_DownloadBlocklists() { url="${sources[$i]}" domain="${sourceDomains[$i]}" id="${sourceIDs[$i]}" + if [[ "${sourceTypes[$i]}" -eq "0" ]]; then + # Gravity list + str="blocklist" + adlist_type="gravity" + else + # AntiGravity list + str="allowlist" + adlist_type="antigravity" + fi # Save the file as list.#.domain saveLocation="${piholeDir}/list.${id}.${domain}.${domainsExtension}" @@ -441,7 +451,7 @@ gravity_DownloadBlocklists() { # Default user-agent (for Cloudflare's Browser Integrity Check: https://support.cloudflare.com/hc/en-us/articles/200170086-What-does-the-Browser-Integrity-Check-do-) agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36" - echo -e " ${INFO} Target: ${url}" + echo -e " ${INFO} Target: ${url} (${str})" local regex check_url # Check for characters NOT allowed in URLs regex="[^a-zA-Z0-9:/?&%=~._()-;]" @@ -453,7 +463,7 @@ gravity_DownloadBlocklists() { if [[ "${check_url}" =~ ${regex} ]]; then echo -e " ${CROSS} Invalid Target" else - gravity_DownloadBlocklistFromUrl "${url}" "${agent}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" + gravity_DownloadBlocklistFromUrl "${url}" "${agent}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" "${adlist_type}" fi echo "" done @@ -485,7 +495,7 @@ compareLists() { # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { - local url="${1}" agent="${2}" adlistID="${3}" saveLocation="${4}" target="${5}" compression="${6}" + local url="${1}" agent="${2}" adlistID="${3}" saveLocation="${4}" target="${5}" compression="${6}" gravity_type="${7}" local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext # Create temp file to store content on disk instead of RAM @@ -579,7 +589,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ "${success}" == true ]]; then if [[ "${httpCode}" == "304" ]]; then # Add domains to database table file - pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" database_adlist_status "${adlistID}" "2" done="true" # Check if $listCurlBuffer is a non-zero length file @@ -589,7 +599,7 @@ gravity_DownloadBlocklistFromUrl() { # Remove curl buffer file after its use rm "${listCurlBuffer}" # Add domains to database table file - pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" # Compare lists, are they identical? compareLists "${adlistID}" "${saveLocation}" done="true" @@ -605,7 +615,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ -r "${saveLocation}" ]]; then echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" # Add domains to database table file - pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" database_adlist_status "${adlistID}" "3" else echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" From 35512c4dc9071502b10f251d514d297a79fe7ea5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 6 Jul 2023 19:19:55 +0200 Subject: [PATCH 063/200] Fix adlist.list migration step failing during tests Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index d784a2e7..89b4ab84 100755 --- a/gravity.sh +++ b/gravity.sh @@ -178,7 +178,7 @@ database_table_from_file() { echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" elif [[ "${table}" == "adlist" ]]; then # Adlist table format - echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0,0" >> "${tmpFile}" + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0,0,0" >> "${tmpFile}" else # White-, black-, and regexlist table format echo "${rowid},${list_type},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\"" >> "${tmpFile}" From 375d4d9bc13b668682ded29efd2387d62d5623c5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 6 Jul 2023 21:33:59 +0200 Subject: [PATCH 064/200] Add type as new field of view vw_adlist Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 17712cf7..46f26ba7 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -65,7 +65,7 @@ CREATE TABLE info value TEXT NOT NULL ); -INSERT INTO "info" VALUES('version','16'); +INSERT INTO "info" VALUES('version','17'); CREATE TABLE domain_audit ( @@ -158,7 +158,7 @@ CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_i LEFT JOIN "group" ON "group".id = adlist_by_group.group_id WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; -CREATE VIEW vw_adlist AS SELECT DISTINCT address, id +CREATE VIEW vw_adlist AS SELECT DISTINCT address, id, type FROM adlist WHERE enabled = 1 ORDER BY id; From 2a03671fb9f15cfe7e6ee378350d7f32d2e04000 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 6 Jul 2023 22:52:28 +0200 Subject: [PATCH 065/200] Reinstall gravity cleanup on error Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 89b4ab84..acf55ae3 100755 --- a/gravity.sh +++ b/gravity.sh @@ -395,7 +395,7 @@ gravity_DownloadBlocklists() { if [[ "${status}" -ne 0 ]]; then echo -e "\\n ${CROSS} Unable to create new database ${gravityTEMPfile}\\n ${output}" - #gravity_Cleanup "error" + gravity_Cleanup "error" else echo -e "${OVER} ${TICK} ${str}" fi From 5ae0405446103a29c525cba9cdeb12ad390372b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 3 Aug 2023 20:46:01 +0200 Subject: [PATCH 066/200] Ensure pihole-FTL can write custom.list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 4 ++-- automated install/basic-install.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index ff4abf3a..f6e28fec 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -17,9 +17,9 @@ mkdir -pm 0755 /run/pihole /var/log/pihole [ -f /var/log/pihole/pihole.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/pihole.log [ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit all necessary files -chown pihole:pihole /run/pihole /etc/pihole /var/log/pihole /var/log/pihole/FTL.log /var/log/pihole/pihole.log /etc/pihole/dhcp.leases +chown pihole:pihole /run/pihole /etc/pihole /var/log/pihole /var/log/pihole/FTL.log /var/log/pihole/pihole.log /etc/pihole/dhcp.leases /etc/pihole/custom.list # Ensure that permissions are set so that pihole-FTL can edit the files. We ignore errors as the file may not (yet) exist -chmod -f 0644 /etc/pihole/macvendor.db /etc/pihole/dhcp.leases /var/log/pihole/FTL.log +chmod -f 0644 /etc/pihole/macvendor.db /etc/pihole/dhcp.leases /var/log/pihole/FTL.log /etc/pihole/custom.list chmod -f 0640 /var/log/pihole/pihole.log # Chown database files to the user FTL runs as. We ignore errors as the files may not (yet) exist chown -f pihole:pihole /etc/pihole/pihole-FTL.db /etc/pihole/gravity.db /etc/pihole/macvendor.db diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index e3d8ff29..050883cd 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1094,7 +1094,7 @@ installConfigs() { # Install empty custom.list file if it does not exist if [[ ! -r "${PI_HOLE_CONFIG_DIR}/custom.list" ]]; then - if ! install -o root -m 644 /dev/null "${PI_HOLE_CONFIG_DIR}/custom.list" &>/dev/null; then + if ! install -o pihole -g pihole -m 644 /dev/null "${PI_HOLE_CONFIG_DIR}/custom.list" &>/dev/null; then printf " %b Error: Unable to initialize configuration file %s/custom.list\\n" "${COL_LIGHT_RED}" "${PI_HOLE_CONFIG_DIR}" return 1 fi From a3ea2cd8c311e4c81d7b0654c0893b4bf5cc1e5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 4 Aug 2023 19:32:19 +0200 Subject: [PATCH 067/200] User pihole should be allowed to edit all its files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index f6e28fec..e6a5aeb2 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -17,14 +17,7 @@ mkdir -pm 0755 /run/pihole /var/log/pihole [ -f /var/log/pihole/pihole.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/pihole.log [ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit all necessary files -chown pihole:pihole /run/pihole /etc/pihole /var/log/pihole /var/log/pihole/FTL.log /var/log/pihole/pihole.log /etc/pihole/dhcp.leases /etc/pihole/custom.list -# Ensure that permissions are set so that pihole-FTL can edit the files. We ignore errors as the file may not (yet) exist -chmod -f 0644 /etc/pihole/macvendor.db /etc/pihole/dhcp.leases /var/log/pihole/FTL.log /etc/pihole/custom.list -chmod -f 0640 /var/log/pihole/pihole.log -# Chown database files to the user FTL runs as. We ignore errors as the files may not (yet) exist -chown -f pihole:pihole /etc/pihole/pihole-FTL.db /etc/pihole/gravity.db /etc/pihole/macvendor.db -# Chmod database file permissions so that the pihole group (web interface) can edit the file. We ignore errors as the files may not (yet) exist -chmod -f 0664 /etc/pihole/pihole-FTL.db +chown -R pihole:pihole /run/pihole /etc/pihole /var/log/pihole # Backward compatibility for user-scripts that still expect log files in /var/log instead of /var/log/pihole # Should be removed with Pi-hole v6.0 From 8ef8a275830341140009bdbf229ae8cebad7aede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 4 Aug 2023 19:41:19 +0200 Subject: [PATCH 068/200] Remove webpage.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 100 ------------------------------------ pihole | 41 ++++++++++++--- 2 files changed, 33 insertions(+), 108 deletions(-) delete mode 100755 advanced/Scripts/webpage.sh diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh deleted file mode 100755 index 67cbe766..00000000 --- a/advanced/Scripts/webpage.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC1090 -# shellcheck disable=SC2154 - - -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Web interface settings -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. - -# TODO - this entire file might be able to be removed in v6 - -readonly dnsmasqconfig="/etc/dnsmasq.d/01-pihole.conf" -readonly dhcpconfig="/etc/dnsmasq.d/02-pihole-dhcp.conf" -readonly FTLconf="/etc/pihole/pihole-FTL.conf" -# 03 -> wildcards -readonly dhcpstaticconfig="/etc/dnsmasq.d/04-pihole-static-dhcp.conf" -readonly dnscustomfile="/etc/pihole/custom.list" -readonly dnscustomcnamefile="/etc/dnsmasq.d/05-pihole-custom-cname.conf" - -readonly gravityDBfile="/etc/pihole/gravity.db" - - -readonly setupVars="/etc/pihole/setupVars.conf" -readonly PI_HOLE_BIN_DIR="/usr/local/bin" - -# Root of the web server -readonly webroot="/var/www/html" - -# Source utils script -utilsfile="/opt/pihole/utils.sh" -source "${utilsfile}" - -coltable="/opt/pihole/COL_TABLE" -if [[ -f ${coltable} ]]; then - source ${coltable} -fi - -helpFunc() { - echo "Usage: pihole -a [options] -Example: pihole -a -p password -Set options for the API/Web interface - -Options: - -p, password Set API/Web interface password - -h, --help Show this help dialog" - exit 0 -} - -# TODO: We can probably remove the reliance on this function too, just tell people to pihole-FTL --config webserver.api.password "password" -SetWebPassword() { - if (( ${#args[2]} > 0 )) ; then - readonly PASSWORD="${args[2]}" - readonly CONFIRM="${PASSWORD}" - else - # Prevents a bug if the user presses Ctrl+C and it continues to hide the text typed. - # So we reset the terminal via stty if the user does press Ctrl+C - trap '{ echo -e "\nNot changed" ; stty sane ; exit 1; }' INT - read -s -r -p "Enter New Password (Blank for no password): " PASSWORD - echo "" - - if [ "${PASSWORD}" == "" ]; then - setFTLConfigValue "webserver.api.pwhash" "" >/dev/null - echo -e " ${TICK} Password Removed" - exit 0 - fi - - read -s -r -p "Confirm Password: " CONFIRM - echo "" - fi - - if [ "${PASSWORD}" == "${CONFIRM}" ] ; then - # pihole-FTL will automatically hash the password - setFTLConfigValue "webserver.api.password" "${PASSWORD}" >/dev/null - echo -e " ${TICK} New password set" - else - echo -e " ${CROSS} Passwords don't match. Your password has not been changed" - exit 1 - fi -} - -main() { - args=("$@") - - case "${args[1]}" in - "-p" | "password" ) SetWebPassword;; - "-h" | "--help" ) helpFunc;; - * ) helpFunc;; - esac - - shift - - if [[ $# = 0 ]]; then - helpFunc - fi -} diff --git a/pihole b/pihole index 66771b9b..54b20f7d 100755 --- a/pihole +++ b/pihole @@ -30,10 +30,36 @@ if [ -f "${versionsfile}" ]; then source "${versionsfile}" fi -webpageFunc() { - source "${PI_HOLE_SCRIPT_DIR}/webpage.sh" - main "$@" - exit 0 +# TODO: We can probably remove the reliance on this function too, just tell people to pihole-FTL --config webserver.api.password "password" +SetWebPassword() { + if [ -n "$2" ] ; then + readonly PASSWORD="$2" + readonly CONFIRM="${PASSWORD}" + else + # Prevents a bug if the user presses Ctrl+C and it continues to hide the text typed. + # So we reset the terminal via stty if the user does press Ctrl+C + trap '{ echo -e "\nNot changed" ; stty sane ; exit 1; }' INT + read -s -r -p "Enter New Password (Blank for no password): " PASSWORD + echo "" + + if [ "${PASSWORD}" == "" ]; then + setFTLConfigValue "webserver.api.pwhash" "" >/dev/null + echo -e " ${TICK} Password Removed" + exit 0 + fi + + read -s -r -p "Confirm Password: " CONFIRM + echo "" + fi + + if [ "${PASSWORD}" == "${CONFIRM}" ] ; then + # pihole-FTL will automatically hash the password + setFTLConfigValue "webserver.api.password" "${PASSWORD}" >/dev/null + echo -e " ${TICK} New password set" + else + echo -e " ${CROSS} Passwords don't match. Your password has not been changed" + exit 1 + fi } listFunc() { @@ -466,8 +492,7 @@ Debugging Options: Options: - -a, admin Web interface options - Add '-h' for more info on Web Interface usage + setpassword set the password for the web interface -c, chronometer Calculates stats and displays to an LCD Add '-h' for more info on chronometer usage -g, updateGravity Update the list of ad-serving domains @@ -526,7 +551,7 @@ case "${1}" in "restartdns" ) ;; "-g" | "updateGravity" ) need_root=0;; "reloaddns" ) need_root=0;; - "-a" | "admin" ) ;; + "setpassword" ) ;; "checkout" ) ;; "updatechecker" ) ;; "arpflush" ) ;; @@ -581,7 +606,7 @@ case "${1}" in "disable" ) piholeEnable 0 "$2";; "restartdns" ) restartDNS "$2";; "reloaddns" ) restartDNS "reload";; - "-a" | "admin" ) webpageFunc "$@";; + "setpassword" ) SetWebPassword "$@";; "checkout" ) piholeCheckoutFunc "$@";; "updatechecker" ) shift; updateCheckFunc "$@";; "arpflush" ) arpFunc "$@";; From c360743d41620972cfa0b9b10190d010d12bba5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 4 Aug 2023 21:12:40 +0200 Subject: [PATCH 069/200] Re-add file/folder permissions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index e6a5aeb2..ef8e1b88 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -18,6 +18,8 @@ mkdir -pm 0755 /run/pihole /var/log/pihole [ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit all necessary files chown -R pihole:pihole /run/pihole /etc/pihole /var/log/pihole +chmod -R 0640 /var/log/pihole +chmod -R 0660 /etc/pihole /run/pihole # Backward compatibility for user-scripts that still expect log files in /var/log instead of /var/log/pihole # Should be removed with Pi-hole v6.0 From 587a2a1c04fa743480d808f74b87da3d8539bc9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 4 Aug 2023 23:52:53 +0200 Subject: [PATCH 070/200] # allow all users to enter der directories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index ef8e1b88..62183db9 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -20,6 +20,8 @@ mkdir -pm 0755 /run/pihole /var/log/pihole chown -R pihole:pihole /run/pihole /etc/pihole /var/log/pihole chmod -R 0640 /var/log/pihole chmod -R 0660 /etc/pihole /run/pihole +# allow all users to enter der directories +chmod 0755 /etc/pihole /run/pihole /var/log/pihole # Backward compatibility for user-scripts that still expect log files in /var/log instead of /var/log/pihole # Should be removed with Pi-hole v6.0 From 7bb0ca59d157f21a50e1e4abcf18734ae2c62347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 5 Aug 2023 14:54:09 +0200 Subject: [PATCH 071/200] Remove traces of /run/pihole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index 62183db9..a0353f34 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -11,17 +11,17 @@ FTL_PID_FILE="$(getFTLPIDFile)" # Touch files to ensure they exist (create if non-existing, preserve if existing) # shellcheck disable=SC2174 -mkdir -pm 0755 /run/pihole /var/log/pihole +mkdir -pm 0755 /var/log/pihole [ -f "${FTL_PID_FILE}" ] || install -D -m 644 -o pihole -g pihole /dev/null "${FTL_PID_FILE}" [ -f /var/log/pihole/FTL.log ] || install -m 644 -o pihole -g pihole /dev/null /var/log/pihole/FTL.log [ -f /var/log/pihole/pihole.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/pihole.log [ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit all necessary files -chown -R pihole:pihole /run/pihole /etc/pihole /var/log/pihole +chown -R pihole:pihole /etc/pihole /var/log/pihole chmod -R 0640 /var/log/pihole chmod -R 0660 /etc/pihole /run/pihole # allow all users to enter der directories -chmod 0755 /etc/pihole /run/pihole /var/log/pihole +chmod 0755 /etc/pihole /var/log/pihole # Backward compatibility for user-scripts that still expect log files in /var/log instead of /var/log/pihole # Should be removed with Pi-hole v6.0 From 3c693c1da54b15869d7d612bdf20153234911073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 6 Aug 2023 12:04:55 +0200 Subject: [PATCH 072/200] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: DL6ER Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 4 ++-- automated install/basic-install.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index a0353f34..17900f0b 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -19,8 +19,8 @@ mkdir -pm 0755 /var/log/pihole # Ensure that permissions are set so that pihole-FTL can edit all necessary files chown -R pihole:pihole /etc/pihole /var/log/pihole chmod -R 0640 /var/log/pihole -chmod -R 0660 /etc/pihole /run/pihole -# allow all users to enter der directories +chmod -R 0660 /etc/pihole +# allow all users to enter the directories chmod 0755 /etc/pihole /var/log/pihole # Backward compatibility for user-scripts that still expect log files in /var/log instead of /var/log/pihole diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 050883cd..bf26631a 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1094,7 +1094,7 @@ installConfigs() { # Install empty custom.list file if it does not exist if [[ ! -r "${PI_HOLE_CONFIG_DIR}/custom.list" ]]; then - if ! install -o pihole -g pihole -m 644 /dev/null "${PI_HOLE_CONFIG_DIR}/custom.list" &>/dev/null; then + if ! install -o pihole -g pihole -m 660 /dev/null "${PI_HOLE_CONFIG_DIR}/custom.list" &>/dev/null; then printf " %b Error: Unable to initialize configuration file %s/custom.list\\n" "${COL_LIGHT_RED}" "${PI_HOLE_CONFIG_DIR}" return 1 fi From 2227a2c5693633d89fb99fe3a2079d26c9bfe225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 9 Aug 2023 21:25:50 +0200 Subject: [PATCH 073/200] Add API functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/api.sh | 140 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 advanced/Scripts/api.sh diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh new file mode 100644 index 00000000..afd88671 --- /dev/null +++ b/advanced/Scripts/api.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env sh +# shellcheck disable=SC3043 #https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions + +# Pi-hole: A black hole for Internet advertisements +# (c) 2017 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# Script to hold api functions for use in other scripts +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + + +# The basic usage steps are +# 1) Test Availability of the API +# 2) Try to authenticate (read password if needed) +# 3) Get the data from the API endpoint +# 4) Delete the session + + +TestAPIAvailability() { + + # as we are running locally, we can get the port value from FTL directly + PORT="$(pihole-FTL --config webserver.port)" + PORT="${PORT%%,*}" + + availabilityResonse=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:${PORT}/api/auth") + + # test if http status code was 200 (OK) or 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 401 ]; then + echo "API not available at: http://localhost:${PORT}/api" + echo "Exiting." + exit 1 + fi +} + +Authenthication() { + # Try to authenticate + LoginAPI + + while [ "${validSession}" = false ] || [ -z "${validSession}" ] ; do + echo "Authentication failed. Please enter your Pi-hole password" + + # secretly read the password + secretRead; printf '\n' + + # Try to authenticate again + LoginAPI + done + + # Loop exited, authentication was successful + echo "Authentication successful." + +} + +LoginAPI() { + sessionResponse="$(curl --silent -X POST "http://localhost:${PORT}/api/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" + + if [ -z "${sessionResponse}" ]; then + echo "No response from FTL server. Please check connectivity" + exit 1 + fi + # obtain validity and session ID from session response + validSession=$(echo "${sessionResponse}"| jq .session.valid 2>/dev/null) + SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null) +} + +DeleteSession() { + # if a valid Session exists (no password required or successful authenthication) and + # SID is not null (successful authenthication only), delete the session + if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then + # Try to delete the session. Omit the output, but get the http status code + deleteResponse=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE "http://localhost:${PORT}/api/auth" -H "Accept: application/json" -H "sid: ${SID}") + + case "${deleteResponse}" in + "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; + "401") printf "%b" "Logout attempt without a valid session. Unauthorized!\n";; + "410") printf "%b" "Session successfully deleted.\n";; + esac; + fi + +} + +GetFTLData() { + local data + # get the data from querying the API as well as the http status code + data=$(curl -s -X GET "http://localhost:${PORT}/api$1" -H "Accept: application/json" -H "sid: ${SID}" ) + echo "${data}" +} + +secretRead() { + + # POSIX compliant function to read user-input and + # mask every character entered by (*) + # + # This is challenging, because in POSIX, `read` does not support + # `-s` option (suppressing the input) or + # `-n` option (reading n chars) + + + # This workaround changes the terminal characteristics to not echo input and later resets this option + # credits https://stackoverflow.com/a/4316765 + # showing asterisk instead of password + # https://stackoverflow.com/a/24600839 + # https://unix.stackexchange.com/a/464963 + + + # Save current terminal settings (needed for later restore after password prompt) + stty_orig=$(stty -g) + + stty -echo # do not echo user input + stty -icanon min 1 time 0 # disable canonical mode https://man7.org/linux/man-pages/man3/termios.3.html + + unset password + unset key + unset charcount + charcount=0 + while key=$(dd ibs=1 count=1 2>/dev/null); do #read one byte of input + if [ "${key}" = "$(printf '\0' | tr -d '\0')" ] ; then + # Enter - accept password + break + fi + if [ "${key}" = "$(printf '\177')" ] ; then + # Backspace + if [ $charcount -gt 0 ] ; then + charcount=$((charcount-1)) + printf '\b \b' + password="${password%?}" + fi + else + # any other character + charcount=$((charcount+1)) + printf '*' + password="$password$key" + fi + done + + # restore original terminal settings + stty "${stty_orig}" +} From bd55b2e5660d31dd5f99e2343ab5c31e871c0075 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 12 Aug 2023 10:25:00 +0000 Subject: [PATCH 074/200] Bump tox from 4.6.4 to 4.7.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.6.4 to 4.7.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.6.4...4.7.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 5a551fa7..c5d002e5 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 -tox == 4.6.4 +tox == 4.7.0 From 8e0a40e71706b70b880a288e0bc93780d4ee54db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 13 Aug 2023 22:00:44 +0200 Subject: [PATCH 075/200] Use api.sh within query.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/COL_TABLE | 2 +- advanced/Scripts/query.sh | 315 ++++++++++++------------------------- 2 files changed, 99 insertions(+), 218 deletions(-) mode change 100755 => 100644 advanced/Scripts/query.sh diff --git a/advanced/Scripts/COL_TABLE b/advanced/Scripts/COL_TABLE index 2d2b074b..4344c7b8 100644 --- a/advanced/Scripts/COL_TABLE +++ b/advanced/Scripts/COL_TABLE @@ -1,5 +1,5 @@ # Determine if terminal is capable of showing colors -if ([[ -t 1 ]] && [[ $(tput colors) -ge 8 ]]) || [[ "${WEBCALL}" ]]; then +if ([ -t 1 ] && [ $(tput colors) -ge 8 ]) || [ "${WEBCALL}" ]; then # Bold and underline may not show up on all clients # If something MUST be emphasized, use both COL_BOLD='' diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh old mode 100755 new mode 100644 index bfa21247..8407d3f6 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -1,259 +1,140 @@ -#!/usr/bin/env bash +#!/usr/bin/env sh # shellcheck disable=SC1090 +# Ignore warning about `local` being undefinded in POSIX +# shellcheck disable=SC3043 +# https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions + # Pi-hole: A black hole for Internet advertisements -# (c) 2018 Pi-hole, LLC (https://pi-hole.net) +# (c) 2023 Pi-hole, LLC (https://pi-hole.net) # Network-wide ad blocking via your own hardware. # -# Query Domain Lists +# Search Adlists # # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. # Globals -piholeDir="/etc/pihole" -GRAVITYDB="${piholeDir}/gravity.db" -options="$*" -all="" -exact="" -matchType="match" -# Source pihole-FTL from install script -pihole_FTL="${piholeDir}/pihole-FTL.conf" -if [[ -f "${pihole_FTL}" ]]; then - source "${pihole_FTL}" -fi - -# Set this only after sourcing pihole-FTL.conf as the gravity database path may -# have changed -gravityDBfile="${GRAVITYDB}" +PI_HOLE_INSTALL_DIR="/opt/pihole" +max_results="20" +partial="true" +domain="" +# Source color table colfile="/opt/pihole/COL_TABLE" -source "${colfile}" +. "${colfile}" -if [[ "${options}" == "-h" ]] || [[ "${options}" == "--help" ]]; then +# Source api functions +. "${PI_HOLE_INSTALL_DIR}/api.sh" + +Help(){ echo "Usage: pihole -q [option] -Example: 'pihole -q -exact domain.com' +Example: 'pihole -q --exact domain.com' Query the adlists for a specified domain Options: - -exact Search the adlists for exact domain matches - -all Return all query matches within the adlists + --exact Search the adlists for exact domain matches + --all Return all query matches within the adlists -h, --help Show this help dialog" exit 0 -fi - -# Handle valid options -[[ "${options}" == *"-all"* ]] && all=true -if [[ "${options}" == *"-exact"* ]]; then - exact="exact"; matchType="exact ${matchType}" -fi - -# Strip valid options, leaving only the domain and invalid options -# This allows users to place the options before or after the domain -options=$(sed -E 's/ +-(all|exact) ?//g' <<< "${options}") - -# Handle remaining options -# If $options contain non ASCII characters, convert to punycode -case "${options}" in - "" ) str="No domain specified";; - *" "* ) str="Unknown query option specified";; - *[![:ascii:]]* ) rawDomainQuery=$(idn2 "${options}");; - * ) rawDomainQuery="${options}";; -esac - -# convert the domain to lowercase -domainQuery=$(echo "${rawDomainQuery}" | tr '[:upper:]' '[:lower:]') - -if [[ -n "${str:-}" ]]; then - echo -e "${str}${COL_NC}\\nTry 'pihole -q --help' for more information." - exit 1 -fi - -# Scan a domain again a list of RegEX -scanRegExList(){ - local domain="${1}" list="${2}" - - for entry in ${list}; do - if [[ "${domain}" =~ ${entry} ]]; then - printf "%b\n" "${entry}"; - fi - done - } -scanDatabaseTable() { - local domain table list_type querystr result extra abpquerystr abpfound abpentry searchstr - domain="$(printf "%q" "${1}")" - table="${2}" - list_type="${3:-}" - # As underscores are legitimate parts of domains, we escape them when using the LIKE operator. - # Underscores are SQLite wildcards matching exactly one character. We obviously want to suppress this - # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched - # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores. - if [[ "${table}" == "gravity" ]]; then +GenerateOutput(){ + local data gravity_data lists_data num_gravity num_lists search_type_str + local gravity_data_csv lists_data_csv line current_domain + data="${1}" - # Are there ABP entries on gravity? - # Return 1 if abp_domain=1 or Zero if abp_domain=0 or not set - abpquerystr="SELECT EXISTS (SELECT 1 FROM info WHERE property='abp_domains' and value='1')" - abpfound="$(pihole-FTL sqlite3 "${gravityDBfile}" "${abpquerystr}")" 2> /dev/null + # construct a new json for the list results where each object contains the domain and the related type + lists_data=$(echo "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]') - # Create search string for ABP entries only if needed - if [ "${abpfound}" -eq 1 ]; then - abpentry="${domain}" + # construct a new json for the gravity results where each object contains the adlist URL and the related domains + gravity_data=$(echo "${data}" | jq '.search.gravity | group_by(.address) | map({ address: (.[0].address), domains: [.[] | .domain] })') - searchstr="'||${abpentry}^'" + # number of objects in each json + num_gravity=$(echo "${gravity_data}" | jq length ) + num_lists=$(echo "${lists_data}" | jq length ) - # While a dot is found ... - while [ "${abpentry}" != "${abpentry/./}" ] - do - # ... remove text before the dot (including the dot) and append the result to $searchstr - abpentry=$(echo "${abpentry}" | cut -f 2- -d '.') - searchstr="$searchstr, '||${abpentry}^'" - done - - # The final search string will look like: - # "domain IN ('||sub2.sub1.domain.com^', '||sub1.domain.com^', '||domain.com^', '||com^') OR" - searchstr="domain IN (${searchstr}) OR " - fi - - case "${exact}" in - "exact" ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain = '${domain}'";; - * ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE ${searchstr} domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; - esac + if [ "${partial}" = true ]; then + search_type_str="partially" else - case "${exact}" in - "exact" ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND domain = '${domain}'";; - * ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; - esac + search_type_str="exactly" fi - # Send prepared query to gravity database - result="$(pihole-FTL sqlite3 -separator ',' "${gravityDBfile}" "${querystr}")" 2> /dev/null - if [[ -z "${result}" ]]; then - # Return early when there are no matches in this table - return + # Results from allow/deny list + printf "%s\n\n" "Found ${num_lists} domains ${search_type_str} matching '${COL_BLUE}${domain}${COL_NC}'." + if [ "${num_lists}" -gt 0 ]; then + # Convert the data to a csv, each line is a "domain,type" string + # not using jq's @csv here as it quotes each value individually + lists_data_csv=$(echo "${lists_data}" | jq --raw-output '.[] | [.domain, .type] | join(",")' ) + + # Generate output for each csv line, separating line in a domain and type substring at the ',' + echo "${lists_data_csv}" | while read -r line; do + printf "%s\n\n" " - ${COL_GREEN}${line%,*}${COL_NC} (type: exact ${line#*,} domain)" + done fi - if [[ "${table}" == "gravity" ]]; then - echo "${result}" - return + # Results from gravity + printf "%s\n\n" "Found ${num_gravity} adlists ${search_type_str} matching '${COL_BLUE}${domain}${COL_NC}'." + if [ "${num_gravity}" -gt 0 ]; then + # Convert the data to a csv, each line is a "URL,domain,domain,...." string + # not using jq's @csv here as it quotes each value individually + gravity_data_csv=$(echo "${gravity_data}" | jq --raw-output '.[] | [.address, .domains[]] | join(",")' ) + + # Generate line-by-line output for each csv line + echo "${gravity_data_csv}" | while read -r line; do + + # print adlist URL + printf "%s\n\n" " - ${COL_BLUE}${line%%,*}${COL_NC}" + + # cut off URL, leaving "domain,domain,...." + line=${line#*,} + # print each domain and remove it from the string until nothing is left + while [ ${#line} -gt 0 ]; do + current_domain=${line%%,*} + printf ' - %s\n' "${COL_GREEN}${current_domain}${COL_NC}" + # we need to remove the current_domain and the comma in two steps because + # the last domain won't have a trailing comma and the while loop wouldn't exit + line=${line#"${current_domain}"} + line=${line#,} + done + printf "\n\n" + done fi - - # Mark domain as having been white-/blacklist matched (global variable) - wbMatch=true - - # Print table name - echo " ${matchType^} found in ${COL_BOLD}exact ${table}${COL_NC}" - - # Loop over results and print them - mapfile -t results <<< "${result}" - for result in "${results[@]}"; do - domain="${result/,*}" - if [[ "${result#*,}" == "0" ]]; then - extra=" (disabled)" - else - extra="" - fi - echo " ${domain}${extra}" - done } -scanRegexDatabaseTable() { - local domain list list_type - domain="${1}" - list="${2}" - list_type="${3:-}" +Main(){ + local data - # Query all regex from the corresponding database tables - mapfile -t regexList < <(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain FROM domainlist WHERE type = ${list_type}" 2> /dev/null) + if [ -z "${domain}" ]; then + echo "No domain specified"; exit 1 + else + # convert domain to punycode + domain=$(idn2 "${domain}") - # If we have regexps to process - if [[ "${#regexList[@]}" -ne 0 ]]; then - # Split regexps over a new line - str_regexList=$(printf '%s\n' "${regexList[@]}") - # Check domain against regexps - mapfile -t regexMatches < <(scanRegExList "${domain}" "${str_regexList}") - # If there were regex matches - if [[ "${#regexMatches[@]}" -ne 0 ]]; then - # Split matching regexps over a new line - str_regexMatches=$(printf '%s\n' "${regexMatches[@]}") - # Form a "matched" message - str_message="${matchType^} found in ${COL_BOLD}regex ${list}${COL_NC}" - # Form a "results" message - str_result="${COL_BOLD}${str_regexMatches}${COL_NC}" - # If we are displaying more than just the source of the block - # Set the wildcard match flag - wcMatch=true - # Echo the "matched" message, indented by one space - echo " ${str_message}" - # Echo the "results" message, each line indented by three spaces - # shellcheck disable=SC2001 - echo "${str_result}" | sed 's/^/ /' - fi + # convert the domain to lowercase + domain=$(echo "${domain}" | tr '[:upper:]' '[:lower:]') fi -} - -# Scan Whitelist and Blacklist -scanDatabaseTable "${domainQuery}" "whitelist" "0" -scanDatabaseTable "${domainQuery}" "blacklist" "1" - -# Scan Regex table -scanRegexDatabaseTable "${domainQuery}" "whitelist" "2" -scanRegexDatabaseTable "${domainQuery}" "blacklist" "3" - -# Query block lists -mapfile -t results <<< "$(scanDatabaseTable "${domainQuery}" "gravity")" -# Handle notices -if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then - echo -e " ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} within the adlists" - exit 0 -elif [[ -z "${results[*]}" ]]; then - # Result found in WL/BL/Wildcards - exit 0 -elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 100 ]]; then - echo -e " ${INFO} Over 100 ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} - This can be overridden using the -all option" - exit 0 -fi + # Test if the authentication endpoint is available + TestAPIAvailability + # Authenticate with the FTL server + Authenthication -# Print "Exact matches for" title -if [[ -n "${exact}" ]]; then - plural=""; [[ "${#results[*]}" -gt 1 ]] && plural="es" - echo " ${matchType^}${plural} for ${COL_BOLD}${domainQuery}${COL_NC} found in:" -fi - -for result in "${results[@]}"; do - match="${result/,*/}" - extra="${result#*,}" - adlistAddress="${extra/,*/}" - extra="${extra#*,}" - if [[ "${extra}" == "0" ]]; then - extra=" (disabled)" - else - extra="" - fi - - if [[ -n "${exact}" ]]; then - echo " - ${adlistAddress}${extra}" - else - if [[ ! "${adlistAddress}" == "${adlistAddress_prev:-}" ]]; then - count="" - echo " ${matchType^} found in ${COL_BOLD}${adlistAddress}${COL_NC}:" - adlistAddress_prev="${adlistAddress}" - fi - : $((count++)) + data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}") + GenerateOutput "${data}" + DeleteSession +} - # Print matching domain if $max_count has not been reached - [[ -z "${all}" ]] && max_count="50" - if [[ -z "${all}" ]] && [[ "${count}" -ge "${max_count}" ]]; then - [[ "${count}" -gt "${max_count}" ]] && continue - echo " ${COL_GRAY}Over ${count} results found, skipping rest of file${COL_NC}" - else - echo " ${match}${extra}" - fi - fi +# Process all options (if present) +while [ "$#" -gt 0 ]; do + case "$1" in + "-h" | "--help" ) Help;; + "--exact" ) partial="false";; + "--all" ) max_results=10000;; # hard-coded FTL limit + * ) domain=$1;; + esac + shift done -exit 0 +Main "${domain}" From ee4eb8db201a375afe3fdf9f7af1d52ce2a630cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 13 Aug 2023 22:26:44 +0200 Subject: [PATCH 076/200] chmod +x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/api.sh | 0 advanced/Scripts/query.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 advanced/Scripts/api.sh mode change 100644 => 100755 advanced/Scripts/query.sh diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh old mode 100644 new mode 100755 diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh old mode 100644 new mode 100755 From 43882693a5d72efdd2fb91b593a6b50dc9041b92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 19 Aug 2023 10:19:26 +0000 Subject: [PATCH 077/200] Bump tox from 4.7.0 to 4.9.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.7.0 to 4.9.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.7.0...4.9.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index c5d002e5..0210aa4e 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 -tox == 4.7.0 +tox == 4.9.0 From 1afc96c05550753a84ad0defe1a9bcf8c95c5970 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Aug 2023 10:46:28 +0000 Subject: [PATCH 078/200] Bump pytest-testinfra from 8.1.0 to 9.0.0 in /test Bumps [pytest-testinfra](https://github.com/pytest-dev/pytest-testinfra) from 8.1.0 to 9.0.0. - [Release notes](https://github.com/pytest-dev/pytest-testinfra/releases) - [Changelog](https://github.com/pytest-dev/pytest-testinfra/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-testinfra/compare/8.1.0...9.0.0) --- updated-dependencies: - dependency-name: pytest-testinfra dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 0210aa4e..b1b36de1 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 -pytest-testinfra == 8.1.0 +pytest-testinfra == 9.0.0 tox == 4.9.0 From e65b171aea04d8f40dceaf5fa4706a2f0b793f14 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Aug 2023 11:24:09 +0000 Subject: [PATCH 079/200] Bump tox from 4.9.0 to 4.10.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.9.0 to 4.10.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.9.0...4.10.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index b1b36de1..90255f7f 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 -tox == 4.9.0 +tox == 4.10.0 From 53e8127781645d5af39644d95c6ffd894bb8d8cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Aug 2023 11:33:09 +0000 Subject: [PATCH 080/200] Bump actions/checkout from 3.5.3 to 3.6.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.5.3 to 3.6.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.5.3...v3.6.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 7fd0e4e4..1194ba0f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 43d5ca96..13b05df2 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 5435b7db..60f38cf6 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 09462eb4..6c76e9fd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Check scripts in repository are executable run: | @@ -72,7 +72,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.0 From eca84e0986c7373ebfe057447fa244f3ecf3dba2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 21 Aug 2023 21:28:17 +0200 Subject: [PATCH 081/200] Remove user agent when downloading adlists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/gravity.sh b/gravity.sh index ed402a34..248aa2da 100755 --- a/gravity.sh +++ b/gravity.sh @@ -382,7 +382,7 @@ gravity_DownloadBlocklists() { unset sources fi - local url domain agent str target compression + local url domain str target compression echo "" # Prepare new gravity database @@ -438,9 +438,6 @@ gravity_DownloadBlocklists() { saveLocation="${piholeDir}/list.${id}.${domain}.${domainsExtension}" activeDomains[$i]="${saveLocation}" - # Default user-agent (for Cloudflare's Browser Integrity Check: https://support.cloudflare.com/hc/en-us/articles/200170086-What-does-the-Browser-Integrity-Check-do-) - agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36" - echo -e " ${INFO} Target: ${url}" local regex check_url # Check for characters NOT allowed in URLs @@ -453,7 +450,7 @@ gravity_DownloadBlocklists() { if [[ "${check_url}" =~ ${regex} ]]; then echo -e " ${CROSS} Invalid Target" else - gravity_DownloadBlocklistFromUrl "${url}" "${agent}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" + gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" fi echo "" done @@ -485,7 +482,7 @@ compareLists() { # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { - local url="${1}" agent="${2}" adlistID="${3}" saveLocation="${4}" target="${5}" compression="${6}" + local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext # Create temp file to store content on disk instead of RAM @@ -545,7 +542,7 @@ gravity_DownloadBlocklistFromUrl() { fi # shellcheck disable=SC2086 - httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${listCurlBuffer}" 2> /dev/null) + httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2> /dev/null) case $url in # Did we "download" a local file? From c33963b25000272d023ff00f1c910bab3c891429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 4 Jun 2023 22:51:10 +0200 Subject: [PATCH 082/200] Check for valid OS via IPv4 and IPv6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 17 ++++++++++++++-- automated install/basic-install.sh | 31 ++++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index c3bc81b0..622ebd7a 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -339,7 +339,7 @@ os_check() { detected_os=$(grep "\bID\b" /etc/os-release | cut -d '=' -f2 | tr -d '"') detected_version=$(grep VERSION_ID /etc/os-release | cut -d '=' -f2 | tr -d '"') - cmdResult="$(dig +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" + cmdResult="$(dig -4 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" #Get the return code of the previous command (last line) digReturnCode="${cmdResult##*$'\n'}" @@ -349,7 +349,20 @@ os_check() { if [ "${digReturnCode}" -ne 0 ]; then log_write "${INFO} Distro: ${detected_os^}" log_write "${INFO} Version: ${detected_version}" - log_write "${CROSS} dig return code: ${COL_RED}${digReturnCode}${COL_NC}" + log_write "${CROSS} dig IPv4 return code: ${COL_RED}${digReturnCode}${COL_NC}" + log_write "${CROSS} dig response: ${response}" + log_write "${INFO} Retrying via IPv6" + + cmdResult="$(dig -6 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" + #Get the return code of the previous command (last line) + digReturnCode="${cmdResult##*$'\n'}" + + # Extract dig response + response="${cmdResult%%$'\n'*}" + fi + # If also no success via IPv6 + if [ "${digReturnCode}" -ne 0 ]; then + log_write "${CROSS} dig IPv6 return code: ${COL_RED}${digReturnCode}${COL_NC}" log_write "${CROSS} dig response: ${response}" log_write "${CROSS} Error: ${COL_RED}dig command failed - Unable to check OS${COL_NC}" else diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index e3d8ff29..990b8789 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -176,7 +176,8 @@ os_check() { detected_os=$(grep '^ID=' /etc/os-release | cut -d '=' -f2 | tr -d '"') detected_version=$(grep VERSION_ID /etc/os-release | cut -d '=' -f2 | tr -d '"') - cmdResult="$(dig +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" + # Test via IPv4 + cmdResult="$(dig -4 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" # Gets the return code of the previous command (last line) digReturnCode="${cmdResult##*$'\n'}" @@ -188,8 +189,34 @@ os_check() { # If the value of ${response} is a single 0, then this is the return code, not an actual response. if [ "${response}" == 0 ]; then valid_response=false + else + valid_response=true fi + fi + + # Try again via IPv6 + if [ "$valid_response" = false ]; then + unset valid_response + + cmdResult="$(dig -6 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" + # Gets the return code of the previous command (last line) + digReturnCode="${cmdResult##*$'\n'}" + + if [ ! "${digReturnCode}" == "0" ]; then + valid_response=false + else + # Dig returned 0 (success), so get the actual response, and loop through it to determine if the detected variables above are valid + response="${cmdResult%%$'\n'*}" + # If the value of ${response} is a single 0, then this is the return code, not an actual response. + if [ "${response}" == 0 ]; then + valid_response=false + else + valid_response=true + fi + fi + fi + if [ "$valid_response" = true ]; then IFS=" " read -r -a supportedOS < <(echo "${response}" | tr -d '"') for distro_and_versions in "${supportedOS[@]}" do @@ -212,7 +239,7 @@ os_check() { done fi - if [ "$valid_os" = true ] && [ "$valid_version" = true ] && [ ! "$valid_response" = false ]; then + if [ "$valid_os" = true ] && [ "$valid_version" = true ] && [ "$valid_response" = true ]; then display_warning=false fi From ff2c2290c7390ef73a34b1c1255af12efe476b61 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Sep 2023 10:27:10 +0000 Subject: [PATCH 083/200] Bump tox from 4.10.0 to 4.11.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.10.0 to 4.11.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.10.0...4.11.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 90255f7f..7f4d7e09 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 -tox == 4.10.0 +tox == 4.11.1 From 52268f01550acf609ddac0d7726cfb3fe03e4cc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 7 Sep 2023 22:27:49 +0200 Subject: [PATCH 084/200] Ignore ABP style entries in debug log dig test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index c3bc81b0..6ecb49b4 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -809,11 +809,15 @@ dig_at() { local record_type="A" fi - # Find a random blocked url that has not been whitelisted. + # Find a random blocked url that has not been whitelisted and is not ABP style. # This helps emulate queries to different domains that a user might query # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains local random_url - random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity ORDER BY RANDOM() LIMIT 1") + random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") + # Falback if no non-ABP style domains were found + if [ -z "${random_url}" ]; then + random_url="flurry.com" + fi # Next we need to check if Pi-hole can resolve a domain when the query is sent to it's IP address # This better emulates how clients will interact with Pi-hole as opposed to above where Pi-hole is From a229a623bbb282c7aa72e4485f4820f273c7fd5c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Sep 2023 10:05:32 +0000 Subject: [PATCH 085/200] Bump tox from 4.11.1 to 4.11.3 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.11.1 to 4.11.3. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.11.1...4.11.3) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 7f4d7e09..139be0fc 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 -tox == 4.11.1 +tox == 4.11.3 From 16ea50ad552de3cb70ffc6637614b1f3ef4878e0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Sep 2023 10:17:15 +0000 Subject: [PATCH 086/200] Bump actions/checkout from 3.6.0 to 4.0.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.6.0 to 4.0.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.6.0...v4.0.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1194ba0f..570fabdd 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 13b05df2..8d09e5e0 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 60f38cf6..184319e0 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6c76e9fd..bf027210 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Check scripts in repository are executable run: | @@ -72,7 +72,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.0 From dc73ace7c46c72d5692945026ae0fea0e45f5756 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Sep 2023 15:07:06 +0000 Subject: [PATCH 087/200] Bump pytest from 7.4.0 to 7.4.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.0 to 7.4.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.0...7.4.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 139be0fc..27417754 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.0 +pytest == 7.4.2 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 tox == 4.11.3 From df92b8ac14988ddeece92b733dec7e8b9fb84d3a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 12 Sep 2023 20:43:34 +0200 Subject: [PATCH 088/200] Add missing creation of view vw_antigravity as well as schema change to vw_adlist Signed-off-by: DL6ER --- .../database_migration/gravity/15_to_16.sql | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/advanced/Scripts/database_migration/gravity/15_to_16.sql b/advanced/Scripts/database_migration/gravity/15_to_16.sql index c6159f40..7c3b8197 100644 --- a/advanced/Scripts/database_migration/gravity/15_to_16.sql +++ b/advanced/Scripts/database_migration/gravity/15_to_16.sql @@ -6,6 +6,20 @@ BEGIN TRANSACTION; ALTER TABLE adlist ADD COLUMN abp_entries INTEGER NOT NULL DEFAULT 0; +CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id + FROM antigravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id + LEFT JOIN adlist ON adlist.id = antigravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; + +DROP VIEW vw_adlist; + +CREATE VIEW vw_adlist AS SELECT DISTINCT address, id, type + FROM adlist + WHERE enabled = 1 + ORDER BY id; + UPDATE info SET value = 16 WHERE property = 'version'; COMMIT; From ea23c8364dd75190b1a8765e8a302877063b9b99 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 16 Sep 2023 10:04:29 +0200 Subject: [PATCH 089/200] Move antigravity-related changed to gravits database migration step 16->17 Signed-off-by: DL6ER --- .../database_migration/gravity/15_to_16.sql | 14 -------------- .../database_migration/gravity/16_to_17.sql | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity/15_to_16.sql b/advanced/Scripts/database_migration/gravity/15_to_16.sql index 7c3b8197..c6159f40 100644 --- a/advanced/Scripts/database_migration/gravity/15_to_16.sql +++ b/advanced/Scripts/database_migration/gravity/15_to_16.sql @@ -6,20 +6,6 @@ BEGIN TRANSACTION; ALTER TABLE adlist ADD COLUMN abp_entries INTEGER NOT NULL DEFAULT 0; -CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id - FROM antigravity - LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id - LEFT JOIN adlist ON adlist.id = antigravity.adlist_id - LEFT JOIN "group" ON "group".id = adlist_by_group.group_id - WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; - -DROP VIEW vw_adlist; - -CREATE VIEW vw_adlist AS SELECT DISTINCT address, id, type - FROM adlist - WHERE enabled = 1 - ORDER BY id; - UPDATE info SET value = 16 WHERE property = 'version'; COMMIT; diff --git a/advanced/Scripts/database_migration/gravity/16_to_17.sql b/advanced/Scripts/database_migration/gravity/16_to_17.sql index c7b9049b..23532e3a 100644 --- a/advanced/Scripts/database_migration/gravity/16_to_17.sql +++ b/advanced/Scripts/database_migration/gravity/16_to_17.sql @@ -8,6 +8,20 @@ ALTER TABLE adlist ADD COLUMN type INTEGER NOT NULL DEFAULT 0; UPDATE adlist SET type = 0; +CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id + FROM antigravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id + LEFT JOIN adlist ON adlist.id = antigravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; + +DROP VIEW vw_adlist; + +CREATE VIEW vw_adlist AS SELECT DISTINCT address, id, type + FROM adlist + WHERE enabled = 1 + ORDER BY id; + UPDATE info SET value = 17 WHERE property = 'version'; COMMIT; From ec9d84692f55c3069ba895631437a050c7199086 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 23 Sep 2023 10:45:09 +0000 Subject: [PATCH 090/200] Bump actions/checkout from 4.0.0 to 4.1.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.0.0 to 4.1.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.0.0...v4.1.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 570fabdd..4685aa2c 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 8d09e5e0..0ff0a24a 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 184319e0..a1025629 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bf027210..054c09ac 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Check scripts in repository are executable run: | @@ -72,7 +72,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.0 From aba41b45b0e7ff67ed88cb6f706bdddb3bbadeb9 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 4 Oct 2023 16:20:38 +0100 Subject: [PATCH 091/200] Some verbiage change to outputs Signed-off-by: Adam Warner --- gravity.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index eced477d..9133d33d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -675,10 +675,10 @@ gravity_ShowCount() { # Here we use the table "gravity" instead of the view "vw_gravity" for speed. # It's safe to replace it here, because right after a gravity run both will show the exactly same number of domains. gravity_Table_Count "gravity" "gravity domains" "" - gravity_Table_Count "vw_blacklist" "exact blacklisted domains" - gravity_Table_Count "vw_regex_blacklist" "regex blacklist filters" - gravity_Table_Count "vw_whitelist" "exact whitelisted domains" - gravity_Table_Count "vw_regex_whitelist" "regex whitelist filters" + gravity_Table_Count "vw_blacklist" "exact denied domains" + gravity_Table_Count "vw_regex_blacklist" "regex denied filters" + gravity_Table_Count "vw_whitelist" "exact allowed domains" + gravity_Table_Count "vw_regex_whitelist" "regex allowed filters" } # Create "localhost" entries into hosts format From 885b626a68099bd44eb952307857438513a45cc9 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 4 Oct 2023 16:22:01 +0100 Subject: [PATCH 092/200] Some unrelated spelling mistakes that spellcheck is grumbling about Signed-off-by: Adam Warner --- advanced/Scripts/piholeDebug.sh | 6 +++--- automated install/basic-install.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 6ecb49b4..0e3bbf3d 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -397,7 +397,7 @@ os_check() { } diagnose_operating_system() { - # error message in a variable so we can easily modify it later (or re-use it) + # error message in a variable so we can easily modify it later (or reuse it) local error_msg="Distribution unknown -- most likely you are on an unsupported platform and may run into issues." # Display the current test that is running echo_current_diagnostic "Operating system" @@ -814,7 +814,7 @@ dig_at() { # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains local random_url random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") - # Falback if no non-ABP style domains were found + # Fallback if no non-ABP style domains were found if [ -z "${random_url}" ]; then random_url="flurry.com" fi @@ -1451,7 +1451,7 @@ upload_to_tricorder() { # If no token was generated else # Show an error and some help instructions - # Skip this if being called from web interface and autmatic mode was not chosen (users opt-out to upload) + # Skip this if being called from web interface and automatic mode was not chosen (users opt-out to upload) if [[ "${WEBCALL}" ]] && [[ ! "${AUTOMATED}" ]]; then : else diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index bf26631a..4e4bdfc7 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2203,7 +2203,7 @@ main() { # Check for and disable systemd-resolved-DNSStubListener before reloading resolved # DNSStubListener needs to remain in place for installer to download needed files, # so this change needs to be made after installation is complete, - # but before starting or resarting the ftl service + # but before starting or restarting the ftl service disable_resolved_stublistener printf " %b Restarting services...\\n" "${INFO}" From 70547755d661add4df58a39b18db01543cac2141 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Oct 2023 10:39:32 +0000 Subject: [PATCH 093/200] Bump actions/setup-python from 4.7.0 to 4.7.1 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.7.0 to 4.7.1. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.7.0...v4.7.1) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 054c09ac..9f32302e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,7 +75,7 @@ jobs: uses: actions/checkout@v4.1.0 - name: Set up Python 3.10 - uses: actions/setup-python@v4.7.0 + uses: actions/setup-python@v4.7.1 with: python-version: "3.10" From 044e856e6bd84a128b46930fb8a1a3489cf16ad1 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 8 Oct 2023 13:23:44 +0100 Subject: [PATCH 094/200] Disable checkout function for (official) docker containers Signed-off-by: Adam Warner --- pihole | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/pihole b/pihole index 54b20f7d..5281c715 100755 --- a/pihole +++ b/pihole @@ -413,26 +413,30 @@ tailFunc() { } piholeCheckoutFunc() { - if [[ "$2" == "-h" ]] || [[ "$2" == "--help" ]]; then - echo "Usage: pihole checkout [repo] [branch] -Example: 'pihole checkout master' or 'pihole checkout core dev' -Switch Pi-hole subsystems to a different GitHub branch - -Repositories: - core [branch] Change the branch of Pi-hole's core subsystem - web [branch] Change the branch of Web Interface subsystem - ftl [branch] Change the branch of Pi-hole's FTL subsystem - -Branches: - master Update subsystems to the latest stable release - dev Update subsystems to the latest development release - branchname Update subsystems to the specified branchname" - exit 0 - fi + if [ -n "${DOCKER_VERSION}" ]; then + unsupportedFunc + else + if [[ "$2" == "-h" ]] || [[ "$2" == "--help" ]]; then + echo "Usage: pihole checkout [repo] [branch] + Example: 'pihole checkout master' or 'pihole checkout core dev' + Switch Pi-hole subsystems to a different GitHub branch + + Repositories: + core [branch] Change the branch of Pi-hole's core subsystem + web [branch] Change the branch of Web Interface subsystem + ftl [branch] Change the branch of Pi-hole's FTL subsystem + + Branches: + master Update subsystems to the latest stable release + dev Update subsystems to the latest development release + branchname Update subsystems to the specified branchname" + exit 0 + fi - source "${PI_HOLE_SCRIPT_DIR}"/piholeCheckout.sh - shift - checkout "$@" + source "${PI_HOLE_SCRIPT_DIR}"/piholeCheckout.sh + shift + checkout "$@" + fi } tricorderFunc() { From 7886dc017266a63041bbbf2f3c751452a6661d9e Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 8 Oct 2023 14:09:47 +0100 Subject: [PATCH 095/200] adminlte->web Signed-off-by: Adam Warner --- README.md | 2 +- advanced/Scripts/chronometer.sh | 2 +- advanced/Scripts/update.sh | 2 +- advanced/Scripts/updatecheck.sh | 4 ++-- advanced/Scripts/version.sh | 16 ++++++++-------- automated install/basic-install.sh | 4 ++-- manpages/pihole.8 | 4 ++-- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index adfd3450..eb50030b 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ You can read our [Core Feature Breakdown](https://docs.pi-hole.net/core/pihole-c ### The Web Interface Dashboard -This [optional dashboard](https://github.com/pi-hole/AdminLTE) allows you to view stats, change settings, and configure your Pi-hole. It's the power of the Command Line Interface, with none of the learning curve! +This [optional dashboard](https://github.com/pi-hole/web) allows you to view stats, change settings, and configure your Pi-hole. It's the power of the Command Line Interface, with none of the learning curve! Some notable features include: diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh index 49de6efd..fc728e17 100755 --- a/advanced/Scripts/chronometer.sh +++ b/advanced/Scripts/chronometer.sh @@ -225,7 +225,7 @@ get_sys_stats() { if [[ -n "${ph_ver_raw[0]}" ]]; then ph_core_ver="${ph_ver_raw[0]}" if [[ ${#ph_ver_raw[@]} -eq 2 ]]; then - # AdminLTE not installed + # web not installed ph_lte_ver="(not installed)" ph_ftl_ver="${ph_ver_raw[1]}" else diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index b6153293..9dae66df 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -11,7 +11,7 @@ # Please see LICENSE file for your rights under this license. # Variables -readonly ADMIN_INTERFACE_GIT_URL="https://github.com/pi-hole/AdminLTE.git" +readonly ADMIN_INTERFACE_GIT_URL="https://github.com/pi-hole/web.git" readonly ADMIN_INTERFACE_DIR="/var/www/html/admin" readonly PI_HOLE_GIT_URL="https://github.com/pi-hole/pi-hole.git" readonly PI_HOLE_FILES_DIR="/etc/.pihole" diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 7d7103d2..8bb1888b 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -91,10 +91,10 @@ addOrEditKeyValPair "${VERSION_FILE}" "WEB_BRANCH" "${WEB_BRANCH}" WEB_HASH="$(get_local_hash /var/www/html/admin)" addOrEditKeyValPair "${VERSION_FILE}" "WEB_HASH" "${WEB_HASH}" -GITHUB_WEB_VERSION="$(get_remote_version AdminLTE)" +GITHUB_WEB_VERSION="$(get_remote_version web)" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_VERSION" "${GITHUB_WEB_VERSION}" -GITHUB_WEB_HASH="$(get_remote_hash AdminLTE "${WEB_BRANCH}")" +GITHUB_WEB_HASH="$(get_remote_hash web "${WEB_BRANCH}")" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_HASH" "${GITHUB_WEB_HASH}" # get FTL versions diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index af86b045..e3b4a6ae 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -24,7 +24,7 @@ fi getLocalVersion() { case ${1} in "Pi-hole" ) echo "${CORE_VERSION:=N/A}";; - "AdminLTE" ) echo "${WEB_VERSION:=N/A}";; + "web" ) echo "${WEB_VERSION:=N/A}";; "FTL" ) echo "${FTL_VERSION:=N/A}";; esac } @@ -32,7 +32,7 @@ getLocalVersion() { getLocalHash() { case ${1} in "Pi-hole" ) echo "${CORE_HASH:=N/A}";; - "AdminLTE" ) echo "${WEB_HASH:=N/A}";; + "web" ) echo "${WEB_HASH:=N/A}";; "FTL" ) echo "${FTL_HASH:=N/A}";; esac } @@ -40,7 +40,7 @@ getLocalHash() { getRemoteHash(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_HASH:=N/A}";; - "AdminLTE" ) echo "${GITHUB_WEB_HASH:=N/A}";; + "web" ) echo "${GITHUB_WEB_HASH:=N/A}";; "FTL" ) echo "${GITHUB_FTL_HASH:=N/A}";; esac } @@ -48,7 +48,7 @@ getRemoteHash(){ getRemoteVersion(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_VERSION:=N/A}";; - "AdminLTE" ) echo "${GITHUB_WEB_VERSION:=N/A}";; + "web" ) echo "${GITHUB_WEB_VERSION:=N/A}";; "FTL" ) echo "${GITHUB_FTL_VERSION:=N/A}";; esac } @@ -56,7 +56,7 @@ getRemoteVersion(){ getLocalBranch(){ case ${1} in "Pi-hole" ) echo "${CORE_BRANCH:=N/A}";; - "AdminLTE" ) echo "${WEB_BRANCH:=N/A}";; + "web" ) echo "${WEB_BRANCH:=N/A}";; "FTL" ) echo "${FTL_BRANCH:=N/A}";; esac } @@ -107,7 +107,7 @@ errorOutput() { defaultOutput() { versionOutput "Pi-hole" "$@" - versionOutput "AdminLTE" "$@" + versionOutput "web" "$@" versionOutput "FTL" "$@" } @@ -118,7 +118,7 @@ Show Pi-hole, Admin Console & FTL versions Repositories: -p, --pihole Only retrieve info regarding Pi-hole repository - -a, --admin Only retrieve info regarding AdminLTE repository + -a, --admin Only retrieve info regarding web repository -f, --ftl Only retrieve info regarding FTL repository Options: @@ -131,7 +131,7 @@ Options: case "${1}" in "-p" | "--pihole" ) shift; versionOutput "Pi-hole" "$@";; - "-a" | "--admin" ) shift; versionOutput "AdminLTE" "$@";; + "-a" | "--admin" ) shift; versionOutput "web" "$@";; "-f" | "--ftl" ) shift; versionOutput "FTL" "$@";; "-h" | "--help" ) helpFunc;; * ) defaultOutput "$@";; diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4e4bdfc7..ef2c8d52 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -64,11 +64,11 @@ webroot="/var/www/html" # We clone (or update) two git repositories during the install. This helps to make sure that we always have the latest versions of the relevant files. -# AdminLTE is used to set up the Web admin interface. +# web is used to set up the Web admin interface. # Pi-hole contains various setup scripts and files which are critical to the installation. # Search for "PI_HOLE_LOCAL_REPO" in this file to see all such scripts. # Two notable scripts are gravity.sh (used to generate the HOSTS file) and advanced/Scripts/webpage.sh (used to install the Web admin interface) -webInterfaceGitUrl="https://github.com/pi-hole/AdminLTE.git" +webInterfaceGitUrl="https://github.com/pi-hole/web.git" webInterfaceDir="${webroot}/admin" piholeGitUrl="https://github.com/pi-hole/pi-hole.git" PI_HOLE_LOCAL_REPO="/etc/.pihole" diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 1cf8ab35..fec1fa5e 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -212,7 +212,7 @@ Available commands and options: .br -p, --pihole Only retrieve info regarding Pi-hole repository .br - -a, --admin Only retrieve info regarding AdminLTE + -a, --admin Only retrieve info regarding web repository .br -f, --ftl Only retrieve info regarding FTL repository @@ -339,7 +339,7 @@ Displaying version information \fBpihole -v -a -c\fR .br - Display the current version of AdminLTE + Display the current version of web .br Temporarily disabling Pi-hole From 6001fe34ec3d3ad652a7013f467d1948efb3e096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 8 Oct 2023 23:03:33 +0200 Subject: [PATCH 096/200] Set owner of gravity output files to pihole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/gravity.sh b/gravity.sh index 9133d33d..4f87c1d7 100755 --- a/gravity.sh +++ b/gravity.sh @@ -488,6 +488,10 @@ compareLists() { # We assume here it was changed upstream database_adlist_status "${adlistID}" "1" fi + + # set owner of the file to pihole + chown pihole:pihole "${target}.sha1" + } # Download specified URL and perform checks on HTTP status and file content @@ -621,6 +625,9 @@ gravity_DownloadBlocklistFromUrl() { database_adlist_status "${adlistID}" "4" fi fi + + # set owner of the file to pihole + chown pihole:pihole "${saveLocation}" } # Parse source files into domains format From 679aab10d0400d55470a5d53215158490736068a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 9 Oct 2023 21:52:43 +0200 Subject: [PATCH 097/200] Run gravity as user pihole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- gravity.sh | 7 ------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ef2c8d52..ae605fcf 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1385,8 +1385,8 @@ installCron() { # Gravity is a very important script as it aggregates all of the domains into a single HOSTS formatted list, # which is what Pi-hole needs to begin blocking ads runGravity() { - # Run gravity in the current shell - { /opt/pihole/gravity.sh --force; } + # Run gravity in the current shell as user pihole + { exec sudo -u pihole bash /opt/pihole/gravity.sh --force; } } # Check if the pihole user exists and create if it does not diff --git a/gravity.sh b/gravity.sh index 4f87c1d7..9133d33d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -488,10 +488,6 @@ compareLists() { # We assume here it was changed upstream database_adlist_status "${adlistID}" "1" fi - - # set owner of the file to pihole - chown pihole:pihole "${target}.sha1" - } # Download specified URL and perform checks on HTTP status and file content @@ -625,9 +621,6 @@ gravity_DownloadBlocklistFromUrl() { database_adlist_status "${adlistID}" "4" fi fi - - # set owner of the file to pihole - chown pihole:pihole "${saveLocation}" } # Parse source files into domains format From 22863845a0018708aaf8b215cd00666e87fceebf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 9 Oct 2023 22:05:01 +0200 Subject: [PATCH 098/200] Set owner/group of /etc/pihole to pihole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ae605fcf..4a938dc7 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2141,8 +2141,8 @@ main() { if [[ "${useUpdateVars}" == false ]]; then # Display welcome dialogs welcomeDialogs - # Create directory for Pi-hole storage - install -d -m 755 /etc/pihole/ + # Create directory for Pi-hole storage (/etc/pihole/) + install -o pihole -g pihole -d -m 660 "${PI_HOLE_CONFIG_DIR}" # Determine available interfaces get_available_interfaces # Find interfaces and let the user choose one From 2061f3a70e7f0e64709e330552ba8e85671080ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 9 Oct 2023 22:35:02 +0200 Subject: [PATCH 099/200] Set owner/permissions at the right place MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4a938dc7..efe1d4f3 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1085,12 +1085,15 @@ installScripts() { installConfigs() { printf "\\n %b Installing configs from %s...\\n" "${INFO}" "${PI_HOLE_LOCAL_REPO}" + # Ensure that permissions are correctly set + chown -R pihole:pihole /etc/pihole # Install list of DNS servers # Format: Name;Primary IPv4;Secondary IPv4;Primary IPv6;Secondary IPv6 # Some values may be empty (for example: DNS servers without IPv6 support) echo "${DNS_SERVERS}" > "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" chmod 644 "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" + chown pihole:pihole "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" # Install empty custom.list file if it does not exist if [[ ! -r "${PI_HOLE_CONFIG_DIR}/custom.list" ]]; then @@ -1386,7 +1389,7 @@ installCron() { # which is what Pi-hole needs to begin blocking ads runGravity() { # Run gravity in the current shell as user pihole - { exec sudo -u pihole bash /opt/pihole/gravity.sh --force; } + { sudo -u pihole bash /opt/pihole/gravity.sh --force; } } # Check if the pihole user exists and create if it does not @@ -1480,7 +1483,7 @@ installLogrotate() { return 2 fi # Copy the file over from the local repo - install -D -m 644 -T "${PI_HOLE_LOCAL_REPO}"/advanced/Templates/logrotate ${target} + install -o pihole -g pihole -D -m 644 -T "${PI_HOLE_LOCAL_REPO}"/advanced/Templates/logrotate ${target} # Different operating systems have different user / group # settings for logrotate that makes it impossible to create # a static logrotate file that will work with e.g. @@ -2049,6 +2052,7 @@ copy_to_install_log() { # Since we use color codes such as '\e[1;33m', they should be removed sed 's/\[[0-9;]\{1,5\}m//g' < /proc/$$/fd/3 > "${installLogLoc}" chmod 644 "${installLogLoc}" + chown pihole:pihole "${installLogLoc}" } main() { @@ -2142,7 +2146,7 @@ main() { # Display welcome dialogs welcomeDialogs # Create directory for Pi-hole storage (/etc/pihole/) - install -o pihole -g pihole -d -m 660 "${PI_HOLE_CONFIG_DIR}" + install -d -m 755 "${PI_HOLE_CONFIG_DIR}" # Determine available interfaces get_available_interfaces # Find interfaces and let the user choose one From 46ff25734485c9f960da0c4312a6c5ebc5e0f8a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 12 Oct 2023 13:44:51 +0200 Subject: [PATCH 100/200] Remove Chronometer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/chronometer.sh | 569 ----------------------------- advanced/bash-completion/pihole | 8 +- automated install/basic-install.sh | 2 +- manpages/pihole.8 | 14 - pihole | 6 +- 5 files changed, 5 insertions(+), 594 deletions(-) delete mode 100755 advanced/Scripts/chronometer.sh diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh deleted file mode 100755 index fc728e17..00000000 --- a/advanced/Scripts/chronometer.sh +++ /dev/null @@ -1,569 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC1090,SC1091 -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Calculates stats and displays to an LCD -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. -LC_ALL=C -LC_NUMERIC=C - -# Retrieve stats from FTL engine -pihole-FTL() { - local ftl_port LINE - # shellcheck disable=SC1091 - . /opt/pihole/utils.sh - ftl_port=$(getFTLConfigValue dns.port) - if [[ -n "$ftl_port" ]]; then - # Open connection to FTL - exec 3<>"/dev/tcp/127.0.0.1/$ftl_port" - - # Test if connection is open - if { "true" >&3; } 2> /dev/null; then - # Send command to FTL and ask to quit when finished - echo -e ">$1 >quit" >&3 - - # Read input until we received an empty string and the connection is - # closed - read -r -t 1 LINE <&3 - until [[ -z "${LINE}" ]] && [[ ! -t 3 ]]; do - echo "$LINE" >&1 - read -r -t 1 LINE <&3 - done - - # Close connection - exec 3>&- - exec 3<&- - fi - else - echo "0" - fi -} - -# Print spaces to align right-side additional text -printFunc() { - local text_last - - title="$1" - title_len="${#title}" - - text_main="$2" - text_main_nocol="$text_main" - if [[ "${text_main:0:1}" == "" ]]; then - text_main_nocol=$(sed 's/\[[0-9;]\{1,5\}m//g' <<< "$text_main") - fi - text_main_len="${#text_main_nocol}" - - text_addn="$3" - if [[ "$text_addn" == "last" ]]; then - text_addn="" - text_last="true" - fi - - # If there is additional text, define max length of text_main - if [[ -n "$text_addn" ]]; then - case "$scr_cols" in - [0-9]|1[0-9]|2[0-9]|3[0-9]|4[0-4]) text_main_max_len="9";; - 4[5-9]) text_main_max_len="14";; - *) text_main_max_len="19";; - esac - fi - - [[ -z "$text_addn" ]] && text_main_max_len="$(( scr_cols - title_len ))" - - # Remove excess characters from main text - if [[ "$text_main_len" -gt "$text_main_max_len" ]]; then - # Trim text without colors - text_main_trim="${text_main_nocol:0:$text_main_max_len}" - # Replace with trimmed text - text_main="${text_main/$text_main_nocol/$text_main_trim}" - fi - - # Determine amount of spaces for each line - if [[ -n "$text_last" ]]; then - # Move cursor to end of screen - spc_num=$(( scr_cols - ( title_len + text_main_len ) )) - else - spc_num=$(( text_main_max_len - text_main_len )) - fi - - [[ "$spc_num" -le 0 ]] && spc_num="0" - spc=$(printf "%${spc_num}s") - #spc="${spc// /.}" # Debug: Visualize spaces - - printf "%s%s$spc" "$title" "$text_main" - - if [[ -n "$text_addn" ]]; then - printf "%s(%s)%s\\n" "$COL_NC$COL_DARK_GRAY" "$text_addn" "$COL_NC" - else - # Do not print trailing newline on final line - [[ -z "$text_last" ]] && printf "%s\\n" "$COL_NC" - fi -} - -# Perform on first Chrono run (not for JSON formatted string) -get_init_stats() { - calcFunc(){ awk "BEGIN {print $*}" 2> /dev/null; } - - # Convert bytes to human-readable format - hrBytes() { - awk '{ - num=$1; - if(num==0) { - print "0 B" - } else { - xxx=(num<0?-num:num) - sss=(num<0?-1:1) - split("B KB MB GB TB PB",type) - for(i=5;yyy < 1;i--) { - yyy=xxx / (2^(10*i)) - } - printf "%.0f " type[i+2], yyy*sss - } - }' <<< "$1"; - } - - # Convert seconds to human-readable format - hrSecs() { - day=$(( $1/60/60/24 )); hrs=$(( $1/3600%24 )) - mins=$(( ($1%3600)/60 )); secs=$(( $1%60 )) - [[ "$day" -ge "2" ]] && plu="s" - [[ "$day" -ge "1" ]] && days="$day day${plu}, " || days="" - printf "%s%02d:%02d:%02d\\n" "$days" "$hrs" "$mins" "$secs" - } - - # Set Color Codes - coltable="/opt/pihole/COL_TABLE" - if [[ -f "${coltable}" ]]; then - source ${coltable} - else - COL_NC="" - COL_DARK_GRAY="" - COL_LIGHT_GREEN="" - COL_LIGHT_BLUE="" - COL_LIGHT_RED="" - COL_YELLOW="" - COL_LIGHT_RED="" - COL_URG_RED="" - fi - - # Get RPi throttle state (RPi 3B only) & model number, or OS distro info - if command -v vcgencmd &> /dev/null; then - local sys_throttle_raw - local sys_rev_raw - - sys_throttle_raw=$(vgt=$(sudo vcgencmd get_throttled); echo "${vgt##*x}") - - # Active Throttle Notice: https://bit.ly/2gnunOo - if [[ "$sys_throttle_raw" != "0" ]]; then - case "$sys_throttle_raw" in - *0001) thr_type="${COL_YELLOW}Under Voltage";; - *0002) thr_type="${COL_LIGHT_BLUE}Arm Freq Cap";; - *0003) thr_type="${COL_YELLOW}UV${COL_DARK_GRAY},${COL_NC} ${COL_LIGHT_BLUE}AFC";; - *0004) thr_type="${COL_LIGHT_RED}Throttled";; - *0005) thr_type="${COL_YELLOW}UV${COL_DARK_GRAY},${COL_NC} ${COL_LIGHT_RED}TT";; - *0006) thr_type="${COL_LIGHT_BLUE}AFC${COL_DARK_GRAY},${COL_NC} ${COL_LIGHT_RED}TT";; - *0007) thr_type="${COL_YELLOW}UV${COL_DARK_GRAY},${COL_NC} ${COL_LIGHT_BLUE}AFC${COL_DARK_GRAY},${COL_NC} ${COL_LIGHT_RED}TT";; - esac - [[ -n "$thr_type" ]] && sys_throttle="$thr_type${COL_DARK_GRAY}" - fi - - sys_rev_raw=$(awk '/Revision/ {print $3}' < /proc/cpuinfo) - case "$sys_rev_raw" in - 000[2-6]) sys_model=" 1, Model B";; # 256MB - 000[7-9]) sys_model=" 1, Model A";; # 256MB - 000d|000e|000f) sys_model=" 1, Model B";; # 512MB - 0010|0013) sys_model=" 1, Model B+";; # 512MB - 0012|0015) sys_model=" 1, Model A+";; # 256MB - a0104[0-1]|a21041|a22042) sys_model=" 2, Model B";; # 1GB - 900021) sys_model=" 1, Model A+";; # 512MB - 900032) sys_model=" 1, Model B+";; # 512MB - 90009[2-3]|920093) sys_model=" Zero";; # 512MB - 9000c1) sys_model=" Zero W";; # 512MB - a02082|a[2-3]2082) sys_model=" 3, Model B";; # 1GB - a020d3) sys_model=" 3, Model B+";; # 1GB - *) sys_model="";; - esac - sys_type="Raspberry Pi$sys_model" - else - source "/etc/os-release" - CODENAME=$(sed 's/[()]//g' <<< "${VERSION/* /}") - sys_type="${NAME/ */} ${CODENAME^} $VERSION_ID" - fi - - # Get core count - sys_cores=$(grep -c "^processor" /proc/cpuinfo) - - # Test existence of clock speed file for ARM CPU - if [[ -f "/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq" ]]; then - scaling_freq_file="/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq" - fi - - # Test existence of temperature file - if [[ -f "/sys/class/thermal/thermal_zone0/temp" ]]; then - temp_file="/sys/class/thermal/thermal_zone0/temp" - elif [[ -f "/sys/class/hwmon/hwmon0/temp1_input" ]]; then - temp_file="/sys/class/hwmon/hwmon0/temp1_input" - else - temp_file="" - fi -} - -get_sys_stats() { - local ph_ver_raw - local cpu_raw - local ram_raw - local disk_raw - - # Update every 12 refreshes (Def: every 60s) - count=$((count+1)) - if [[ "$count" == "1" ]] || (( "$count" % 12 == 0 )); then - mapfile -t ph_ver_raw < <(pihole -v -c 2> /dev/null | sed -n 's/^.* v/v/p') - if [[ -n "${ph_ver_raw[0]}" ]]; then - ph_core_ver="${ph_ver_raw[0]}" - if [[ ${#ph_ver_raw[@]} -eq 2 ]]; then - # web not installed - ph_lte_ver="(not installed)" - ph_ftl_ver="${ph_ver_raw[1]}" - else - ph_lte_ver="${ph_ver_raw[1]}" - ph_ftl_ver="${ph_ver_raw[2]}" - fi - else - ph_core_ver="-1" - fi - - sys_name=$(hostname) - - [[ -n "$TEMPERATUREUNIT" ]] && temp_unit="${TEMPERATUREUNIT^^}" || temp_unit="C" - - # Get storage stats for partition mounted on / - read -r -a disk_raw <<< "$(df -B1 / 2> /dev/null | awk 'END{ print $3,$2,$5 }')" - disk_used="${disk_raw[0]}" - disk_total="${disk_raw[1]}" - disk_perc="${disk_raw[2]}" - - net_gateway=$(ip route | grep default | cut -d ' ' -f 3 | head -n 1) - - # Get DHCP stats, if feature is enabled - if [[ "$DHCP_ACTIVE" == "true" ]]; then - ph_dhcp_max=$(( ${DHCP_END##*.} - ${DHCP_START##*.} + 1 )) - fi - - # Get DNS server count - dns_count="0" - [[ -n "${PIHOLE_DNS_1}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_2}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_3}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_4}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_5}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_6}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_7}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_8}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_9}" ]] && dns_count="$dns_count+" - fi - - # Get screen size - read -r -a scr_size <<< "$(stty size 2>/dev/null || echo 24 80)" - scr_lines="${scr_size[0]}" - scr_cols="${scr_size[1]}" - - # Determine Chronometer size behavior - if [[ "$scr_cols" -ge 58 ]]; then - chrono_width="large" - elif [[ "$scr_cols" -gt 40 ]]; then - chrono_width="medium" - else - chrono_width="small" - fi - - # Determine max length of divider string - scr_line_len=$(( scr_cols - 2 )) - [[ "$scr_line_len" -ge 58 ]] && scr_line_len="58" - scr_line_str=$(printf "%${scr_line_len}s") - scr_line_str="${scr_line_str// /—}" - - sys_uptime=$(hrSecs "$(cut -d. -f1 /proc/uptime)") - sys_loadavg=$(cut -d " " -f1,2,3 /proc/loadavg) - - # Get CPU usage, only counting processes over 1% as active - # shellcheck disable=SC2009 - cpu_raw=$(ps -eo pcpu,rss --no-headers | grep -E -v " 0") - cpu_tasks=$(wc -l <<< "$cpu_raw") - cpu_taskact=$(sed -r "/(^ 0.)/d" <<< "$cpu_raw" | wc -l) - cpu_perc=$(awk '{sum+=$1} END {printf "%.0f\n", sum/'"$sys_cores"'}' <<< "$cpu_raw") - - # Get CPU clock speed - if [[ -n "$scaling_freq_file" ]]; then - cpu_mhz=$(( $(< /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq) / 1000 )) - else - cpu_mhz=$(lscpu | awk -F ":" '/MHz/ {print $2;exit}') - cpu_mhz=$(printf "%.0f" "${cpu_mhz//[[:space:]]/}") - fi - - # Determine whether to display CPU clock speed as MHz or GHz - if [[ -n "$cpu_mhz" ]]; then - [[ "$cpu_mhz" -le "999" ]] && cpu_freq="$cpu_mhz MHz" || cpu_freq="$(printf "%.1f" $(calcFunc "$cpu_mhz"/1000)) GHz" - [[ "${cpu_freq}" == *".0"* ]] && cpu_freq="${cpu_freq/.0/}" - fi - - # Determine color for temperature - if [[ -n "$temp_file" ]]; then - if [[ "$temp_unit" == "C" ]]; then - cpu_temp=$(printf "%.0fc\\n" "$(calcFunc "$(< $temp_file) / 1000")") - - case "${cpu_temp::-1}" in - -*|[0-9]|[1-3][0-9]) cpu_col="$COL_LIGHT_BLUE";; - 4[0-9]) cpu_col="";; - 5[0-9]) cpu_col="$COL_YELLOW";; - 6[0-9]) cpu_col="$COL_LIGHT_RED";; - *) cpu_col="$COL_URG_RED";; - esac - - # $COL_NC$COL_DARK_GRAY is needed for $COL_URG_RED - cpu_temp_str=" @ $cpu_col$cpu_temp$COL_NC$COL_DARK_GRAY" - - elif [[ "$temp_unit" == "F" ]]; then - cpu_temp=$(printf "%.0ff\\n" "$(calcFunc "($(< $temp_file) / 1000) * 9 / 5 + 32")") - - case "${cpu_temp::-1}" in - -*|[0-9]|[0-9][0-9]) cpu_col="$COL_LIGHT_BLUE";; - 1[0-1][0-9]) cpu_col="";; - 1[2-3][0-9]) cpu_col="$COL_YELLOW";; - 1[4-5][0-9]) cpu_col="$COL_LIGHT_RED";; - *) cpu_col="$COL_URG_RED";; - esac - - cpu_temp_str=" @ $cpu_col$cpu_temp$COL_NC$COL_DARK_GRAY" - - else - cpu_temp_str=$(printf " @ %.0fk\\n" "$(calcFunc "($(< $temp_file) / 1000) + 273.15")") - fi - else - cpu_temp_str="" - fi - - read -r -a ram_raw <<< "$(awk '/MemTotal:/{total=$2} /MemFree:/{free=$2} /Buffers:/{buffers=$2} /^Cached:/{cached=$2} END {printf "%.0f %.0f %.0f", (total-free-buffers-cached)*100/total, (total-free-buffers-cached)*1024, total*1024}' /proc/meminfo)" - ram_perc="${ram_raw[0]}" - ram_used="${ram_raw[1]}" - ram_total="${ram_raw[2]}" - - if [[ "$(pihole status web 2> /dev/null)" -ge "1" ]]; then - ph_status="${COL_LIGHT_GREEN}Active" - else - ph_status="${COL_LIGHT_RED}Offline" - fi - - if [[ "$DHCP_ACTIVE" == "true" ]]; then - local ph_dhcp_range - - ph_dhcp_range=$(seq -s "|" -f "${DHCP_START%.*}.%g" "${DHCP_START##*.}" "${DHCP_END##*.}") - - # Count dynamic leases from available range, and not static leases - ph_dhcp_num=$(grep -cE "$ph_dhcp_range" "/etc/pihole/dhcp.leases") - ph_dhcp_percent=$(( ph_dhcp_num * 100 / ph_dhcp_max )) - fi -} - -get_ftl_stats() { - local stats_raw - - mapfile -t stats_raw < <(pihole-FTL "stats") - domains_being_blocked_raw="${stats_raw[0]#* }" - dns_queries_today_raw="${stats_raw[1]#* }" - ads_blocked_today_raw="${stats_raw[2]#* }" - ads_percentage_today_raw="${stats_raw[3]#* }" - queries_forwarded_raw="${stats_raw[5]#* }" - queries_cached_raw="${stats_raw[6]#* }" - - # Only retrieve these stats when not called from jsonFunc - if [[ -z "$1" ]]; then - local top_ad_raw - local top_domain_raw - local top_client_raw - - domains_being_blocked=$(printf "%.0f\\n" "${domains_being_blocked_raw}" 2> /dev/null) - dns_queries_today=$(printf "%.0f\\n" "${dns_queries_today_raw}") - ads_blocked_today=$(printf "%.0f\\n" "${ads_blocked_today_raw}") - ads_percentage_today=$(printf "%'.0f\\n" "${ads_percentage_today_raw}") - queries_cached_percentage=$(printf "%.0f\\n" "$(calcFunc "$queries_cached_raw * 100 / ( $queries_forwarded_raw + $queries_cached_raw )")") - recent_blocked=$(pihole-FTL recentBlocked) - read -r -a top_ad_raw <<< "$(pihole-FTL "top-ads (1)")" - read -r -a top_domain_raw <<< "$(pihole-FTL "top-domains (1)")" - read -r -a top_client_raw <<< "$(pihole-FTL "top-clients (1)")" - - top_ad="${top_ad_raw[2]}" - top_domain="${top_domain_raw[2]}" - if [[ "${top_client_raw[3]}" ]]; then - top_client="${top_client_raw[3]}" - else - top_client="${top_client_raw[2]}" - fi - fi -} - -get_strings() { - # Expand or contract strings depending on screen size - if [[ "$chrono_width" == "large" ]]; then - phc_str=" ${COL_DARK_GRAY}Core" - lte_str=" ${COL_DARK_GRAY}Web" - ftl_str=" ${COL_DARK_GRAY}FTL" - api_str="${COL_LIGHT_RED}API Offline" - - host_info="$sys_type" - sys_info="$sys_throttle" - sys_info2="Active: $cpu_taskact of $cpu_tasks tasks" - used_str="Used: " - leased_str="Leased: " - domains_being_blocked=$(printf "%'.0f" "$domains_being_blocked") - ads_blocked_today=$(printf "%'.0f" "$ads_blocked_today") - dns_queries_today=$(printf "%'.0f" "$dns_queries_today") - ph_info="Blocking: $domains_being_blocked sites" - total_str="Total: " - else - phc_str=" ${COL_DARK_GRAY}Core" - lte_str=" ${COL_DARK_GRAY}Web" - ftl_str=" ${COL_DARK_GRAY}FTL" - api_str="${COL_LIGHT_RED}API Down" - ph_info="$domains_being_blocked blocked" - fi - - [[ "$sys_cores" -ne 1 ]] && sys_cores_txt="${sys_cores}x " - cpu_info="$sys_cores_txt$cpu_freq$cpu_temp_str" - ram_info="$used_str$(hrBytes "$ram_used") of $(hrBytes "$ram_total")" - disk_info="$used_str$(hrBytes "$disk_used") of $(hrBytes "$disk_total")" - - lan_info="Gateway: $net_gateway" - dhcp_info="$leased_str$ph_dhcp_num of $ph_dhcp_max" - - ads_info="$total_str$ads_blocked_today of $dns_queries_today" - dns_info="$dns_count DNS servers" - - [[ "$recent_blocked" == "0" ]] && recent_blocked="${COL_LIGHT_RED}FTL offline${COL_NC}" -} - -chronoFunc() { - local extra_arg="$1" - local extra_value="$2" - - get_init_stats - - for (( ; ; )); do - get_sys_stats - get_ftl_stats - get_strings - - # Strip excess development version numbers - if [[ "$ph_core_ver" != "-1" ]]; then - phc_ver_str="$phc_str: ${ph_core_ver%-*}${COL_NC}" - lte_ver_str="$lte_str: ${ph_lte_ver%-*}${COL_NC}" - ftl_ver_str="$ftl_str: ${ph_ftl_ver%-*}${COL_NC}" - else - phc_ver_str="$phc_str: $api_str${COL_NC}" - fi - - # Get refresh number - if [[ "${extra_arg}" = "refresh" ]]; then - num="${extra_value}" - num_str="Refresh set for every $num seconds" - else - num_str="" - fi - - clear - - # Remove exit message heading on third refresh - if [[ "$count" -le 2 ]] && [[ "${extra_arg}" != "exit" ]]; then - echo -e " ${COL_LIGHT_GREEN}Pi-hole Chronometer${COL_NC} - $num_str - ${COL_LIGHT_RED}Press Ctrl-C to exit${COL_NC} - ${COL_DARK_GRAY}$scr_line_str${COL_NC}" - else - echo -e "|¯¯¯(¯)_|¯|_ ___|¯|___$phc_ver_str\\n| ¯_/¯|_| ' \\/ _ \\ / -_)$lte_ver_str\\n|_| |_| |_||_\\___/_\\___|$ftl_ver_str\\n ${COL_DARK_GRAY}$scr_line_str${COL_NC}" - fi - - printFunc " Hostname: " "$sys_name" "$host_info" - printFunc " Uptime: " "$sys_uptime" "$sys_info" - printFunc " Task Load: " "$sys_loadavg" "$sys_info2" - printFunc " CPU usage: " "$cpu_perc%" "$cpu_info" - printFunc " RAM usage: " "$ram_perc%" "$ram_info" - printFunc " HDD usage: " "$disk_perc" "$disk_info" - - if [[ "$DHCP_ACTIVE" == "true" ]]; then - printFunc "DHCP usage: " "$ph_dhcp_percent%" "$dhcp_info" - fi - - printFunc " Pi-hole: " "$ph_status" "$ph_info" - printFunc " Blocked: " "$ads_percentage_today%" "$ads_info" - printFunc "Local Qrys: " "$queries_cached_percentage%" "$dns_info" - - printFunc "Last Block: " "$recent_blocked" - printFunc " Top Block: " "$top_ad" - - # Provide more stats on screens with more lines - if [[ "$scr_lines" -eq 17 ]]; then - if [[ "$DHCP_ACTIVE" == "true" ]]; then - printFunc "Top Domain: " "$top_domain" "last" - else - print_client="true" - fi - else - print_client="true" - fi - - if [[ -n "$print_client" ]]; then - printFunc "Top Domain: " "$top_domain" - printFunc "Top Client: " "$top_client" "last" - fi - - # Handle exit/refresh options - if [[ "${extra_arg}" == "exit" ]]; then - exit 0 - else - if [[ "${extra_arg}" == "refresh" ]]; then - sleep "$num" - else - sleep 5 - fi - fi - - done -} - -jsonFunc() { - get_ftl_stats "json" - echo "{\"domains_being_blocked\":${domains_being_blocked_raw},\"dns_queries_today\":${dns_queries_today_raw},\"ads_blocked_today\":${ads_blocked_today_raw},\"ads_percentage_today\":${ads_percentage_today_raw}}" -} - -helpFunc() { - if [[ "$1" == "?" ]]; then - echo "Unknown option. Please view 'pihole -c --help' for more information" - else - echo "Usage: pihole -c [options] -Example: 'pihole -c -j' -Calculates stats and displays to an LCD - -Options: - -j, --json Output stats as JSON formatted string - -r, --refresh Set update frequency (in seconds) - -e, --exit Output stats and exit without refreshing - -h, --help Display this help text" - fi - - exit 0 -} - -if [[ $# = 0 ]]; then - chronoFunc -fi - -case "$1" in - "-j" | "--json" ) jsonFunc;; - "-h" | "--help" ) helpFunc;; - "-r" | "--refresh" ) chronoFunc refresh "$2";; - "-e" | "--exit" ) chronoFunc exit;; - * ) helpFunc "?";; -esac diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 29a3270d..305a3f5b 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -1,5 +1,5 @@ _pihole() { - local cur prev opts opts_admin opts_checkout opts_chronometer opts_debug opts_interface opts_logging opts_privacy opts_query opts_update opts_version + local cur prev opts opts_admin opts_checkout opts_debug opts_interface opts_logging opts_privacy opts_query opts_update opts_version COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" @@ -7,7 +7,7 @@ _pihole() { case "${prev}" in "pihole") - opts="admin blacklist checkout chronometer debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" + opts="admin blacklist checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) ;; "whitelist"|"blacklist"|"wildcard"|"regex") @@ -22,10 +22,6 @@ _pihole() { opts_checkout="core ftl web master dev" COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) ;; - "chronometer") - opts_chronometer="\--exit \--json \--refresh" - COMPREPLY=( $(compgen -W "${opts_chronometer}" -- ${cur}) ) - ;; "debug") opts_debug="-a" COMPREPLY=( $(compgen -W "${opts_debug}" -- ${cur}) ) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ef2c8d52..cc187499 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -73,7 +73,7 @@ webInterfaceDir="${webroot}/admin" piholeGitUrl="https://github.com/pi-hole/pi-hole.git" PI_HOLE_LOCAL_REPO="/etc/.pihole" # List of pihole scripts, stored in an array -PI_HOLE_FILES=(chronometer list piholeDebug piholeLogFlush setupLCD update version gravity uninstall webpage) +PI_HOLE_FILES=(list piholeDebug piholeLogFlush setupLCD update version gravity uninstall webpage) # This directory is where the Pi-hole scripts will be installed PI_HOLE_INSTALL_DIR="/opt/pihole" PI_HOLE_CONFIG_DIR="/etc/pihole" diff --git a/manpages/pihole.8 b/manpages/pihole.8 index fec1fa5e..55bbe6cb 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -141,20 +141,6 @@ Available commands and options: (0 = lowest, 3 = highest) .br -\fB-c, chronometer\fR [options] -.br - Calculates stats and displays to an LCD -.br - - (Chronometer Options): -.br - -j, --json Output stats as JSON formatted string -.br - -r, --refresh Set update frequency (in seconds) -.br - -e, --exit Output stats and exit without refreshing -.br - \fB-g, updateGravity\fR .br Update the list of ad-serving domains diff --git a/pihole b/pihole index 5281c715..817bfd01 100755 --- a/pihole +++ b/pihole @@ -125,8 +125,7 @@ queryFunc() { } chronometerFunc() { - shift - "${PI_HOLE_SCRIPT_DIR}"/chronometer.sh "$@" + echo "Chronometer is gone, use PADD (https://github.com/pi-hole/PADD)" exit 0 } @@ -497,8 +496,7 @@ Debugging Options: Options: setpassword set the password for the web interface - -c, chronometer Calculates stats and displays to an LCD - Add '-h' for more info on chronometer usage + -c, chronometer Chronometer is gone, use PADD -g, updateGravity Update the list of ad-serving domains -h, --help, help Show this help dialog -l, logging Specify whether the Pi-hole log should be used From 188b2b858acb907c2a856bbe032db833db9df75e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 12 Oct 2023 13:48:01 +0200 Subject: [PATCH 101/200] PADD is fine, really! MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .codespellignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.codespellignore b/.codespellignore index 501a3d67..0755931d 100644 --- a/.codespellignore +++ b/.codespellignore @@ -1,3 +1,4 @@ doubleclick wan nwe +padd From 40c75289b5c88bea75bf3803729a1ecf49b4b2d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 13 Oct 2023 19:59:29 +0200 Subject: [PATCH 102/200] Allow pihole to access subdirs in /etc/pihole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index 17900f0b..abeaabc4 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -9,20 +9,27 @@ utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" # Get file paths FTL_PID_FILE="$(getFTLPIDFile)" -# Touch files to ensure they exist (create if non-existing, preserve if existing) -# shellcheck disable=SC2174 -mkdir -pm 0755 /var/log/pihole -[ -f "${FTL_PID_FILE}" ] || install -D -m 644 -o pihole -g pihole /dev/null "${FTL_PID_FILE}" -[ -f /var/log/pihole/FTL.log ] || install -m 644 -o pihole -g pihole /dev/null /var/log/pihole/FTL.log -[ -f /var/log/pihole/pihole.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/pihole.log -[ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit all necessary files +# shellcheck disable=SC2174 +mkdir -pm 0640 /var/log/pihole chown -R pihole:pihole /etc/pihole /var/log/pihole chmod -R 0640 /var/log/pihole chmod -R 0660 /etc/pihole + # allow all users to enter the directories chmod 0755 /etc/pihole /var/log/pihole +# allow pihole to access subdirs in /etc/pihole (sets execution bit on dirs) +# credits https://stackoverflow.com/a/11512211 +find /etc/pihole -type d -exec chmod 0755 {} \; + +# Touch files to ensure they exist (create if non-existing, preserve if existing) +[ -f "${FTL_PID_FILE}" ] || install -D -m 644 -o pihole -g pihole /dev/null "${FTL_PID_FILE}" +[ -f /var/log/pihole/FTL.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/FTL.log +[ -f /var/log/pihole/pihole.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/pihole.log +[ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases + + # Backward compatibility for user-scripts that still expect log files in /var/log instead of /var/log/pihole # Should be removed with Pi-hole v6.0 if [ ! -f /var/log/pihole.log ]; then From f7ba059b94215160731d4e30fbc239c375c104ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 14 Oct 2023 14:01:20 +0200 Subject: [PATCH 103/200] Query directly and authenticate only if required MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/api.sh | 21 ++++++++++++++++++--- advanced/Scripts/query.sh | 15 +++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index afd88671..449f146f 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -82,10 +82,25 @@ DeleteSession() { } GetFTLData() { - local data + local data response status # get the data from querying the API as well as the http status code - data=$(curl -s -X GET "http://localhost:${PORT}/api$1" -H "Accept: application/json" -H "sid: ${SID}" ) - echo "${data}" + response=$(curl -s -w "%{http_code}" -X GET "http://localhost:${PORT}/api$1" -H "Accept: application/json" -H "sid: ${SID}" ) + + # status are the last 3 characters + status=$(printf %s "${response#"${response%???}"}") + # data is everything from response without the last 3 characters + data=$(printf %s "${response%???}") + + if [ "${status}" = 200 ]; then + # response OK + echo "${data}" + elif [ "${status}" = 000 ]; then + # connection lost + echo "000" + elif [ "${status}" = 401 ]; then + # unauthorized + echo "401" + fi } secretRead() { diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 8407d3f6..a26d249c 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -118,10 +118,21 @@ Main(){ # Test if the authentication endpoint is available TestAPIAvailability - # Authenticate with the FTL server - Authenthication + + # Users can configure FTL in a way, that for accessing a) all endpoints (webserver.api.localAPIauth) + # or b) for the /search endpoint (webserver.api.searchAPIauth) no authentication is required. + # Therefore, we try to query directly without authentication but do authenticat if 401 is returned data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}") + + if [ "${data}" = 401 ]; then + # Unauthenticated, so authenticate with the FTL server required + Authenthication + + # send query again + data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}") + fi + GenerateOutput "${data}" DeleteSession } From ec83d6b7931dc3493efb535ce8c66586bec53d7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 14 Oct 2023 14:06:47 +0200 Subject: [PATCH 104/200] No help for gone functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- pihole | 1 - 1 file changed, 1 deletion(-) diff --git a/pihole b/pihole index 817bfd01..7be13a6f 100755 --- a/pihole +++ b/pihole @@ -496,7 +496,6 @@ Debugging Options: Options: setpassword set the password for the web interface - -c, chronometer Chronometer is gone, use PADD -g, updateGravity Update the list of ad-serving domains -h, --help, help Show this help dialog -l, logging Specify whether the Pi-hole log should be used From 2005d04625c48948c1010563dac3a060f114f311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 14 Oct 2023 21:50:57 +0200 Subject: [PATCH 105/200] Exact search is the new default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/query.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index a26d249c..27a33f39 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -17,7 +17,7 @@ # Globals PI_HOLE_INSTALL_DIR="/opt/pihole" max_results="20" -partial="true" +partial="false" domain="" # Source color table @@ -29,13 +29,13 @@ colfile="/opt/pihole/COL_TABLE" Help(){ echo "Usage: pihole -q [option] -Example: 'pihole -q --exact domain.com' +Example: 'pihole -q --partial domain.com' Query the adlists for a specified domain Options: - --exact Search the adlists for exact domain matches + --partial Search the adlists for partially matching domains --all Return all query matches within the adlists - -h, --help Show this help dialog" + -h, --help Show this help dialog" exit 0 } @@ -141,7 +141,7 @@ Main(){ while [ "$#" -gt 0 ]; do case "$1" in "-h" | "--help" ) Help;; - "--exact" ) partial="false";; + "--partial" ) partial="true";; "--all" ) max_results=10000;; # hard-coded FTL limit * ) domain=$1;; esac From 123ba1f154845ad9b549f3f81638c3ef340dbda6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 15 Oct 2023 22:15:11 +0200 Subject: [PATCH 106/200] Remove temp dir created when downloading FTL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ac516415..4dd31976 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1777,7 +1777,8 @@ FTLinstall() { # Move into the temp ftl directory pushd "$(mktemp -d)" > /dev/null || { printf "Unable to make temporary directory for FTL binary download\\n"; return 1; } - + local tempdir + tempdir="$(pwd)" local ftlBranch local url @@ -1819,12 +1820,19 @@ FTLinstall() { # Installed the FTL service printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" + + # Remove temp dir + remove_dir "${tempdir}" + return 0 else # Otherwise, the hash download failed, so print and exit. popd > /dev/null || { printf "Unable to return to original directory after FTL binary download.\\n"; return 1; } printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" printf " %b Error: Download of %s/%s failed (checksum error)%b\\n" "${COL_LIGHT_RED}" "${url}" "${binary}" "${COL_NC}" + + # Remove temp dir + remove_dir "${tempdir}" return 1 fi else @@ -1833,10 +1841,19 @@ FTLinstall() { printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" # The URL could not be found printf " %b Error: URL %s/%s not found%b\\n" "${COL_LIGHT_RED}" "${url}" "${binary}" "${COL_NC}" + + # Remove temp dir + remove_dir "${tempdir}" return 1 fi } +remove_dir() { + # Delete dir + rm -r "${1}" > /dev/null 2>&1 || \ + echo -e " ${CROSS} Unable to remove ${1}" +} + get_binary_name() { # Get the OS architecture (we cannot use uname -m as this may return an incorrect architecture when buildx-compiling with QEMU for arm) local machine From 27522fbc18dc53cd0828778d5b7a8a2dfd72ff6d Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Mon, 16 Oct 2023 17:19:44 -0300 Subject: [PATCH 107/200] Add a final message to gravity The terminal version doesn't really need a final message, but this will be read by the web interface to show a success message Signed-off-by: RD WebDesign --- gravity.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gravity.sh b/gravity.sh index 9133d33d..e61ade42 100755 --- a/gravity.sh +++ b/gravity.sh @@ -901,4 +901,6 @@ gravity_ShowCount gravity_Cleanup echo "" +echo " ${TICK} Done." + # "${PIHOLE_COMMAND}" status From 2c7fa4a7b36aa4ae440ea5500438f72e56bb0f91 Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Wed, 18 Oct 2023 02:07:06 -0300 Subject: [PATCH 108/200] Avoid printing getFTLConfigValue return in statusFunc() Signed-off-by: RD WebDesign --- pihole | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pihole b/pihole index 7be13a6f..9cc1069f 100755 --- a/pihole +++ b/pihole @@ -345,7 +345,7 @@ analyze_ports() { statusFunc() { # Determine if there is pihole-FTL service is listening - local pid port ftl_pid_file + local pid port ftl_pid_file block_status ftl_pid_file="$(getFTLPIDFile)" @@ -375,7 +375,8 @@ statusFunc() { fi # Determine if Pi-hole's blocking is enabled - if getFTLConfigValue dns.blocking.active; then + block_status=$(getFTLConfigValue dns.blocking.active) + if [ ${block_status} == "true" ]; then case "${1}" in "web") echo "$port";; *) echo -e " ${TICK} Pi-hole blocking is enabled";; From d77dbf736cbeb1f5bc7e885e7ccb0b8cfc9c1bdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 18 Oct 2023 22:09:38 +0200 Subject: [PATCH 109/200] Logrotate config file needs to be owned by root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 4 ++++ automated install/basic-install.sh | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index abeaabc4..c6817828 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -16,6 +16,10 @@ chown -R pihole:pihole /etc/pihole /var/log/pihole chmod -R 0640 /var/log/pihole chmod -R 0660 /etc/pihole +# Logrotate config file need to be owned by root and must not be writable by group and others +chown root:root /etc/pihole/logrotate +chmod 0644 /etc/pihole/logrotate + # allow all users to enter the directories chmod 0755 /etc/pihole /var/log/pihole diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ac516415..4d656283 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1483,7 +1483,8 @@ installLogrotate() { return 2 fi # Copy the file over from the local repo - install -o pihole -g pihole -D -m 644 -T "${PI_HOLE_LOCAL_REPO}"/advanced/Templates/logrotate ${target} + # Logrotate config file must be owned by root and not writable by group or other + install -o root -g root -D -m 644 -T "${PI_HOLE_LOCAL_REPO}"/advanced/Templates/logrotate ${target} # Different operating systems have different user / group # settings for logrotate that makes it impossible to create # a static logrotate file that will work with e.g. From c571d8d37d9b92336024fdf8e5fe98cfa2aee517 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 19 Oct 2023 22:04:11 +0200 Subject: [PATCH 110/200] Drop support for ancient ARMv4 and ARMv5, the Debian binaries turn out to cause unresolvable SIGFPE on ARMv6 and lower. Fortunately, we can still use the Alpine binaries on all Raspberry Pi models (even the oldest ones) Signed-off-by: DL6ER --- automated install/basic-install.sh | 26 ++--- test/test_any_automated_install.py | 154 +---------------------------- 2 files changed, 10 insertions(+), 170 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4d656283..b6806141 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1870,9 +1870,9 @@ get_binary_name() { printf "%b %b Detected AArch64 (64 Bit ARM) architecture\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-arm64" - elif [[ "${cpu_arch}" == "armv6KZ" ]]; then - printf "%b %b Detected ARMv6KZ architecture\\n" "${OVER}" "${TICK}" - # set the binary to be used + elif [[ "${cpu_arch}" == "armv6"* ]]; then + printf "%b %b Detected ARMv6 architecture\\n" "${OVER}" "${TICK}" + # set the binary to be used (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) l_binary="pihole-FTL-armv6" else # If ARMv8 or higher is found (e.g., BCM2837 as found in Raspberry Pi Model 3B) @@ -1885,24 +1885,10 @@ get_binary_name() { printf "%b %b Detected ARMv7 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" # set the binary to be used l_binary="pihole-FTL-armv6" - elif [[ "${cpu_arch}" == "v5TE" || "${rev}" -gt 5 ]]; then - # Check if the system is using GLIBC 2.29 or higher - if [[ -n "${l_glibc_version}" && "$(printf '%s\n' "2.29" "${l_glibc_version}" | sort -V | head -n1)" == "2.29" ]]; then - # If so, use the ARMv6 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv6 architecture (running GLIBC 2.29 or higher, %s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" - # set the binary to be used - l_binary="pihole-FTL-armv5" - else - # Otherwise, use the ARMv5 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv6 architecture (running GLIBC older than 2.29, %s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" - # set the binary to be used - l_binary="pihole-FTL-armv4" - fi else - # Otherwise, use the ARMv4 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv4 or ARMv5 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" - # set the binary to be used - l_binary="pihole-FTL-armv4" + # Otherwise, Pi-hole does not support this architecture + printf "%b %b This processor architecture is not supported by Pi-hole (%s)\\n" "${OVER}" "${CROSS}" "${cpu_arch}" + l_binary="" fi fi elif [[ "${machine}" == "x86_64" ]]; then diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index a8e0e082..1980f0b6 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -259,158 +259,12 @@ def test_FTL_detect_aarch64_no_errors(host): assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv4_no_errors(host): - """ - confirms only armv4 package is downloaded for FTL engine - """ - # mock uname to return armv4 platform - mock_command("uname", {"-m": ("armv4t", "0")}, host) - # mock readelf to respond with armv4 CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv4t", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv4t", "0"), - }, - host, - ) - detectPlatform = host.run( - """ - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture (armv4t)" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv5_no_errors(host): - """ - confirms only armv5 package is downloaded for FTL engine - """ - # mock uname to return armv5te platform - mock_command("uname", {"-m": ("armv5te", "0")}, host) - # mock readelf to respond with armv5 CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv5te", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv5te", "0"), - }, - host, - ) - detectPlatform = host.run( - """ - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture (armv5te)" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv6_old_no_errors(host): +def test_FTL_detect_armv6_no_errors(host): """ confirms only armv6 package is downloaded for FTL engine """ - # mock uname to return armv6l platform - mock_command("uname", {"-m": ("armv6l", "0")}, host) - # mock readelf to respond with armv6l CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv6l", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv6l", "0"), - }, - host, - ) - # Mock old ldd GLIBC version - mock_command( - "ldd", {"--version": ("ldd (Debian GLIBC 2.13-38+deb7u8) 2.13", "0")}, host - ) - - detectPlatform = host.run( - """ - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + ( - " Detected ARMv6 architecture (running GLIBC older than 2.29, armv6l)" - ) - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv6_recent_no_errors(host): - """ - confirms only armv6 package is downloaded for FTL engine - """ - # mock uname to return armv6l platform - mock_command("uname", {"-m": ("armv6l", "0")}, host) - # mock readelf to respond with armv6l CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv6l", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv6l", "0"), - }, - host, - ) - # Mock old ldd GLIBC version - mock_command( - "ldd", {"--version": ("'ldd (Debian GLIBC 2.35-38+deb7u8) 2.35'", "0")}, host - ) - - detectPlatform = host.run( - """ - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + ( - " Detected ARMv6 architecture (running GLIBC 2.29 or higher, armv6l)" - ) - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv6KZ_no_errors(host): - """ - confirms only armv6KZ package is downloaded for FTL engine - """ - # mock uname to return armv6KZ platform - mock_command("uname", {"-m": ("armv6KZ", "0")}, host) + # mock uname to return armv6 platform + mock_command("uname", {"-m": ("armv6", "0")}, host) # mock readelf to respond with armv6l CPU architecture mock_command_2( "readelf", @@ -432,7 +286,7 @@ def test_FTL_detect_armv6KZ_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv6KZ architecture" + expected_stdout = tick_box + " Detected ARMv6 architecture" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout From 79ebbacc4af790164ee927443bb53b55d7a87a93 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 21 Oct 2023 10:27:01 +0000 Subject: [PATCH 111/200] Bump actions/checkout from 4.1.0 to 4.1.1 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.0 to 4.1.1. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.0...v4.1.1) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4685aa2c..6544db61 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 0ff0a24a..d9de09d2 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index a1025629..e52d4ae9 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9f32302e..8166d253 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 - name: Check scripts in repository are executable run: | @@ -72,7 +72,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.1 From ca7836bf717c34f97719ba8aa3c3f6b07f0a591d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 21 Oct 2023 19:05:45 +0200 Subject: [PATCH 112/200] Remove now obsolete GLIBC version check Signed-off-by: DL6ER --- automated install/basic-install.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index b9ed04a4..443ee85e 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1860,15 +1860,6 @@ get_binary_name() { local machine machine=$(uname -m) - # Get local GLIBC version (leave at "0.0" if no GLIBC, e.g., on musl) - local l_glibc_version="0.0" - if ldd --version 2>&1 | grep -q "GLIBC"; then - l_glibc_version=$(ldd --version | head -n1 | grep -o '[0-9.]*$') - printf "%b %b Detected GLIBC version %s\\n" "${OVER}" "${TICK}" "${l_glibc_version}" - else - printf "%b %b No GLIBC detected\\n" "${OVER}" "${CROSS}" - fi - local l_binary local str="Detecting processor" From 45687d675becf21f23953625c00efdb637531c21 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 21 Oct 2023 19:13:45 +0200 Subject: [PATCH 113/200] Fix the binary detection for ARMv6 and simplify the router overall Signed-off-by: DL6ER --- automated install/basic-install.sh | 37 ++++++++++++++---------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 443ee85e..054d8eef 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1875,29 +1875,27 @@ get_binary_name() { local rev rev=$(echo "${cpu_arch}" | grep -o '[0-9]*') if [[ "${machine}" == "aarch64" ]]; then + # If AArch64 is found (e.g., BCM2711 in Raspberry Pi 4) printf "%b %b Detected AArch64 (64 Bit ARM) architecture\\n" "${OVER}" "${TICK}" - # set the binary to be used l_binary="pihole-FTL-arm64" - elif [[ "${cpu_arch}" == "armv6"* ]]; then + elif [[ "${cpu_arch}" == "v6"* ]]; then + # If ARMv6 is found (e.g., BCM2835 in Raspberry Pi 1 and Zero) printf "%b %b Detected ARMv6 architecture\\n" "${OVER}" "${TICK}" - # set the binary to be used (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) + l_binary="pihole-FTL-armv6" + elif [[ "${cpu_arch}" == "v7"* || "${rev}" -ge 7 ]]; then + # If ARMv7 or higher is found (e.g., BCM2836 in Raspberry PI 2 Mod. B) + # This path is also used for ARMv8 when the OS is in 32bit mode + # (e.g., BCM2837 in Raspberry Pi Model 3B, or BCM2711 in Raspberry Pi 4) + printf "%b %b Detected ARMv7 (or newer) architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" + l_binary="pihole-FTL-armv7" + elif [[ "${rev}" -gt 6 ]]; then + # Otherwise, if ARMv7 is found (e.g., BCM2836 in Raspberry Pi Model 2) + printf "%b %b Detected ARMv7 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" l_binary="pihole-FTL-armv6" else - # If ARMv8 or higher is found (e.g., BCM2837 as found in Raspberry Pi Model 3B) - if [[ "${cpu_arch}" == "v7" || "${rev}" -gt 7 ]]; then - printf "%b %b Detected ARMv7 (or newer) architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" - # set the binary to be used - l_binary="pihole-FTL-armv7" - elif [[ "${rev}" -gt 6 ]]; then - # Otherwise, if ARMv7 is found (e.g., BCM2836 as found in Raspberry Pi Model 2) - printf "%b %b Detected ARMv7 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" - # set the binary to be used - l_binary="pihole-FTL-armv6" - else - # Otherwise, Pi-hole does not support this architecture - printf "%b %b This processor architecture is not supported by Pi-hole (%s)\\n" "${OVER}" "${CROSS}" "${cpu_arch}" - l_binary="" - fi + # Otherwise, Pi-hole does not support this architecture + printf "%b %b This processor architecture is not supported by Pi-hole (%s)\\n" "${OVER}" "${CROSS}" "${cpu_arch}" + l_binary="" fi elif [[ "${machine}" == "x86_64" ]]; then # This gives the processor of packages dpkg installs (for example, "i386") @@ -1912,9 +1910,8 @@ get_binary_name() { printf "%b %b Detected 32bit (i686) architecture\\n" "${OVER}" "${TICK}" l_binary="pihole-FTL-386" else - # 64bit + # 64bit OS printf "%b %b Detected x86_64 architecture\\n" "${OVER}" "${TICK}" - # set the binary to be used l_binary="pihole-FTL-amd64" fi elif [[ "${machine}" == "riscv64" ]]; then From 30bfc7cc9f44cd9b0138ebc2e7400f4e592fbd42 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 21 Oct 2023 19:15:10 +0200 Subject: [PATCH 114/200] Add binutils tot he OS check dependencies as we need it to check if the local system is abel to run any of our precompiled FTL binaries Signed-off-by: DL6ER --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 054d8eef..019cc3e3 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -302,9 +302,9 @@ package_manager_detect() { # Update package cache update_package_cache || exit 1 # Packages required to perform the os_check (stored as an array) - OS_CHECK_DEPS=(grep dnsutils) + OS_CHECK_DEPS=(grep dnsutils binutils) # Packages required to run this install script (stored as an array) - INSTALLER_DEPS=(git iproute2 dialog ca-certificates binutils) + INSTALLER_DEPS=(git iproute2 dialog ca-certificates) # Packages required to run Pi-hole (stored as an array) PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) From 38ecc1693f50764209b8a0eff289064eef0ed881 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 21 Oct 2023 19:28:59 +0200 Subject: [PATCH 115/200] Further simplify the ARMv6 test Signed-off-by: DL6ER --- automated install/basic-install.sh | 8 ++------ test/test_any_automated_install.py | 8 ++++---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 019cc3e3..2d47a548 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1878,20 +1878,16 @@ get_binary_name() { # If AArch64 is found (e.g., BCM2711 in Raspberry Pi 4) printf "%b %b Detected AArch64 (64 Bit ARM) architecture\\n" "${OVER}" "${TICK}" l_binary="pihole-FTL-arm64" - elif [[ "${cpu_arch}" == "v6"* ]]; then + elif [[ "${rev}" -eq 6 ]]; then # If ARMv6 is found (e.g., BCM2835 in Raspberry Pi 1 and Zero) printf "%b %b Detected ARMv6 architecture\\n" "${OVER}" "${TICK}" l_binary="pihole-FTL-armv6" - elif [[ "${cpu_arch}" == "v7"* || "${rev}" -ge 7 ]]; then + elif [[ "${rev}" -ge 7 ]]; then # If ARMv7 or higher is found (e.g., BCM2836 in Raspberry PI 2 Mod. B) # This path is also used for ARMv8 when the OS is in 32bit mode # (e.g., BCM2837 in Raspberry Pi Model 3B, or BCM2711 in Raspberry Pi 4) printf "%b %b Detected ARMv7 (or newer) architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" l_binary="pihole-FTL-armv7" - elif [[ "${rev}" -gt 6 ]]; then - # Otherwise, if ARMv7 is found (e.g., BCM2836 in Raspberry Pi Model 2) - printf "%b %b Detected ARMv7 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" - l_binary="pihole-FTL-armv6" else # Otherwise, Pi-hole does not support this architecture printf "%b %b This processor architecture is not supported by Pi-hole (%s)\\n" "${OVER}" "${CROSS}" "${cpu_arch}" diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 1980f0b6..7d9d15f6 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -269,8 +269,8 @@ def test_FTL_detect_armv6_no_errors(host): mock_command_2( "readelf", { - "-A /bin/sh": ("Tag_CPU_arch: armv6KZ", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv6KZ", "0"), + "-A /bin/sh": ("Tag_CPU_arch: armv6", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv6", "0"), }, host, ) @@ -319,7 +319,7 @@ def test_FTL_detect_armv7l_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv7 architecture (armv7l)") + expected_stdout = tick_box + (" Detected ARMv7 (or newer) architecture") assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -352,7 +352,7 @@ def test_FTL_detect_armv7_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv7 architecture (armv7)") + expected_stdout = tick_box + (" Detected ARMv7 (or newer) architecture") assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout From 369ccf13a846bb64ac7b6d8a454f6bda4cf2a89a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 21 Oct 2023 21:40:21 +0200 Subject: [PATCH 116/200] Move FTL binary availability check after the supported OS check Signed-off-by: DL6ER --- automated install/basic-install.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 2d47a548..1ecde7a9 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2088,16 +2088,6 @@ main() { fi fi - # Check if there is a usable FTL binary available on this architecture - do - # this early on as FTL is a hard dependency for Pi-hole - local funcOutput - funcOutput=$(get_binary_name) #Store output of get_binary_name here - # Abort early if this processor is not supported (get_binary_name returns empty string) - if [[ "${funcOutput}" == "" ]]; then - printf " %b Upgrade/install aborted\\n" "${CROSS}" "${DISTRO_NAME}" - exit 1 - fi - # Check if SELinux is Enforcing and exit before doing anything else checkSelinux @@ -2114,6 +2104,16 @@ main() { # Check that the installed OS is officially supported - display warning if not os_check + # Check if there is a usable FTL binary available on this architecture - do + # this early on as FTL is a hard dependency for Pi-hole + local funcOutput + funcOutput=$(get_binary_name) #Store output of get_binary_name here + # Abort early if this processor is not supported (get_binary_name returns empty string) + if [[ "${funcOutput}" == "" ]]; then + printf " %b Upgrade/install aborted\\n" "${CROSS}" "${DISTRO_NAME}" + exit 1 + fi + # Install packages used by this installation script printf " %b Checking for / installing Required dependencies for this install script...\\n" "${INFO}" install_dependent_packages "${INSTALLER_DEPS[@]}" From 7523c49f624a57f5b0ae87e39328813bf5b72f83 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 22 Oct 2023 08:14:11 +0200 Subject: [PATCH 117/200] Swapping the databases must be the last step before the cleanup. Otherwise, FTL may be reloading from an only partially completed database causing spurious errors and/or warnings. Signed-off-by: DL6ER --- gravity.sh | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/gravity.sh b/gravity.sh index e61ade42..7b7234ee 100755 --- a/gravity.sh +++ b/gravity.sh @@ -73,9 +73,9 @@ generate_gravity_database() { chmod g+w "${piholeDir}" "${gravityDBfile}" } -# Copy data from old to new database file and swap them -gravity_swap_databases() { - local str copyGravity oldAvail +# Build gravity tree +gravity_build_tree() { + local str str="Building tree" echo -ne " ${INFO} ${str}..." @@ -88,7 +88,10 @@ gravity_swap_databases() { return 1 fi echo -e "${OVER} ${TICK} ${str}" +} +# Copy data from old to new database file and swap them +gravity_swap_databases() { str="Swapping databases" echo -ne " ${INFO} ${str}..." @@ -116,11 +119,11 @@ gravity_swap_databases() { # Update timestamp when the gravity table was last updated successfully update_gravity_timestamp() { - output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to update gravity timestamp in database ${gravityDBfile}\\n ${output}" + echo -e "\\n ${CROSS} Unable to update gravity timestamp in database ${gravityTEMPfile}\\n ${output}" return 1 fi return 0 @@ -659,12 +662,12 @@ gravity_Table_Count() { local table="${1}" local str="${2}" local num - num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table};")" + num="$(pihole-FTL sqlite3 "${gravityTEMPfile}" "SELECT COUNT(*) FROM ${table};")" if [[ "${table}" == "gravity" ]]; then local unique - unique="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" + unique="$(pihole-FTL sqlite3 "${gravityTEMPfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" echo -e " ${INFO} Number of ${str}: ${num} (${COL_BOLD}${unique} unique domains${COL_NC})" - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" + pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" else echo -e " ${INFO} Number of ${str}: ${num}" fi @@ -882,22 +885,27 @@ fi # Create local.list gravity_generateLocalList -# Migrate rest of the data from old to new database -if ! gravity_swap_databases; then - echo -e " ${CROSS} Unable to create database. Please contact support." - exit 1 -fi - # Update gravity timestamp update_gravity_timestamp # Ensure proper permissions are set for the database -chown pihole:pihole "${gravityDBfile}" -chmod g+w "${piholeDir}" "${gravityDBfile}" +chown pihole:pihole "${gravityTEMPfile}" +chmod g+w "${piholeDir}" "${gravityTEMPfile}" + +# Build the tree +gravity_build_tree -# Compute numbers to be displayed +# Compute numbers to be displayed (do this after building the tree to get the +# numbers quickly from the tree instead of having to scan the whole database) gravity_ShowCount +# Migrate rest of the data from old to new database +# IMPORTANT: Swapping the databases must be the last step before the cleanup +if ! gravity_swap_databases; then + echo -e " ${CROSS} Unable to create database. Please contact support." + exit 1 +fi + gravity_Cleanup echo "" From 842a9d7778e5de72f99d39b4fe78ed8523c94eb1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 22 Oct 2023 08:32:52 +0200 Subject: [PATCH 118/200] Address review comments Signed-off-by: DL6ER --- automated install/basic-install.sh | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 1ecde7a9..87bf1ea4 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -90,7 +90,6 @@ IPV6_ADDRESS=${IPV6_ADDRESS} QUERY_LOGGING=true WEBPORT=8080 PRIVACY_LEVEL=0 -CACHE_SIZE=10000 if [ -z "${USER}" ]; then USER="$(id -un)" @@ -301,11 +300,11 @@ package_manager_detect() { PKG_COUNT="${PKG_MANAGER} -s -o Debug::NoLocking=true upgrade | grep -c ^Inst || true" # Update package cache update_package_cache || exit 1 - # Packages required to perform the os_check (stored as an array) + # Packages required to perform the os_check and FTL binary detection OS_CHECK_DEPS=(grep dnsutils binutils) - # Packages required to run this install script (stored as an array) + # Packages required to run this install script INSTALLER_DEPS=(git iproute2 dialog ca-certificates) - # Packages required to run Pi-hole (stored as an array) + # Packages required to run Pi-hole PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) # If apt-get is not found, check for rpm. @@ -1856,29 +1855,30 @@ remove_dir() { } get_binary_name() { - # Get the OS architecture (we cannot use uname -m as this may return an incorrect architecture when buildx-compiling with QEMU for arm) + local l_binary local machine machine=$(uname -m) - local l_binary - local str="Detecting processor" printf " %b %s..." "${INFO}" "${str}" - # If the machine is arm or aarch - if [[ "${machine}" == "arm"* || "${machine}" == *"aarch"* ]]; then - # ARM + + # If the machine is aarch64 (armv8) + if [[ "${machine}" == "aarch64" ]]; then + # If AArch64 is found (e.g., BCM2711 in Raspberry Pi 4) + printf "%b %b Detected AArch64 (64 Bit ARM) architecture\\n" "${OVER}" "${TICK}" + l_binary="pihole-FTL-arm64" + elif [[ "${machine}" == "arm"* ]]; then + # ARM 32 bit # Get supported processor from other binaries installed on the system + # We cannot really rely on the output of $(uname -m) above as this may + # return an incorrect architecture when buildx-compiling with QEMU local cpu_arch cpu_arch=$(readelf -A "$(command -v sh)" | grep Tag_CPU_arch | awk '{ print $2 }') # Get the revision from the CPU architecture local rev rev=$(echo "${cpu_arch}" | grep -o '[0-9]*') - if [[ "${machine}" == "aarch64" ]]; then - # If AArch64 is found (e.g., BCM2711 in Raspberry Pi 4) - printf "%b %b Detected AArch64 (64 Bit ARM) architecture\\n" "${OVER}" "${TICK}" - l_binary="pihole-FTL-arm64" - elif [[ "${rev}" -eq 6 ]]; then + if [[ "${rev}" -eq 6 ]]; then # If ARMv6 is found (e.g., BCM2835 in Raspberry Pi 1 and Zero) printf "%b %b Detected ARMv6 architecture\\n" "${OVER}" "${TICK}" l_binary="pihole-FTL-armv6" From fd8fdd3513f7d41afddfe1877d11261c7e4ac0b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 23 Oct 2023 21:36:18 +0200 Subject: [PATCH 119/200] Use suffixed temp file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gravity.sh b/gravity.sh index 7b7234ee..edfe89a9 100755 --- a/gravity.sh +++ b/gravity.sh @@ -141,6 +141,7 @@ database_table_from_file() { # implementations of mktemp support it, e.g. on Alpine tmpFile="$(mktemp -p "${GRAVITY_TMPDIR}")" mv "${tmpFile}" "${tmpFile%.*}.gravity" + tmpFile="${tmpFile%.*}.gravity" local timestamp timestamp="$(date --utc +'%s')" @@ -502,6 +503,7 @@ gravity_DownloadBlocklistFromUrl() { # We don't use '--suffix' here because not all implementations of mktemp support it, e.g. on Alpine listCurlBuffer="$(mktemp -p "${GRAVITY_TMPDIR}")" mv "${listCurlBuffer}" "${listCurlBuffer%.*}.phgpb" + listCurlBuffer="${listCurlBuffer%.*}.phgpb" # Determine if $saveLocation has read permission if [[ -r "${saveLocation}" && $url != "file"* ]]; then From 00d7e998b462949fa0091a4b55ad82f1572b07c4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:03:09 +0200 Subject: [PATCH 120/200] setupVars.conf and pihole-FTL.conf are no more - use pihole.toml instead Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 36 ++++++++++----------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 0e3bbf3d..977bce35 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -77,7 +77,7 @@ PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" -PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole-FTL.conf" +PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml" PIHOLE_CUSTOM_HOSTS_FILE="${PIHOLE_DIRECTORY}/custom.list" PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" @@ -138,7 +138,6 @@ REQUIRED_FILES=("${PIHOLE_CRON_FILE}" "${PIHOLE_RAW_BLOCKLIST_FILES}" "${PIHOLE_LOCAL_HOSTS_FILE}" "${PIHOLE_LOGROTATE_FILE}" -"${PIHOLE_SETUP_VARS_FILE}" "${PIHOLE_FTL_CONF_FILE}" "${PIHOLE_COMMAND}" "${PIHOLE_COLTABLE_FILE}" @@ -165,20 +164,6 @@ show_disclaimer(){ log_write "${DISCLAIMER}" } -source_setup_variables() { - # Display the current test that is running - log_write "\\n${COL_PURPLE}*** [ INITIALIZING ]${COL_NC} Sourcing setup variables" - # If the variable file exists, - if ls "${PIHOLE_SETUP_VARS_FILE}" 1> /dev/null 2>&1; then - log_write "${INFO} Sourcing ${PIHOLE_SETUP_VARS_FILE}..."; - # source it - source ${PIHOLE_SETUP_VARS_FILE} - else - # If it can't, show an error - log_write "${PIHOLE_SETUP_VARS_FILE} ${COL_RED}does not exist or cannot be read.${COL_NC}" - fi -} - make_temporary_log() { # Create a random temporary file for the log TEMPLOG=$(mktemp /tmp/pihole_temp.XXXXXX) @@ -546,15 +531,15 @@ disk_usage() { done } -parse_setup_vars() { - echo_current_diagnostic "Setup variables" +parse_pihole_toml() { + echo_current_diagnostic "Pi-hole configuration" # If the file exists, - if [[ -r "${PIHOLE_SETUP_VARS_FILE}" ]]; then + if [[ -r "${PIHOLE_FTL_CONF_FILE}" ]]; then # parse it - parse_file "${PIHOLE_SETUP_VARS_FILE}" + parse_file "${PIHOLE_FTL_CONF_FILE}" else # If not, show an error - log_write "${CROSS} ${COL_RED}Could not read ${PIHOLE_SETUP_VARS_FILE}.${COL_NC}" + log_write "${CROSS} ${COL_RED}Could not read ${PIHOLE_FTL_CONF_FILE}.${COL_NC}" fi } @@ -1010,8 +995,10 @@ parse_file() { # For each line in the file, for file_lines in "${file_info[@]}"; do if [[ -n "${file_lines}" ]]; then - # don't include the Web password hash - [[ "${file_lines}" =~ ^\#.*$ || ! "${file_lines}" || "${file_lines}" == "WEBPASSWORD="* ]] && continue + # skip empty and comment lines line + [[ "${file_lines}" =~ ^[[:space:]]*\#.*$ || ! "${file_lines}" ]] && continue + # remove the password hash from the output (*"pwhash = "*) + [[ "${file_lines}" == *"pwhash ="* ]] && file_lines=$(echo "${file_lines}" | sed -e 's/\(pwhash = \).*/\1/') # otherwise, display the lines of the file log_write " ${file_lines}" fi @@ -1076,7 +1063,6 @@ list_files_in_dir() { elif [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_DEBUG_LOG}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_RAW_BLOCKLIST_FILES}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_INSTALL_LOG_FILE}" ]] || \ - [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_SETUP_VARS_FILE}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_LOG}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_WEB_SERVER_ACCESS_LOG_FILE}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_LOG_GZIPS}" ]]; then @@ -1484,7 +1470,7 @@ check_dhcp_servers process_status ftl_full_status lighttpd_test_configuration -parse_setup_vars +parse_pihole_toml check_x_headers analyze_ftl_db analyze_gravity_list From ccd8c470a535c5a42085f1e68a91d4b4bff5bf18 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:03:37 +0200 Subject: [PATCH 121/200] check_x_headers is no more Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 977bce35..88723cdc 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -732,35 +732,6 @@ check_networking() { [ -z "${DOCKER_VERSION}" ] && check_required_ports } -# check_x_headers() { -# # The X-Headers allow us to determine from the command line if the Web -# # lighttpd.conf has a directive to show "X-Pi-hole: A black hole for Internet advertisements." -# # in the header of any Pi-holed domain -# # Similarly, it will show "X-Pi-hole: The Pi-hole Web interface is working!" if you view the header returned -# # when accessing the dashboard (i.e curl -I pi.hole/admin/) -# # server is operating correctly -# echo_current_diagnostic "Dashboard headers" -# # Use curl -I to get the header and parse out just the X-Pi-hole one -# local full_curl_output_dashboard -# local dashboard -# full_curl_output_dashboard="$(curl -Is localhost/admin/)" -# dashboard=$(echo "${full_curl_output_dashboard}" | awk '/X-Pi-hole/' | tr -d '\r') -# # Store what the X-Header should be in variables for comparison later -# local dashboard_working -# dashboard_working="X-Pi-hole: The Pi-hole Web interface is working!" - -# # If the X-Header matches what a working system should have, -# if [[ $dashboard == "$dashboard_working" ]]; then -# # then we can show a success -# log_write "$TICK Web interface X-Header: ${COL_GREEN}${dashboard}${COL_NC}" -# else -# # Otherwise, it's a failure since the X-Headers either don't exist or have been modified in some way -# log_write "$CROSS Web interface X-Header: ${COL_RED}X-Header does not match or could not be retrieved.${COL_NC}" - -# log_write "${COL_RED}${full_curl_output_dashboard}${COL_NC}" -# fi -# } - dig_at() { # We need to test if Pi-hole can properly resolve domain names # as it is an essential piece of the software @@ -1471,7 +1442,6 @@ process_status ftl_full_status lighttpd_test_configuration parse_pihole_toml -check_x_headers analyze_ftl_db analyze_gravity_list show_groups From dcc7e96d2f4b963cae8630a6db3e04b7fc7d2802 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:07:11 +0200 Subject: [PATCH 122/200] Remove detection of addresses of PIHOLE_INTERFACE as it is gone, too Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 35 +++------------------------------ 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 88723cdc..517ab4ec 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -550,33 +550,6 @@ parse_locale() { parse_file "${pihole_locale}" } -detect_ip_addresses() { - # First argument should be a 4 or a 6 - local protocol=${1} - # Use ip to show the addresses for the chosen protocol - # Store the values in an array so they can be looped through - # Get the lines that are in the file(s) and store them in an array for parsing later - mapfile -t ip_addr_list < <(ip -"${protocol}" addr show dev "${PIHOLE_INTERFACE}" | awk -F ' ' '{ for(i=1;i<=NF;i++) if ($i ~ '/^inet/') print $(i+1) }') - - # If there is something in the IP address list, - if [[ -n ${ip_addr_list[*]} ]]; then - # Local iterator - local i - # Display the protocol and interface - log_write "${TICK} IPv${protocol} address(es) bound to the ${PIHOLE_INTERFACE} interface:" - # Since there may be more than one IP address, store them in an array - for i in "${!ip_addr_list[@]}"; do - log_write " ${ip_addr_list[$i]}" - done - # Print a blank line just for formatting - log_write "" - else - # If there are no IPs detected, explain that the protocol is not configured - log_write "${CROSS} ${COL_RED}No IPv${protocol} address(es) found on the ${PIHOLE_INTERFACE}${COL_NC} interface.\\n" - return 1 - fi -} - ping_ipv4_or_ipv6() { # Give the first argument a readable name (a 4 or a six should be the argument) local protocol="${1}" @@ -605,9 +578,9 @@ ping_gateway() { while IFS= read -r gateway; do log_write " ${gateway}" - done < <(ip -"${protocol}" route | grep default | grep "${PIHOLE_INTERFACE}" | cut -d ' ' -f 3) + done < <(ip -"${protocol}" route | grep default | cut -d ' ' -f 3) - gateway=$(ip -"${protocol}" route | grep default | grep "${PIHOLE_INTERFACE}" | cut -d ' ' -f 3 | head -n 1) + gateway=$(ip -"${protocol}" route | grep default | cut -d ' ' -f 3 | head -n 1) # If there was at least one gateway if [ -n "${gateway}" ]; then # Let the user know we will ping the gateway for a response @@ -615,7 +588,7 @@ ping_gateway() { # Try to quietly ping the gateway 3 times, with a timeout of 3 seconds, using numeric output only, # on the pihole interface, and tail the last three lines of the output # If pinging the gateway is not successful, - if ! ${cmd} -c 1 -W 2 -n "${gateway}" -I "${PIHOLE_INTERFACE}" >/dev/null; then + if ! ${cmd} -c 1 -W 2 -n "${gateway}" >/dev/null; then # let the user know log_write "${CROSS} ${COL_RED}Gateway did not respond.${COL_NC} ($FAQ_GATEWAY)\\n" # and return an error code @@ -723,8 +696,6 @@ check_networking() { # Runs through several of the functions made earlier; we just clump them # together since they are all related to the networking aspect of things echo_current_diagnostic "Networking" - detect_ip_addresses "4" - detect_ip_addresses "6" ping_gateway "4" ping_gateway "6" # Skip the following check if installed in docker container. Unpriv'ed containers do not have access to the information required From c86e7b3f96ebcd3c99b19a360f2b7410a6847778 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:13:31 +0200 Subject: [PATCH 123/200] Generalize CPU check to a short hardware information output Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 50 ++++++++++++++++----------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 517ab4ec..4c0e3cd0 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -482,34 +482,34 @@ check_firewalld() { fi } -processor_check() { - echo_current_diagnostic "Processor" - # Store the processor type in a variable - PROCESSOR=$(uname -m) - # If it does not contain a value, - if [[ -z "${PROCESSOR}" ]]; then - # we couldn't detect it, so show an error - PROCESSOR=$(lscpu | awk '/Architecture/ {print $2}') - log_write "${CROSS} ${COL_RED}${PROCESSOR}${COL_NC} has not been tested with FTL, but may still work: (${FAQ_FTL_COMPATIBILITY})" +run_and_print_command() { + # Run the command passed as an argument + local cmd="${1}" + # Show the command that is being run + log_write "${INFO} ${cmd}" + # Run the command and store the output in a variable + local output + output=$(${cmd} 2>&1) + # If the command was successful, + if [[ $? -eq 0 ]]; then + # show the output + log_write "${output}" else - # Check if the architecture is currently supported for FTL - case "${PROCESSOR}" in - "amd64" | "x86_64") log_write "${TICK} ${COL_GREEN}${PROCESSOR}${COL_NC}" - ;; - "armv6l") log_write "${TICK} ${COL_GREEN}${PROCESSOR}${COL_NC}" - ;; - "armv6") log_write "${TICK} ${COL_GREEN}${PROCESSOR}${COL_NC}" - ;; - "armv7l") log_write "${TICK} ${COL_GREEN}${PROCESSOR}${COL_NC}" - ;; - "aarch64") log_write "${TICK} ${COL_GREEN}${PROCESSOR}${COL_NC}" - ;; - # Otherwise, show the processor type - *) log_write "${INFO} ${PROCESSOR}"; - esac + # otherwise, show an error + log_write "${CROSS} ${COL_RED}Command failed${COL_NC}" fi } +hardware_check() { + echo_current_diagnostic "System hardware configuration" + # Store the output of the command in a variable + run_and_print_command "lshw -short" + + echo_current_diagnostic "Processor details" + # Store the output of the command in a variable + run_and_print_command "lscpu" +} + disk_usage() { local file_system local hide @@ -1403,7 +1403,7 @@ check_component_versions diagnose_operating_system check_selinux check_firewalld -processor_check +hardware_check disk_usage check_ip_command check_networking From 635b4e952f9c18c1f3ba20280f17b1c1eab20463 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:20:31 +0200 Subject: [PATCH 124/200] Add NOERROR/NXDOMAIN support in the DNS resolution test Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 4c0e3cd0..9225c059 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -781,8 +781,16 @@ dig_at() { if [ -n "${addresses}" ]; then while IFS= read -r local_address ; do # Check if Pi-hole can use itself to block a domain - if local_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${local_address}" +short "${record_type}"); then + if local_dig="$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${local_address}" "${record_type}")"; then # If it can, show success + if [[ "${local_dig}" == *"status: NOERROR"* ]]; then + local_dig="NOERROR" + elif [[ "${local_dig}" == *"status: NXDOMAIN"* ]]; then + local_dig="NXDOMAIN" + else + # Extract the IPv4/6 address from the output + local_dig="$(echo "${local_dig}" | grep -Eo '[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*|([0-9a-f]{0,4}:){1,7}[0-9a-f]{0,4}')" + fi log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} on ${COL_CYAN}${iface}${COL_NC} (${COL_CYAN}${local_address}${COL_NC})" else # Otherwise, show a failure From 15be8eeffad5fd209ff08ca8078bd43a4989274c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:21:08 +0200 Subject: [PATCH 125/200] Remove lighttpd config test Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 9225c059..0e57722e 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -868,20 +868,6 @@ ftl_full_status(){ fi } -lighttpd_test_configuration(){ - # let lighttpd test it's own configuration - local lighttpd_conf_test - echo_current_diagnostic "Lighttpd configuration test" - lighttpd_conf_test=$(lighttpd -tt -f /etc/lighttpd/lighttpd.conf) - if [ -z "${lighttpd_conf_test}" ]; then - # empty output - log_write "${TICK} ${COL_GREEN}No error in lighttpd configuration${COL_NC}" - else - log_write "${CROSS} ${COL_RED}Error in lighttpd configuration${COL_NC}" - log_write " ${lighttpd_conf_test}" - fi -} - make_array_from_file() { local filename="${1}" # The second argument can put a limit on how many line should be read from the file @@ -1419,7 +1405,6 @@ check_name_resolution check_dhcp_servers process_status ftl_full_status -lighttpd_test_configuration parse_pihole_toml analyze_ftl_db analyze_gravity_list From d377cfbc3eb745f112f741f8997d57191f4daa37 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:34:30 +0200 Subject: [PATCH 126/200] Extract the first entry in the answer section from dig's output, replacing any multiple spaces and tabs with a single space Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 0e57722e..53123b9a 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -788,8 +788,9 @@ dig_at() { elif [[ "${local_dig}" == *"status: NXDOMAIN"* ]]; then local_dig="NXDOMAIN" else - # Extract the IPv4/6 address from the output - local_dig="$(echo "${local_dig}" | grep -Eo '[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*|([0-9a-f]{0,4}:){1,7}[0-9a-f]{0,4}')" + # Extract the first entry in the answer section from dig's output, + # replacing any multiple spaces and tabs with a single space + local_dig="$(echo "${local_dig}" | grep -A1 "ANSWER SECTION" | grep -v "ANSWER SECTION" | tr -s " \t" " ")" fi log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} on ${COL_CYAN}${iface}${COL_NC} (${COL_CYAN}${local_address}${COL_NC})" else From 2e9e579bba9122febfa8605452455dc5397f68d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 28 Oct 2023 10:31:54 +0000 Subject: [PATCH 127/200] Bump pytest from 7.4.2 to 7.4.3 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.2 to 7.4.3. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.2...7.4.3) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 27417754..45c7c7c0 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.2 +pytest == 7.4.3 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 tox == 4.11.3 From 6292e65b376ce1d5ad350da2a0b951fc1480f744 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 28 Oct 2023 17:56:37 +0100 Subject: [PATCH 128/200] When setting a blank password, use `webserver.api.password` instead of `webserver.api.pwhash` (fixed in https://github.com/pi-hole/FTL/pull/1702) This prevents the password from being blanked out on the command line when it has been set by an environment variable --- pihole | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pihole b/pihole index 9cc1069f..8e3f1f98 100755 --- a/pihole +++ b/pihole @@ -43,7 +43,7 @@ SetWebPassword() { echo "" if [ "${PASSWORD}" == "" ]; then - setFTLConfigValue "webserver.api.pwhash" "" >/dev/null + setFTLConfigValue "webserver.api.password" "" >/dev/null echo -e " ${TICK} Password Removed" exit 0 fi From ab3b6dfa67b0699a0fa895f2318a7a28f78ddca7 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 28 Oct 2023 19:46:45 +0100 Subject: [PATCH 129/200] No need to >/dev/null because setFTLConfigValue already does this adjust output of setFTLConfigValue to test for the exit code of `pihole-FTL --config` --- advanced/Scripts/utils.sh | 4 ++++ pihole | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 3f9b7031..2fe419e8 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -145,4 +145,8 @@ getFTLConfigValue(){ ####################### setFTLConfigValue(){ pihole-FTL --config "${1}" "${2}" >/dev/null + if [[ $? -eq 5 ]]; then + echo -e " ${CROSS} ${1} set by environment variable. Please unset it to use this function" + exit 5 + fi } diff --git a/pihole b/pihole index 8e3f1f98..279977e4 100755 --- a/pihole +++ b/pihole @@ -43,7 +43,7 @@ SetWebPassword() { echo "" if [ "${PASSWORD}" == "" ]; then - setFTLConfigValue "webserver.api.password" "" >/dev/null + setFTLConfigValue "webserver.api.password" "" echo -e " ${TICK} Password Removed" exit 0 fi @@ -54,7 +54,7 @@ SetWebPassword() { if [ "${PASSWORD}" == "${CONFIRM}" ] ; then # pihole-FTL will automatically hash the password - setFTLConfigValue "webserver.api.password" "${PASSWORD}" >/dev/null + setFTLConfigValue "webserver.api.password" "${PASSWORD}" echo -e " ${TICK} New password set" else echo -e " ${CROSS} Passwords don't match. Your password has not been changed" From 480a8c8d7fd588e3cb35081b6064e60c18643d4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 31 Oct 2023 21:02:26 +0100 Subject: [PATCH 130/200] Remove idn2 as punycode conversion is handled by FTL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/query.sh | 10 ++++------ automated install/basic-install.sh | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 27a33f39..2279df85 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -108,13 +108,11 @@ Main(){ if [ -z "${domain}" ]; then echo "No domain specified"; exit 1 - else - # convert domain to punycode - domain=$(idn2 "${domain}") - - # convert the domain to lowercase - domain=$(echo "${domain}" | tr '[:upper:]' '[:lower:]') fi + # domains are lowercased and converted to punycode by FTL since + # https://github.com/pi-hole/FTL/pull/1715 + # no need to do it here + # Test if the authentication endpoint is available TestAPIAvailability diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 87bf1ea4..ef052dbd 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -305,7 +305,7 @@ package_manager_detect() { # Packages required to run this install script INSTALLER_DEPS=(git iproute2 dialog ca-certificates) # Packages required to run Pi-hole - PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) + PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) # If apt-get is not found, check for rpm. elif is_command rpm ; then @@ -322,7 +322,7 @@ package_manager_detect() { PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src|.riscv64)' | wc -l || true" OS_CHECK_DEPS=(grep bind-utils) INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates binutils) - PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc libcap nmap-ncat jq) + PIHOLE_DEPS=(cronie curl findutils sudo unzip psmisc libcap nmap-ncat jq) # If neither apt-get or yum/dnf package managers were found else From ced1c5d50b5e5beedefed9f2747a21b82b60e833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 17 Dec 2022 16:53:01 +0100 Subject: [PATCH 131/200] Start counting at postion 1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/updatecheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 8bb1888b..66f1a7ab 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -31,7 +31,7 @@ function get_remote_version() { function get_remote_hash(){ - git ls-remote "https://github.com/pi-hole/${1}" --tags "${2}" | awk '{print substr($0, 0,8);}' || return 1 + git ls-remote "https://github.com/pi-hole/${1}" --tags "${2}" | awk '{print substr($0, 1,8);}' || return 1 } # Source the utils file for addOrEditKeyValPair() From bc96d3b0a9635202f56ef42be6ea1ef100a5d27c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 5 Nov 2023 21:15:13 +0100 Subject: [PATCH 132/200] Tweak help text of pihole setpassword MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- pihole | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pihole b/pihole index 279977e4..47da4ddd 100755 --- a/pihole +++ b/pihole @@ -496,7 +496,9 @@ Debugging Options: Options: - setpassword set the password for the web interface + setpassword [pwd] Set the password for the web interface + Without optional argument, password is read interactively. + When specifying a password directly, enclose it in single quotes. -g, updateGravity Update the list of ad-serving domains -h, --help, help Show this help dialog -l, logging Specify whether the Pi-hole log should be used From 72c972175dcb485fefc2de424c4df6eeea0c4d14 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 6 Nov 2023 13:53:08 +0100 Subject: [PATCH 133/200] Remove left-over parts of setupVars processing and only include pihole.toml once Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index c53eada4..8c2388d8 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -544,18 +544,6 @@ disk_usage() { done } -parse_pihole_toml() { - echo_current_diagnostic "Pi-hole configuration" - # If the file exists, - if [[ -r "${PIHOLE_FTL_CONF_FILE}" ]]; then - # parse it - parse_file "${PIHOLE_FTL_CONF_FILE}" - else - # If not, show an error - log_write "${CROSS} ${COL_RED}Could not read ${PIHOLE_FTL_CONF_FILE}.${COL_NC}" - fi -} - parse_locale() { local pihole_locale echo_current_diagnostic "Locale" @@ -1402,10 +1390,6 @@ upload_to_tricorder() { # Run through all the functions we made make_temporary_log initialize_debug -# TODO: Address the reliance on setupVars.conf here. Should debug read pihole.toml directly, or rely on pihole-FTL --config? -# setupVars.conf needs to be sourced before the networking so the values are -# available to the other functions -source_setup_variables check_component_versions # check_critical_program_versions diagnose_operating_system @@ -1419,7 +1403,6 @@ check_name_resolution check_dhcp_servers process_status ftl_full_status -parse_pihole_toml analyze_ftl_db analyze_gravity_list show_groups From de6e61705e7d93d9c0f07ba74f343f43220770de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 6 Nov 2023 21:40:32 +0100 Subject: [PATCH 134/200] Remove local.list and openVPN traces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 1 - gravity.sh | 25 +------------------------ 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 8c2388d8..55659690 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -75,7 +75,6 @@ PIHOLE_CRON_FILE="${CRON_D_DIRECTORY}/pihole" PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" -PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml" PIHOLE_CUSTOM_HOSTS_FILE="${PIHOLE_DIRECTORY}/custom.list" diff --git a/gravity.sh b/gravity.sh index edfe89a9..5c944454 100755 --- a/gravity.sh +++ b/gravity.sh @@ -36,9 +36,6 @@ blacklistFile="${piholeDir}/blacklist.txt" regexFile="${piholeDir}/regex.list" adListFile="${piholeDir}/adlists.list" -localList="${piholeDir}/local.list" -VPNList="/etc/openvpn/ipp.txt" - piholeGitDir="/etc/.pihole" GRAVITYDB=$(getFTLConfigValue files.gravity) gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql" @@ -299,12 +296,7 @@ migrate_to_database() { # Determine if DNS resolution is available before proceeding gravity_CheckDNSResolutionAvailable() { - local lookupDomain="pi.hole" - - # Determine if $localList does not exist, and ensure it is not empty - if [[ ! -e "${localList}" ]] || [[ -s "${localList}" ]]; then - lookupDomain="raw.githubusercontent.com" - fi + local lookupDomain="raw.githubusercontent.com" # Determine if $lookupDomain is resolvable if timeout 4 getent hosts "${lookupDomain}" &> /dev/null; then @@ -686,18 +678,6 @@ gravity_ShowCount() { gravity_Table_Count "vw_regex_whitelist" "regex allowed filters" } -# Create "localhost" entries into hosts format -gravity_generateLocalList() { - # Empty $localList if it already exists, otherwise, create it - echo "### Do not modify this file, it will be overwritten by pihole -g" > "${localList}" - chmod 644 "${localList}" - - # Add additional LAN hosts provided by OpenVPN (if available) - if [[ -f "${VPNList}" ]]; then - awk -F, '{printf $2"\t"$1".vpn\n"}' "${VPNList}" >> "${localList}" - fi -} - # Trap Ctrl-C gravity_Trap() { trap '{ echo -e "\\n\\n ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT @@ -884,9 +864,6 @@ if ! gravity_DownloadBlocklists; then exit 1 fi -# Create local.list -gravity_generateLocalList - # Update gravity timestamp update_gravity_timestamp From 766cb26af5884281f343b2cba87cf7cafc207447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 7 Nov 2023 22:24:34 +0100 Subject: [PATCH 135/200] Drop Fedora 36 and add Fedora 39 to the test suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 2 +- test/{_fedora_36.Dockerfile => _fedora_39.Dockerfile} | 2 +- test/{tox.fedora_36.ini => tox.fedora_39.ini} | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename test/{_fedora_36.Dockerfile => _fedora_39.Dockerfile} (97%) rename test/{tox.fedora_36.ini => tox.fedora_39.ini} (80%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8166d253..9d90b2e7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,9 +64,9 @@ jobs: ubuntu_23, centos_8, centos_9, - fedora_36, fedora_37, fedora_38, + fedora_39, ] env: DISTRO: ${{matrix.distro}} diff --git a/test/_fedora_36.Dockerfile b/test/_fedora_39.Dockerfile similarity index 97% rename from test/_fedora_36.Dockerfile rename to test/_fedora_39.Dockerfile index 847767e7..1727a3aa 100644 --- a/test/_fedora_36.Dockerfile +++ b/test/_fedora_39.Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:36 +FROM fedora:39 RUN dnf install -y git initscripts ENV GITDIR /etc/.pihole diff --git a/test/tox.fedora_36.ini b/test/tox.fedora_39.ini similarity index 80% rename from test/tox.fedora_36.ini rename to test/tox.fedora_39.ini index 1896a45f..5c8557c9 100644 --- a/test/tox.fedora_36.ini +++ b/test/tox.fedora_39.ini @@ -1,8 +1,8 @@ [tox] envlist = py3 -[testenv:py3] +[testenv] allowlist_externals = docker deps = -rrequirements.txt -commands = docker buildx build --load --progress plain -f _fedora_36.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _fedora_39.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py From f65b12643345c74d27c1c82a7bf3c177e609cce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 13 Nov 2023 21:12:28 +0100 Subject: [PATCH 136/200] Move custom.list to /hosts/custom.list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 2 +- automated install/basic-install.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 8c2388d8..c346da57 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -78,7 +78,7 @@ PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml" -PIHOLE_CUSTOM_HOSTS_FILE="${PIHOLE_DIRECTORY}/custom.list" +PIHOLE_CUSTOM_HOSTS_FILE="${PIHOLE_DIRECTORY}/hosts/custom.list" PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" # Read the value of an FTL config key. The value is printed to stdout. diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ef5add0e..9b619cbf 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1122,9 +1122,9 @@ installConfigs() { chown pihole:pihole "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" # Install empty custom.list file if it does not exist - if [[ ! -r "${PI_HOLE_CONFIG_DIR}/custom.list" ]]; then - if ! install -o pihole -g pihole -m 660 /dev/null "${PI_HOLE_CONFIG_DIR}/custom.list" &>/dev/null; then - printf " %b Error: Unable to initialize configuration file %s/custom.list\\n" "${COL_LIGHT_RED}" "${PI_HOLE_CONFIG_DIR}" + if [[ ! -r "${PI_HOLE_CONFIG_DIR}/hosts/custom.list" ]]; then + if ! install -D -T -o pihole -g pihole -m 660 /dev/null "${PI_HOLE_CONFIG_DIR}/hosts/custom.list" &>/dev/null; then + printf " %b Error: Unable to initialize configuration file %s/custom.list\\n" "${COL_LIGHT_RED}" "${PI_HOLE_CONFIG_DIR}/hosts" return 1 fi fi From 86aa6b1df64ad263cdc6b48f3d9ce870cf0a5fe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 6 Nov 2023 22:38:01 +0100 Subject: [PATCH 137/200] Improve v6 debug log and remove leftovers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 43 +++++++++------------------------ 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index c346da57..a45e52a9 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -49,7 +49,6 @@ FAQ_HARDWARE_REQUIREMENTS="${COL_CYAN}https://docs.pi-hole.net/main/prerequisite FAQ_HARDWARE_REQUIREMENTS_PORTS="${COL_CYAN}https://docs.pi-hole.net/main/prerequisites/#ports${COL_NC}" FAQ_HARDWARE_REQUIREMENTS_FIREWALLD="${COL_CYAN}https://docs.pi-hole.net/main/prerequisites/#firewalld${COL_NC}" FAQ_GATEWAY="${COL_CYAN}https://discourse.pi-hole.net/t/why-is-a-default-gateway-important-for-pi-hole/3546${COL_NC}" -FAQ_FTL_COMPATIBILITY="${COL_CYAN}https://github.com/pi-hole/FTL#compatibility-list${COL_NC}" # Other URLs we may use FORUMS_URL="${COL_CYAN}https://discourse.pi-hole.net${COL_NC}" @@ -78,7 +77,6 @@ PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml" -PIHOLE_CUSTOM_HOSTS_FILE="${PIHOLE_DIRECTORY}/hosts/custom.list" PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" # Read the value of an FTL config key. The value is printed to stdout. @@ -91,12 +89,12 @@ get_ftl_conf_value() { local default=$2 local value - # Obtain key=... setting from pihole-FTL.conf + # Obtain key=... setting from FTL directly if [[ -e "$PIHOLE_FTL_CONF_FILE" ]]; then # Constructed to return nothing when # a) the setting is not present in the config file, or # b) the setting is commented out (e.g. "#DBFILE=...") - value="$(sed -n -e "s/^\\s*$key=\\s*//p" ${PIHOLE_FTL_CONF_FILE})" + value="$(pihole-FTL --config "${key}")" fi # Test for missing value. Use default value in this case. @@ -107,9 +105,9 @@ get_ftl_conf_value() { echo "$value" } -PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "GRAVITYDB" "${PIHOLE_DIRECTORY}/gravity.db")" +PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "files.gravity" "${PIHOLE_DIRECTORY}/gravity.db")" -PIHOLE_FTL_DB_FILE="$(get_ftl_conf_value "DBFILE" "${PIHOLE_DIRECTORY}/pihole-FTL.db")" +PIHOLE_FTL_DB_FILE="$(get_ftl_conf_value "files.database" "${PIHOLE_DIRECTORY}/pihole-FTL.db")" PIHOLE_COMMAND="${BIN_DIRECTORY}/pihole" PIHOLE_COLTABLE_FILE="${BIN_DIRECTORY}/COL_TABLE" @@ -119,10 +117,8 @@ FTL_PID="${RUN_DIRECTORY}/pihole-FTL.pid" PIHOLE_LOG="${LOG_DIRECTORY}/pihole.log" PIHOLE_LOG_GZIPS="${LOG_DIRECTORY}/pihole.log.[0-9].*" PIHOLE_DEBUG_LOG="${LOG_DIRECTORY}/pihole_debug.log" -PIHOLE_FTL_LOG="$(get_ftl_conf_value "LOGFILE" "${LOG_DIRECTORY}/FTL.log")" - -# PIHOLE_WEB_SERVER_ACCESS_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/access-pihole.log" #TODO: FTL Error log? -# PIHOLE_WEB_SERVER_ERROR_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/error-pihole.log" #TODO: FTL Error log? +PIHOLE_FTL_LOG="$(get_ftl_conf_value "files.log.ftl" "${LOG_DIRECTORY}/FTL.log")" +PIHOLE_WEBSERVER_LOG="$(get_ftl_conf_value "files.log.webserver" "${LOG_DIRECTORY}/webserver.log")" RESOLVCONF="${ETC}/resolv.conf" DNSMASQ_CONF="${ETC}/dnsmasq.conf" @@ -132,8 +128,6 @@ PIHOLE_PROCESSES=( "pihole-FTL" ) # Store the required directories in an array so it can be parsed through REQUIRED_FILES=("${PIHOLE_CRON_FILE}" -# "${WEB_SERVER_CONFIG_FILE}" -# "${WEB_SERVER_CUSTOM_CONFIG_FILE}" "${PIHOLE_INSTALL_LOG_FILE}" "${PIHOLE_RAW_BLOCKLIST_FILES}" "${PIHOLE_LOCAL_HOSTS_FILE}" @@ -146,11 +140,9 @@ REQUIRED_FILES=("${PIHOLE_CRON_FILE}" "${PIHOLE_LOG_GZIPS}" "${PIHOLE_DEBUG_LOG}" "${PIHOLE_FTL_LOG}" -"${PIHOLE_WEB_SERVER_ACCESS_LOG_FILE}" -"${PIHOLE_WEB_SERVER_ERROR_LOG_FILE}" +"${PIHOLE_WEBSERVER_LOG}" "${RESOLVCONF}" "${DNSMASQ_CONF}" -"${PIHOLE_CUSTOM_HOSTS_FILE}" "${PIHOLE_VERSIONS_FILE}") DISCLAIMER="This process collects information from your Pi-hole, and optionally uploads it to a unique and random directory on tricorder.pi-hole.net. @@ -983,12 +975,6 @@ list_files_in_dir() { if [[ "${dir_to_parse}" == "${SHM_DIRECTORY}" ]]; then # SHM file - we do not want to see the content, but we want to see the files and their sizes log_write "$(ls -lh "${dir_to_parse}/")" - elif [[ "${dir_to_parse}" == "${WEB_SERVER_CONFIG_DIRECTORY_FEDORA}" ]]; then - # we want to see all files files in /etc/lighttpd/conf.d - log_write "$(ls -lh "${dir_to_parse}/" 2> /dev/null )" - elif [[ "${dir_to_parse}" == "${WEB_SERVER_CONFIG_DIRECTORY_DEBIAN}" ]]; then - # we want to see all files files in /etc/lighttpd/conf.d - log_write "$(ls -lh "${dir_to_parse}/"/ 2> /dev/null )" fi # Store the files found in an array @@ -1002,7 +988,6 @@ list_files_in_dir() { [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_RAW_BLOCKLIST_FILES}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_INSTALL_LOG_FILE}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_LOG}" ]] || \ - [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_WEB_SERVER_ACCESS_LOG_FILE}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_LOG_GZIPS}" ]]; then : elif [[ "${dir_to_parse}" == "${DNSMASQ_D_DIRECTORY}" ]]; then @@ -1017,8 +1002,8 @@ list_files_in_dir() { log_write "\\n${COL_GREEN}$(ls -lhd "${dir_to_parse}"/"${each_file}")${COL_NC}" # Check if the file we want to view has a limit (because sometimes we just need a little bit of info from the file, not the entire thing) case "${dir_to_parse}/${each_file}" in - # If it's Web server error log, give the first and last 25 lines - "${PIHOLE_WEB_SERVER_ERROR_LOG_FILE}") head_tail_log "${dir_to_parse}/${each_file}" 25 + # If it's Web server log, give the first and last 25 lines + "${PIHOLE_WEBSERVER_LOG}") head_tail_log "${dir_to_parse}/${each_file}" 25 ;; # Same for the FTL log "${PIHOLE_FTL_LOG}") head_tail_log "${dir_to_parse}/${each_file}" 35 @@ -1049,11 +1034,7 @@ show_content_of_pihole_files() { # Show the content of the files in each of Pi-hole's folders show_content_of_files_in_dir "${PIHOLE_DIRECTORY}" show_content_of_files_in_dir "${DNSMASQ_D_DIRECTORY}" - show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY}" - show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY_FEDORA}" - show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY_DEBIAN}" show_content_of_files_in_dir "${CRON_D_DIRECTORY}" - show_content_of_files_in_dir "${WEB_SERVER_LOG_DIRECTORY}" show_content_of_files_in_dir "${LOG_DIRECTORY}" show_content_of_files_in_dir "${SHM_DIRECTORY}" show_content_of_files_in_dir "${ETC}" @@ -1290,10 +1271,10 @@ spinner(){ analyze_pihole_log() { echo_current_diagnostic "Pi-hole log" local pihole_log_permissions - local logging_enabled + local queryLogging - logging_enabled=$(grep -c "^log-queries" /etc/dnsmasq.d/01-pihole.conf) - if [[ "${logging_enabled}" == "0" ]]; then + queryLogging=$(pihole-FTL --config dns.queryLogging) + if [[ "${queryLogging}" == "false" ]]; then # Inform user that logging has been disabled and pihole.log does not contain queries log_write "${INFO} Query logging is disabled" log_write "" From d854eb1a9789a99a24b3ca7b7c101418d47a383c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 7 Nov 2023 21:59:35 +0100 Subject: [PATCH 138/200] Add lshw to Pi-hole deps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9b619cbf..72f00992 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -332,7 +332,7 @@ package_manager_detect() { # Packages required to run this install script INSTALLER_DEPS=(git iproute2 dialog ca-certificates) # Packages required to run Pi-hole - PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) + PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq lshw) # If apt-get is not found, check for rpm. elif is_command rpm ; then @@ -349,7 +349,7 @@ package_manager_detect() { PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src|.riscv64)' | wc -l || true" OS_CHECK_DEPS=(grep bind-utils) INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates binutils) - PIHOLE_DEPS=(cronie curl findutils sudo unzip psmisc libcap nmap-ncat jq) + PIHOLE_DEPS=(cronie curl findutils sudo unzip psmisc libcap nmap-ncat jq lshw) # If neither apt-get or yum/dnf package managers were found else From d3813c4be5eb058ae27ff74167f6794b1c7a2e8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 8 Nov 2023 21:28:05 +0100 Subject: [PATCH 139/200] Update check for required ports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index a45e52a9..a3cfaedd 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -634,10 +634,8 @@ compare_port_to_service_assigned() { check_required_ports() { echo_current_diagnostic "Ports in use" - # Since Pi-hole needs 53 and 4711, check what they are being used by + # Since Pi-hole needs 53,80 and 443 check what they are being used by # so we can detect any issues - local resolver="pihole-FTL" - local web_server="pihole-FTL" local ftl="pihole-FTL" # Create an array for these ports in use ports_in_use=() @@ -656,17 +654,13 @@ check_required_ports() { local port_number port_number="$(echo "${ports_in_use[$i]}" | awk '{print $5}')" # | awk '{gsub(/^.*:/,"",$5);print $5}') - # Use a case statement to determine if the right services are using the right ports - case "$(echo "${port_number}" | rev | cut -d: -f1 | rev)" in - 53) compare_port_to_service_assigned "${resolver}" "${service_name}" "${protocol_type}:${port_number}" - ;; - 80) compare_port_to_service_assigned "${web_server}" "${service_name}" "${protocol_type}:${port_number}" - ;; - 4711) compare_port_to_service_assigned "${ftl}" "${service_name}" "${protocol_type}:${port_number}" - ;; + # Check if the right services are using the right ports + if [[ "$(echo "${port_number}" | rev | cut -d: -f1 | rev)" == @(53|80|443) ]]; then + compare_port_to_service_assigned "${ftl}" "${service_name}" "${protocol_type}:${port_number}" + else # If it's not a default port that Pi-hole needs, just print it out for the user to see - *) log_write " ${protocol_type}:${port_number} is in use by ${service_name:=}"; - esac + log_write " ${protocol_type}:${port_number} is in use by ${service_name:=}"; + fi done } From e7c89ce25fb3a8b4e59f61df63ab00838543ac59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 12 Nov 2023 20:45:45 +0100 Subject: [PATCH 140/200] No need to pass a default value to get_ftl_conf_value() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index a3cfaedd..1a3c2a34 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -80,34 +80,18 @@ PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml" PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" # Read the value of an FTL config key. The value is printed to stdout. -# -# Args: -# 1. The key to read -# 2. The default if the setting or config does not exist get_ftl_conf_value() { local key=$1 - local default=$2 local value - # Obtain key=... setting from FTL directly - if [[ -e "$PIHOLE_FTL_CONF_FILE" ]]; then - # Constructed to return nothing when - # a) the setting is not present in the config file, or - # b) the setting is commented out (e.g. "#DBFILE=...") - value="$(pihole-FTL --config "${key}")" - fi - - # Test for missing value. Use default value in this case. - if [[ -z "$value" ]]; then - value="$default" - fi - + # Obtain setting from FTL directly + value="$(pihole-FTL --config "${key}")" echo "$value" } -PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "files.gravity" "${PIHOLE_DIRECTORY}/gravity.db")" +PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "files.gravity")" -PIHOLE_FTL_DB_FILE="$(get_ftl_conf_value "files.database" "${PIHOLE_DIRECTORY}/pihole-FTL.db")" +PIHOLE_FTL_DB_FILE="$(get_ftl_conf_value "files.database")" PIHOLE_COMMAND="${BIN_DIRECTORY}/pihole" PIHOLE_COLTABLE_FILE="${BIN_DIRECTORY}/COL_TABLE" @@ -117,8 +101,8 @@ FTL_PID="${RUN_DIRECTORY}/pihole-FTL.pid" PIHOLE_LOG="${LOG_DIRECTORY}/pihole.log" PIHOLE_LOG_GZIPS="${LOG_DIRECTORY}/pihole.log.[0-9].*" PIHOLE_DEBUG_LOG="${LOG_DIRECTORY}/pihole_debug.log" -PIHOLE_FTL_LOG="$(get_ftl_conf_value "files.log.ftl" "${LOG_DIRECTORY}/FTL.log")" -PIHOLE_WEBSERVER_LOG="$(get_ftl_conf_value "files.log.webserver" "${LOG_DIRECTORY}/webserver.log")" +PIHOLE_FTL_LOG="$(get_ftl_conf_value "files.log.ftl")" +PIHOLE_WEBSERVER_LOG="$(get_ftl_conf_value "files.log.webserver")" RESOLVCONF="${ETC}/resolv.conf" DNSMASQ_CONF="${ETC}/dnsmasq.conf" @@ -1267,7 +1251,7 @@ analyze_pihole_log() { local pihole_log_permissions local queryLogging - queryLogging=$(pihole-FTL --config dns.queryLogging) + queryLogging="$(get_ftl_conf_value "dns.queryLogging")" if [[ "${queryLogging}" == "false" ]]; then # Inform user that logging has been disabled and pihole.log does not contain queries log_write "${INFO} Query logging is disabled" From 54ab71d8176acdfd1c0db676b8f06090cd4850ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 12 Nov 2023 21:20:03 +0100 Subject: [PATCH 141/200] Don't use hardcoded ports to check for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 1a3c2a34..703c308b 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -618,7 +618,7 @@ compare_port_to_service_assigned() { check_required_ports() { echo_current_diagnostic "Ports in use" - # Since Pi-hole needs 53,80 and 443 check what they are being used by + # Since Pi-hole needs various ports, check what they are being used by # so we can detect any issues local ftl="pihole-FTL" # Create an array for these ports in use @@ -628,6 +628,15 @@ check_required_ports() { ports_in_use+=( "$line" ) done < <( ss --listening --numeric --tcp --udp --processes --no-header ) + local ports_configured + # Get all configured ports + ports_configured="$(pihole-FTL --config "webserver.port")" + # Remove all non-didgits, split into an array at "," + ports_configured="${ports_configured//[!0-9,]/}" + mapfile -d "," -t ports_configured < <(echo "${ports_configured}") + # Add port 53 + ports_configured+=("53") + # Now that we have the values stored, for i in "${!ports_in_use[@]}"; do # loop through them and assign some local variables @@ -639,7 +648,7 @@ check_required_ports() { port_number="$(echo "${ports_in_use[$i]}" | awk '{print $5}')" # | awk '{gsub(/^.*:/,"",$5);print $5}') # Check if the right services are using the right ports - if [[ "$(echo "${port_number}" | rev | cut -d: -f1 | rev)" == @(53|80|443) ]]; then + if [[ ${ports_configured[*]} =~ $(echo "${port_number}" | rev | cut -d: -f1 | rev) ]]; then compare_port_to_service_assigned "${ftl}" "${service_name}" "${protocol_type}:${port_number}" else # If it's not a default port that Pi-hole needs, just print it out for the user to see From b61e8be3b46bae8be8fddb8b4d7315b0afd9801c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 12 Nov 2023 21:36:18 +0100 Subject: [PATCH 142/200] Address reviewer's comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: DL6ER Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 703c308b..931e95a4 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -82,11 +82,9 @@ PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" # Read the value of an FTL config key. The value is printed to stdout. get_ftl_conf_value() { local key=$1 - local value # Obtain setting from FTL directly - value="$(pihole-FTL --config "${key}")" - echo "$value" + pihole-FTL --config "${key}" } PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "files.gravity")" From ab09233753cd522ca84faacbe02d7bdbb7f5c40f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Nov 2023 10:17:34 +0000 Subject: [PATCH 143/200] Bump pytest-testinfra from 9.0.0 to 10.0.0 in /test Bumps [pytest-testinfra](https://github.com/pytest-dev/pytest-testinfra) from 9.0.0 to 10.0.0. - [Release notes](https://github.com/pytest-dev/pytest-testinfra/releases) - [Changelog](https://github.com/pytest-dev/pytest-testinfra/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-testinfra/compare/9.0.0...10.0.0) --- updated-dependencies: - dependency-name: pytest-testinfra dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 45c7c7c0..c3147a10 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.3 pytest-xdist == 3.3.1 -pytest-testinfra == 9.0.0 +pytest-testinfra == 10.0.0 tox == 4.11.3 From f4b67065ccb547c374afc42274207297748deb08 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Nov 2023 10:55:51 +0000 Subject: [PATCH 144/200] Bump pytest-xdist from 3.3.1 to 3.4.0 in /test Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.3.1 to 3.4.0. - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.3.1...v3.4.0) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index c3147a10..74c67fd9 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.3 -pytest-xdist == 3.3.1 +pytest-xdist == 3.4.0 pytest-testinfra == 10.0.0 tox == 4.11.3 From 19d3489bcbe5aca79ec08a38008b6d5a2f1dcaf1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 20:56:23 +0100 Subject: [PATCH 145/200] gravity_DownloadBlocklistFromUrl needs ${domain} but it was declared local before and was not passed as argument, making gravity currently depend on undefined behavior. It seems to be working well in the vast majority of cases, however, it seems we have at least one report where it is not working. Signed-off-by: DL6ER --- gravity.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index edfe89a9..421ec035 100755 --- a/gravity.sh +++ b/gravity.sh @@ -464,7 +464,7 @@ gravity_DownloadBlocklists() { if [[ "${check_url}" =~ ${regex} ]]; then echo -e " ${CROSS} Invalid Target" else - gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" "${adlist_type}" + gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" "${adlist_type}" "${domain}" fi echo "" done @@ -496,7 +496,7 @@ compareLists() { # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { - local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" gravity_type="${6}" + local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" gravity_type="${6}" domain="${7}" local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext # Create temp file to store content on disk instead of RAM From f16cf7178186e33004bcfb44c3838a9fd2912007 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 21:04:46 +0100 Subject: [PATCH 146/200] ${PIHOLE_DNS_1} is gone, use the first server from pihole-FTL --config dns.upstreams instead Signed-off-by: DL6ER --- gravity.sh | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index 421ec035..90cba0f6 100755 --- a/gravity.sh +++ b/gravity.sh @@ -538,12 +538,25 @@ gravity_DownloadBlocklistFromUrl() { fi;; esac + if [[ "${blocked}" == true ]]; then - printf -v ip_addr "%s" "${PIHOLE_DNS_1%#*}" - if [[ ${PIHOLE_DNS_1} != *"#"* ]]; then + # Get first defined upstream server + local upstream + upstream="$(getFTLConfigValue dns.upstreams)" + + # Isolate first upstream server from a string like + # [ 1.2.3.4#1234, 5.6.7.8#5678, ... ] + upstream="${upstream%%,*}" + upstream="${upstream##*[}" + upstream="${upstream%%]*}" + + # Get IP address and port of this upstream server + local ip_addr port + printf -v ip_addr "%s" "${upstream%#*}" + if [[ ${upstream} != *"#"* ]]; then port=53 else - printf -v port "%s" "${PIHOLE_DNS_1#*#}" + printf -v port "%s" "${upstream#*#}" fi ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}" | tail -1) if [[ $(echo "${url}" | awk -F '://' '{print $1}') = "https" ]]; then @@ -551,7 +564,7 @@ gravity_DownloadBlocklistFromUrl() { else port=80 fi bad_list=$(pihole -q -adlist "${domain}" | head -n1 | awk -F 'Match found in ' '{print $2}') - echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${PIHOLE_DNS_1} to download ${url}"; + echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${upstream} to download ${url}"; echo -ne " ${INFO} ${str} Pending..." cmd_ext="--resolve $domain:$port:$ip" fi From 4efcdf11890ca069019c5b9cb3c1206e10a7f73e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 21:06:09 +0100 Subject: [PATCH 147/200] Add missing double quotes to prevent globbing and word splitting Signed-off-by: DL6ER --- gravity.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index 90cba0f6..7a4e52c2 100755 --- a/gravity.sh +++ b/gravity.sh @@ -99,7 +99,7 @@ gravity_swap_databases() { # Number of available blocks on disk availableBlocks=$(stat -f --format "%a" "${gravityDIR}") # Number of blocks, used by gravity.db - gravityBlocks=$(stat --format "%b" ${gravityDBfile}) + gravityBlocks=$(stat --format "%b" "${gravityDBfile}") # Only keep the old database if available disk space is at least twice the size of the existing gravity.db. # Better be safe than sorry... oldAvail=false @@ -604,7 +604,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ "${success}" == true ]]; then if [[ "${httpCode}" == "304" ]]; then # Add domains to database table file - pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" database_adlist_status "${adlistID}" "2" done="true" # Check if $listCurlBuffer is a non-zero length file @@ -614,7 +614,7 @@ gravity_DownloadBlocklistFromUrl() { # Remove curl buffer file after its use rm "${listCurlBuffer}" # Add domains to database table file - pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" # Compare lists, are they identical? compareLists "${adlistID}" "${saveLocation}" done="true" @@ -630,7 +630,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ -r "${saveLocation}" ]]; then echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" # Add domains to database table file - pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" database_adlist_status "${adlistID}" "3" else echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" From df7633bd1b2f4c8e653620ead7c00ab86604428a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 21:08:06 +0100 Subject: [PATCH 148/200] Add missing value for ${gravityDBfile_default} Signed-off-by: DL6ER --- gravity.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/gravity.sh b/gravity.sh index 7a4e52c2..64ba662e 100755 --- a/gravity.sh +++ b/gravity.sh @@ -59,6 +59,7 @@ fi # Set this only after sourcing pihole-FTL.conf as the gravity database path may # have changed gravityDBfile="${GRAVITYDB}" +gravityDBfile_default="/etc/pihole/gravity.db" gravityTEMPfile="${GRAVITYDB}_temp" gravityDIR="$(dirname -- "${gravityDBfile}")" gravityOLDfile="${gravityDIR}/gravity_old.db" From cc333f79ccba8358b87bb38d6f3694ab3bf24809 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 21:10:22 +0100 Subject: [PATCH 149/200] Check if this domain is blocked by Pi-hole but only if the domain is not a local file or empty Signed-off-by: DL6ER --- gravity.sh | 102 ++++++++++++++++++++++++++++------------------------- 1 file changed, 53 insertions(+), 49 deletions(-) diff --git a/gravity.sh b/gravity.sh index 64ba662e..4aea4d68 100755 --- a/gravity.sh +++ b/gravity.sh @@ -517,57 +517,61 @@ gravity_DownloadBlocklistFromUrl() { str="Status:" echo -ne " ${INFO} ${str} Pending..." blocked=false - case $(getFTLConfigValue dns.blocking.mode) in - "IP-NODATA-AAAA"|"IP") - # Get IP address of this domain - ip="$(dig "${domain}" +short)" - # Check if this IP matches any IP of the system - if [[ -n "${ip}" && $(grep -Ec "inet(|6) ${ip}" <<< "$(ip a)") -gt 0 ]]; then - blocked=true - fi;; - "NXDOMAIN") - if [[ $(dig "${domain}" | grep "NXDOMAIN" -c) -ge 1 ]]; then - blocked=true - fi;; - "NODATA") - if [[ $(dig "${domain}" | grep "NOERROR" -c) -ge 1 ]] && [[ -z $(dig +short "${domain}") ]]; then - blocked=true - fi;; - "NULL"|*) - if [[ $(dig "${domain}" +short | grep "0.0.0.0" -c) -ge 1 ]]; then - blocked=true - fi;; - esac - - if [[ "${blocked}" == true ]]; then - # Get first defined upstream server - local upstream - upstream="$(getFTLConfigValue dns.upstreams)" - - # Isolate first upstream server from a string like - # [ 1.2.3.4#1234, 5.6.7.8#5678, ... ] - upstream="${upstream%%,*}" - upstream="${upstream##*[}" - upstream="${upstream%%]*}" - - # Get IP address and port of this upstream server - local ip_addr port - printf -v ip_addr "%s" "${upstream%#*}" - if [[ ${upstream} != *"#"* ]]; then - port=53 - else - printf -v port "%s" "${upstream#*#}" - fi - ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}" | tail -1) - if [[ $(echo "${url}" | awk -F '://' '{print $1}') = "https" ]]; then - port=443; - else port=80 + # Check if this domain is blocked by Pi-hole but only if the domain is not a + # local file or empty + if [[ $url != "file"* ]] && [[ -n "${domain}" ]]; then + case $(getFTLConfigValue dns.blocking.mode) in + "IP-NODATA-AAAA"|"IP") + # Get IP address of this domain + ip="$(dig "${domain}" +short)" + # Check if this IP matches any IP of the system + if [[ -n "${ip}" && $(grep -Ec "inet(|6) ${ip}" <<< "$(ip a)") -gt 0 ]]; then + blocked=true + fi;; + "NXDOMAIN") + if [[ $(dig "${domain}" | grep "NXDOMAIN" -c) -ge 1 ]]; then + blocked=true + fi;; + "NODATA") + if [[ $(dig "${domain}" | grep "NOERROR" -c) -ge 1 ]] && [[ -z $(dig +short "${domain}") ]]; then + blocked=true + fi;; + "NULL"|*) + if [[ $(dig "${domain}" +short | grep "0.0.0.0" -c) -ge 1 ]]; then + blocked=true + fi;; + esac + + if [[ "${blocked}" == true ]]; then + # Get first defined upstream server + local upstream + upstream="$(getFTLConfigValue dns.upstreams)" + + # Isolate first upstream server from a string like + # [ 1.2.3.4#1234, 5.6.7.8#5678, ... ] + upstream="${upstream%%,*}" + upstream="${upstream##*[}" + upstream="${upstream%%]*}" + + # Get IP address and port of this upstream server + local ip_addr port + printf -v ip_addr "%s" "${upstream%#*}" + if [[ ${upstream} != *"#"* ]]; then + port=53 + else + printf -v port "%s" "${upstream#*#}" + fi + ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}" | tail -1) + if [[ $(echo "${url}" | awk -F '://' '{print $1}') = "https" ]]; then + port=443; + else port=80 + fi + bad_list=$(pihole -q -adlist "${domain}" | head -n1 | awk -F 'Match found in ' '{print $2}') + echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${upstream} to download ${url}"; + echo -ne " ${INFO} ${str} Pending..." + cmd_ext="--resolve $domain:$port:$ip" fi - bad_list=$(pihole -q -adlist "${domain}" | head -n1 | awk -F 'Match found in ' '{print $2}') - echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${upstream} to download ${url}"; - echo -ne " ${INFO} ${str} Pending..." - cmd_ext="--resolve $domain:$port:$ip" fi # shellcheck disable=SC2086 From c785667efea9e579db37c7d450bd3fbe9e7612a0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 23 Nov 2023 09:56:43 +0100 Subject: [PATCH 150/200] Trim leading and trailing spaces and tabs in upstream servers (if any) Signed-off-by: DL6ER --- gravity.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 4aea4d68..20ad6215 100755 --- a/gravity.sh +++ b/gravity.sh @@ -553,6 +553,9 @@ gravity_DownloadBlocklistFromUrl() { upstream="${upstream%%,*}" upstream="${upstream##*[}" upstream="${upstream%%]*}" + # Trim leading and trailing spaces and tabs + upstream="${upstream#"${upstream%%[![:space:]]*}"}" + upstream="${upstream%"${upstream##*[![:space:]]}"}" # Get IP address and port of this upstream server local ip_addr port @@ -567,8 +570,7 @@ gravity_DownloadBlocklistFromUrl() { port=443; else port=80 fi - bad_list=$(pihole -q -adlist "${domain}" | head -n1 | awk -F 'Match found in ' '{print $2}') - echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${upstream} to download ${url}"; + echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by one of your lists. Using DNS server ${upstream} instead"; echo -ne " ${INFO} ${str} Pending..." cmd_ext="--resolve $domain:$port:$ip" fi From a6565bf9a17ad4998b00ca239be9044be7f51674 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 23 Nov 2023 22:07:31 +0100 Subject: [PATCH 151/200] Support special webserver.port ports ending in "s" (secure) and "r" (redirect) Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 43 ++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 449f146f..000c0717 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -21,14 +21,31 @@ TestAPIAvailability() { # as we are running locally, we can get the port value from FTL directly - PORT="$(pihole-FTL --config webserver.port)" - PORT="${PORT%%,*}" + local ports port availabilityResonse + ports="$(pihole-FTL --config webserver.port)" + port="${ports%%,*}" + + # if the port ends with an "s", it is a secure connection + if [ "${port#"${port%?}"}" = "s" ]; then + # remove the "s" from the port + API_PROT="https" + API_PORT="${port%?}" + elif [ "${port#"${port%?}"}" = "r" ]; then + # if the port ends in "r", it is a redirect + API_PROT="http" + # remove the "r" from the port + API_PORT="${port%?}" + else + API_PROT="http" + API_PORT="${port}" + fi - availabilityResonse=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:${PORT}/api/auth") + API_URL="${API_PROT}://localhost:${API_PORT}/api" + availabilityResonse=$(curl -skSL -o /dev/null -w "%{http_code}" "${API_URL}/auth") - # test if http status code was 200 (OK) or 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 401 ]; then - echo "API not available at: http://localhost:${PORT}/api" + # test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + echo "API not available at: ${API_URL}" echo "Exiting." exit 1 fi @@ -54,15 +71,15 @@ Authenthication() { } LoginAPI() { - sessionResponse="$(curl --silent -X POST "http://localhost:${PORT}/api/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" + sessionResponse="$(curl -skSL -X POST "${API_URL}/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" if [ -z "${sessionResponse}" ]; then echo "No response from FTL server. Please check connectivity" exit 1 fi - # obtain validity and session ID from session response - validSession=$(echo "${sessionResponse}"| jq .session.valid 2>/dev/null) - SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null) + # obtain validity and session ID from session response + validSession=$(echo "${sessionResponse}"| jq .session.valid 2>/dev/null) + SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null) } DeleteSession() { @@ -70,7 +87,7 @@ DeleteSession() { # SID is not null (successful authenthication only), delete the session if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then # Try to delete the session. Omit the output, but get the http status code - deleteResponse=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE "http://localhost:${PORT}/api/auth" -H "Accept: application/json" -H "sid: ${SID}") + deleteResponse=$(curl -skSL -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}/auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; @@ -84,14 +101,14 @@ DeleteSession() { GetFTLData() { local data response status # get the data from querying the API as well as the http status code - response=$(curl -s -w "%{http_code}" -X GET "http://localhost:${PORT}/api$1" -H "Accept: application/json" -H "sid: ${SID}" ) + response=$(curl -skSL -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" ) # status are the last 3 characters status=$(printf %s "${response#"${response%???}"}") # data is everything from response without the last 3 characters data=$(printf %s "${response%???}") - if [ "${status}" = 200 ]; then + if [ "${status}" = 200 ] || [ "${status}" = 308 ]; then # response OK echo "${data}" elif [ "${status}" = 000 ]; then From 1276242a4ea3164cf2c24c9727fa7396b1495936 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 25 Nov 2023 10:16:35 +0000 Subject: [PATCH 152/200] Bump pytest-xdist from 3.4.0 to 3.5.0 in /test Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.4.0 to 3.5.0. - [Release notes](https://github.com/pytest-dev/pytest-xdist/releases) - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.4.0...v3.5.0) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 74c67fd9..799e3fad 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.3 -pytest-xdist == 3.4.0 +pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.11.3 From 974fea592df99d97332763b6f9cf9812a6c907a4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 Nov 2023 00:05:55 +0100 Subject: [PATCH 153/200] Iterate over ports, skip redirected ports Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 64 +++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 000c0717..b50e416a 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -25,26 +25,48 @@ TestAPIAvailability() { ports="$(pihole-FTL --config webserver.port)" port="${ports%%,*}" - # if the port ends with an "s", it is a secure connection - if [ "${port#"${port%?}"}" = "s" ]; then - # remove the "s" from the port - API_PROT="https" - API_PORT="${port%?}" - elif [ "${port#"${port%?}"}" = "r" ]; then - # if the port ends in "r", it is a redirect - API_PROT="http" - # remove the "r" from the port - API_PORT="${port%?}" - else - API_PROT="http" - API_PORT="${port}" - fi + # Iterate over comma separated list of ports + while [ "${port}" != "${ports}" ]; do + # if the port ends with an "s", it is a secure connection + if [ "${port#"${port%?}"}" = "s" ]; then + # remove the "s" from the port + API_PROT="https" + API_PORT="${port%?}" + elif [ "${port#"${port%?}"}" = "r" ]; then + # Ignore this port + API_PORT="0" + else + API_PROT="http" + API_PORT="${port}" + fi + + if [ ! "${API_PORT}" = "0" ]; then + # If the port is of form "ip:port", we need to remove everything before + # the last ":" in the string, e.g., "[::]:80" -> "80" + if [ "${API_PORT#*:}" != "${API_PORT}" ]; then + API_PORT="${API_PORT##*:}" + fi - API_URL="${API_PROT}://localhost:${API_PORT}/api" - availabilityResonse=$(curl -skSL -o /dev/null -w "%{http_code}" "${API_URL}/auth") + API_URL="${API_PROT}://localhost:${API_PORT}/api" + availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}/auth") + + # test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + API_PORT="0" + else + # API is available at this port/protocol combination + break + fi + fi + + # remove the first port from the list + ports="${ports#*,}" + # get the next port + port="${ports%%,*}" + done - # test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + # if API_PORT is 0, no working API port was found + if [ "${API_PORT}" = "0" ]; then echo "API not available at: ${API_URL}" echo "Exiting." exit 1 @@ -71,7 +93,7 @@ Authenthication() { } LoginAPI() { - sessionResponse="$(curl -skSL -X POST "${API_URL}/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" + sessionResponse="$(curl -skS -X POST "${API_URL}/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" if [ -z "${sessionResponse}" ]; then echo "No response from FTL server. Please check connectivity" @@ -87,7 +109,7 @@ DeleteSession() { # SID is not null (successful authenthication only), delete the session if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then # Try to delete the session. Omit the output, but get the http status code - deleteResponse=$(curl -skSL -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}/auth" -H "Accept: application/json" -H "sid: ${SID}") + deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}/auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; @@ -101,7 +123,7 @@ DeleteSession() { GetFTLData() { local data response status # get the data from querying the API as well as the http status code - response=$(curl -skSL -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" ) + response=$(curl -skS -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" ) # status are the last 3 characters status=$(printf %s "${response#"${response%???}"}") From 6016131280ce5f24cc53fa7989026f593f4d2d4f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 Nov 2023 22:59:49 +0100 Subject: [PATCH 154/200] Ensure we also check the last port Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index b50e416a..2952fb43 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -26,16 +26,18 @@ TestAPIAvailability() { port="${ports%%,*}" # Iterate over comma separated list of ports - while [ "${port}" != "${ports}" ]; do + while [ -n "${ports}" ]; do # if the port ends with an "s", it is a secure connection if [ "${port#"${port%?}"}" = "s" ]; then # remove the "s" from the port API_PROT="https" API_PORT="${port%?}" elif [ "${port#"${port%?}"}" = "r" ]; then - # Ignore this port + # Ignore this port, the client may not be able to follow the + # redirected target when FTL is not used as local resolver API_PORT="0" else + # otherwise it is an insecure (plain HTTP) connection API_PROT="http" API_PORT="${port}" fi @@ -50,8 +52,9 @@ TestAPIAvailability() { API_URL="${API_PROT}://localhost:${API_PORT}/api" availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}/auth") - # test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) + # Test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + # API is not available at this port/protocol combination API_PORT="0" else # API is available at this port/protocol combination @@ -59,9 +62,9 @@ TestAPIAvailability() { fi fi - # remove the first port from the list + # If the loop has not been broken, remove the first port from the list + # and get the next port ports="${ports#*,}" - # get the next port port="${ports%%,*}" done @@ -192,3 +195,6 @@ secretRead() { # restore original terminal settings stty "${stty_orig}" } + + +TestAPIAvailability From 96bf07863f6bdd4d9cad91d5444fef5f85d47b61 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 1 Dec 2023 09:10:06 +0100 Subject: [PATCH 155/200] Use CHAOS TXT local.api.txt instead of trying to parse pihole-FTL --config webserver.ports Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 92 +++++++++++++++++++-------------------- advanced/Scripts/query.sh | 6 +-- 2 files changed, 48 insertions(+), 50 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 2952fb43..46da37cd 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -21,62 +21,60 @@ TestAPIAvailability() { # as we are running locally, we can get the port value from FTL directly - local ports port availabilityResonse - ports="$(pihole-FTL --config webserver.port)" - port="${ports%%,*}" - - # Iterate over comma separated list of ports - while [ -n "${ports}" ]; do - # if the port ends with an "s", it is a secure connection - if [ "${port#"${port%?}"}" = "s" ]; then - # remove the "s" from the port - API_PROT="https" - API_PORT="${port%?}" - elif [ "${port#"${port%?}"}" = "r" ]; then - # Ignore this port, the client may not be able to follow the - # redirected target when FTL is not used as local resolver - API_PORT="0" - else - # otherwise it is an insecure (plain HTTP) connection - API_PROT="http" - API_PORT="${port}" - fi + local chaos_api_list availabilityResonse - if [ ! "${API_PORT}" = "0" ]; then - # If the port is of form "ip:port", we need to remove everything before - # the last ":" in the string, e.g., "[::]:80" -> "80" - if [ "${API_PORT#*:}" != "${API_PORT}" ]; then - API_PORT="${API_PORT##*:}" - fi + # Query the API URLs from FTL using CHAOS TXT local.api.ftl + # The result is a space-separated enumeration of full URLs + # e.g., "http://localhost:80/api" "https://localhost:443/api" + chaos_api_list="$(dig +short chaos txt local.api.ftl @127.0.0.1)" - API_URL="${API_PROT}://localhost:${API_PORT}/api" - availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}/auth") + # If the query was not successful, the variable is empty + if [ -z "${chaos_api_list}" ]; then + echo "API not available. Please check connectivity" + exit 1 + fi - # Test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then - # API is not available at this port/protocol combination - API_PORT="0" - else - # API is available at this port/protocol combination - break - fi + # Iterate over space-separated list of URLs + while [ -n "${chaos_api_list}" ]; do + # Get the first URL + API_URL="${chaos_api_list%% *}" + # Strip leading and trailing quotes + API_URL="${API_URL%\"}" + API_URL="${API_URL#\"}" + + # Test if the API is available at this URL + availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}auth") + + # Test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + # API is not available at this port/protocol combination + API_PORT="" + else + # API is available at this URL combination + break fi - # If the loop has not been broken, remove the first port from the list - # and get the next port - ports="${ports#*,}" - port="${ports%%,*}" + # Remove the first URL from the list + local last_api_list + last_api_list="${chaos_api_list}" + chaos_api_list="${chaos_api_list#* }" + + # If the list did not change, we are at the last element + if [ "${last_api_list}" = "${chaos_api_list}" ]; then + # Remove the last element + chaos_api_list="" + fi done - # if API_PORT is 0, no working API port was found - if [ "${API_PORT}" = "0" ]; then + # if API_PORT is empty, no working API port was found + if [ -n "${API_PORT}" ]; then echo "API not available at: ${API_URL}" echo "Exiting." exit 1 fi } -Authenthication() { +Authentication() { # Try to authenticate LoginAPI @@ -96,7 +94,7 @@ Authenthication() { } LoginAPI() { - sessionResponse="$(curl -skS -X POST "${API_URL}/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" + sessionResponse="$(curl -skS -X POST "${API_URL}auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" if [ -z "${sessionResponse}" ]; then echo "No response from FTL server. Please check connectivity" @@ -108,11 +106,11 @@ LoginAPI() { } DeleteSession() { - # if a valid Session exists (no password required or successful authenthication) and - # SID is not null (successful authenthication only), delete the session + # if a valid Session exists (no password required or successful Authentication) and + # SID is not null (successful Authentication only), delete the session if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then # Try to delete the session. Omit the output, but get the http status code - deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}/auth" -H "Accept: application/json" -H "sid: ${SID}") + deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 2279df85..62d29d5b 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -121,14 +121,14 @@ Main(){ # or b) for the /search endpoint (webserver.api.searchAPIauth) no authentication is required. # Therefore, we try to query directly without authentication but do authenticat if 401 is returned - data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}") + data=$(GetFTLData "search/${domain}?N=${max_results}&partial=${partial}") if [ "${data}" = 401 ]; then # Unauthenticated, so authenticate with the FTL server required - Authenthication + Authentication # send query again - data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}") + data=$(GetFTLData "search/${domain}?N=${max_results}&partial=${partial}") fi GenerateOutput "${data}" From 29d010dc2ccc04cd6563d8154415e2070b5bd9a7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 1 Dec 2023 10:21:02 +0100 Subject: [PATCH 156/200] Use files.gravity_tmp as temporary directory for the intermediate lists Signed-off-by: DL6ER --- gravity.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index edfe89a9..8dc8edd8 100755 --- a/gravity.sh +++ b/gravity.sh @@ -41,6 +41,7 @@ VPNList="/etc/openvpn/ipp.txt" piholeGitDir="/etc/.pihole" GRAVITYDB=$(getFTLConfigValue files.gravity) +GRAVITY_TMPDIR=$(getFTLConfigValue files.gravity_tmp) gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql" gravityDBcopy="${piholeGitDir}/advanced/Templates/gravity_copy.sql" @@ -48,9 +49,7 @@ domainsExtension="domains" curl_connect_timeout=10 -# Set up tmp dir variable in case it's not configured -: "${GRAVITY_TMPDIR:=/tmp}" - +# Check gravity temp directory if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then echo -e " ${COL_LIGHT_RED}Gravity temporary directory does not exist or is not a writeable directory, falling back to /tmp. ${COL_NC}" GRAVITY_TMPDIR="/tmp" From f5fe550a2e27e5b81ffbe55d3f209c890dda3b94 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Dec 2023 10:35:21 +0000 Subject: [PATCH 157/200] Bump tox from 4.11.3 to 4.11.4 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.11.3 to 4.11.4. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.11.3...4.11.4) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 799e3fad..bfc6d027 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.3 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.11.3 +tox == 4.11.4 From 32a741b5c74bab31ed5c02f5ba33f65f631fea7f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 2 Dec 2023 22:42:36 +0100 Subject: [PATCH 158/200] We do not follow 308 but FTL also doesn't suggest it Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 46da37cd..b7bc2a86 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -25,7 +25,7 @@ TestAPIAvailability() { # Query the API URLs from FTL using CHAOS TXT local.api.ftl # The result is a space-separated enumeration of full URLs - # e.g., "http://localhost:80/api" "https://localhost:443/api" + # e.g., "http://localhost:80/api/" "https://localhost:443/api/" chaos_api_list="$(dig +short chaos txt local.api.ftl @127.0.0.1)" # If the query was not successful, the variable is empty @@ -45,8 +45,8 @@ TestAPIAvailability() { # Test if the API is available at this URL availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}auth") - # Test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + # Test if http status code was 200 (OK) or 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 401 ]; then # API is not available at this port/protocol combination API_PORT="" else @@ -131,7 +131,7 @@ GetFTLData() { # data is everything from response without the last 3 characters data=$(printf %s "${response%???}") - if [ "${status}" = 200 ] || [ "${status}" = 308 ]; then + if [ "${status}" = 200 ]; then # response OK echo "${data}" elif [ "${status}" = 000 ]; then @@ -193,6 +193,3 @@ secretRead() { # restore original terminal settings stty "${stty_orig}" } - - -TestAPIAvailability From 2681835f94ec072e183d6cd6de267f25b93fb0a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 6 Dec 2023 00:21:04 +0100 Subject: [PATCH 159/200] Treat FTL return data as strings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/api.sh | 2 +- advanced/Scripts/query.sh | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 449f146f..bf6e3654 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -93,7 +93,7 @@ GetFTLData() { if [ "${status}" = 200 ]; then # response OK - echo "${data}" + printf %s "${data}" elif [ "${status}" = 000 ]; then # connection lost echo "000" diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 2279df85..789efe75 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -46,14 +46,14 @@ GenerateOutput(){ data="${1}" # construct a new json for the list results where each object contains the domain and the related type - lists_data=$(echo "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]') + lists_data=$(printf %s "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]') # construct a new json for the gravity results where each object contains the adlist URL and the related domains - gravity_data=$(echo "${data}" | jq '.search.gravity | group_by(.address) | map({ address: (.[0].address), domains: [.[] | .domain] })') + gravity_data=$(printf %s "${data}" | jq '.search.gravity | group_by(.address) | map({ address: (.[0].address), domains: [.[] | .domain] })') # number of objects in each json - num_gravity=$(echo "${gravity_data}" | jq length ) - num_lists=$(echo "${lists_data}" | jq length ) + num_gravity=$(printf %s "${gravity_data}" | jq length ) + num_lists=$(printf %s "${lists_data}" | jq length ) if [ "${partial}" = true ]; then search_type_str="partially" @@ -66,7 +66,7 @@ GenerateOutput(){ if [ "${num_lists}" -gt 0 ]; then # Convert the data to a csv, each line is a "domain,type" string # not using jq's @csv here as it quotes each value individually - lists_data_csv=$(echo "${lists_data}" | jq --raw-output '.[] | [.domain, .type] | join(",")' ) + lists_data_csv=$(printf %s "${lists_data}" | jq --raw-output '.[] | [.domain, .type] | join(",")' ) # Generate output for each csv line, separating line in a domain and type substring at the ',' echo "${lists_data_csv}" | while read -r line; do @@ -79,7 +79,7 @@ GenerateOutput(){ if [ "${num_gravity}" -gt 0 ]; then # Convert the data to a csv, each line is a "URL,domain,domain,...." string # not using jq's @csv here as it quotes each value individually - gravity_data_csv=$(echo "${gravity_data}" | jq --raw-output '.[] | [.address, .domains[]] | join(",")' ) + gravity_data_csv=$(printf %s "${gravity_data}" | jq --raw-output '.[] | [.address, .domains[]] | join(",")' ) # Generate line-by-line output for each csv line echo "${gravity_data_csv}" | while read -r line; do From 159817b7e286c4015030bfe61c6100b7d5b2c028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 6 Dec 2023 20:35:26 +0100 Subject: [PATCH 160/200] Use development-v6 as ftl branch for binary test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/test_any_automated_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 7d9d15f6..62588c0a 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -465,7 +465,7 @@ def test_FTL_development_binary_installed_and_responsive_no_errors(host): source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) - echo "development" > /etc/pihole/ftlbranch + echo "development-v6" > /etc/pihole/ftlbranch binary="pihole-FTL${funcOutput##*pihole-FTL}" theRest="${funcOutput%pihole-FTL*}" FTLdetect "${binary}" "${theRest}" From 8a71e4253e0733fec4a4cb805ea219ff8ec8dab2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Dec 2023 17:39:18 +0000 Subject: [PATCH 161/200] Bump actions/stale from 8.0.0 to 9.0.0 Bumps [actions/stale](https://github.com/actions/stale) from 8.0.0 to 9.0.0. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v8.0.0...v9.0.0) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yml | 2 +- .github/workflows/stale_pr.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index d9de09d2..74ee9bef 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -17,7 +17,7 @@ jobs: issues: write steps: - - uses: actions/stale@v8.0.0 + - uses: actions/stale@v9.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 30 diff --git a/.github/workflows/stale_pr.yml b/.github/workflows/stale_pr.yml index 2db2a25d..96650818 100644 --- a/.github/workflows/stale_pr.yml +++ b/.github/workflows/stale_pr.yml @@ -17,7 +17,7 @@ jobs: pull-requests: write steps: - - uses: actions/stale@v8.0.0 + - uses: actions/stale@v9.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} # Do not automatically mark PR/issue as stale From e49d7fa5f1dd19a80c798318a270e80f0a6d9415 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Dec 2023 17:39:42 +0000 Subject: [PATCH 162/200] Bump actions/setup-python from 4.7.1 to 5.0.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.7.1 to 5.0.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.7.1...v5.0.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8166d253..ec4da6bb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,7 +75,7 @@ jobs: uses: actions/checkout@v4.1.1 - name: Set up Python 3.10 - uses: actions/setup-python@v4.7.1 + uses: actions/setup-python@v5.0.0 with: python-version: "3.10" From a87d1bbc4fcf107753324f551ac54bfe7bc45a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 14:22:43 +0100 Subject: [PATCH 163/200] Remove pihole.sudo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole.sudo | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 advanced/Templates/pihole.sudo diff --git a/advanced/Templates/pihole.sudo b/advanced/Templates/pihole.sudo deleted file mode 100644 index 708309be..00000000 --- a/advanced/Templates/pihole.sudo +++ /dev/null @@ -1,9 +0,0 @@ -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Allows the WebUI to use Pi-hole commands -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. -# From 54186a63eee39a2e420dda8d170807e3492bbc3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 14:33:13 +0100 Subject: [PATCH 164/200] Remove test for sudo file as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/test_any_automated_install.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 62588c0a..c53070dc 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -174,10 +174,6 @@ def test_installPihole_fresh_install_readableFiles(host): ) actual_rc = host.run(check_man).rc assert exit_status_success == actual_rc - # check not readable sudoers file - check_sudo = test_cmd.format("r", "/etc/sudoers.d/pihole", piholeuser) - actual_rc = host.run(check_sudo).rc - assert exit_status_success != actual_rc # check not readable cron file check_sudo = test_cmd.format("x", "/etc/cron.d/", piholeuser) actual_rc = host.run(check_sudo).rc From 274d4c263c2ec4c8ef3d3b4a55bae45aa3fb694b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 21:23:39 +0100 Subject: [PATCH 165/200] Add bash-completion to PIHOLE_DEPS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 72f00992..f5bf15e6 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -332,7 +332,7 @@ package_manager_detect() { # Packages required to run this install script INSTALLER_DEPS=(git iproute2 dialog ca-certificates) # Packages required to run Pi-hole - PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq lshw) + PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq lshw bash-completion) # If apt-get is not found, check for rpm. elif is_command rpm ; then @@ -349,7 +349,7 @@ package_manager_detect() { PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src|.riscv64)' | wc -l || true" OS_CHECK_DEPS=(grep bind-utils) INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates binutils) - PIHOLE_DEPS=(cronie curl findutils sudo unzip psmisc libcap nmap-ncat jq lshw) + PIHOLE_DEPS=(cronie curl findutils sudo unzip psmisc libcap nmap-ncat jq lshw bash-completion) # If neither apt-get or yum/dnf package managers were found else From fe7299323c389ca86fe941ecadaee6ce082695a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 21:40:03 +0100 Subject: [PATCH 166/200] Update bash_completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/bash-completion/pihole | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 305a3f5b..b2740724 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -1,5 +1,5 @@ _pihole() { - local cur prev opts opts_admin opts_checkout opts_debug opts_interface opts_logging opts_privacy opts_query opts_update opts_version + local cur prev opts opts_checkout opts_debug opts_logging opts_query opts_update opts_version COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" @@ -7,17 +7,13 @@ _pihole() { case "${prev}" in "pihole") - opts="admin blacklist checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" + opts="blacklist checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) ;; "whitelist"|"blacklist"|"wildcard"|"regex") opts_lists="\--delmode \--noreload \--quiet \--list \--nuke" COMPREPLY=( $(compgen -W "${opts_lists}" -- ${cur}) ) ;; - "admin") - opts_admin="celsius fahrenheit interface kelvin password privacylevel" - COMPREPLY=( $(compgen -W "${opts_admin}" -- ${cur}) ) - ;; "checkout") opts_checkout="core ftl web master dev" COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) @@ -31,7 +27,7 @@ _pihole() { COMPREPLY=( $(compgen -W "${opts_logging}" -- ${cur}) ) ;; "query") - opts_query="-adlist -all -exact" + opts_query="--partial --all" COMPREPLY=( $(compgen -W "${opts_query}" -- ${cur}) ) ;; "updatePihole"|"-up") @@ -41,23 +37,7 @@ _pihole() { "version") opts_version="\--admin \--current \--ftl \--hash \--latest \--pihole" COMPREPLY=( $(compgen -W "${opts_version}" -- ${cur}) ) - ;; - "interface") - if ( [[ "$prev2" == "admin" ]] || [[ "$prev2" == "-a" ]] ); then - opts_interface="$(cat /proc/net/dev | cut -d: -s -f1)" - COMPREPLY=( $(compgen -W "${opts_interface}" -- ${cur}) ) - else - return 1 - fi - ;; - "privacylevel") - if ( [[ "$prev2" == "admin" ]] || [[ "$prev2" == "-a" ]] ); then - opts_privacy="0 1 2 3" - COMPREPLY=( $(compgen -W "${opts_privacy}" -- ${cur}) ) - else - return 1 - fi - ;; + ;; "core"|"admin"|"ftl") if [[ "$prev2" == "checkout" ]]; then opts_checkout="master dev" From 6d02d4056f6abd63ff3d9479979502b6523f715e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 22:01:30 +0100 Subject: [PATCH 167/200] Rename option --admin to --web in version function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/version.sh | 16 ++++++++-------- advanced/bash-completion/pihole | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index e3b4a6ae..2983c04e 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -24,7 +24,7 @@ fi getLocalVersion() { case ${1} in "Pi-hole" ) echo "${CORE_VERSION:=N/A}";; - "web" ) echo "${WEB_VERSION:=N/A}";; + "web" ) echo "${WEB_VERSION:=N/A}";; "FTL" ) echo "${FTL_VERSION:=N/A}";; esac } @@ -32,7 +32,7 @@ getLocalVersion() { getLocalHash() { case ${1} in "Pi-hole" ) echo "${CORE_HASH:=N/A}";; - "web" ) echo "${WEB_HASH:=N/A}";; + "web" ) echo "${WEB_HASH:=N/A}";; "FTL" ) echo "${FTL_HASH:=N/A}";; esac } @@ -40,7 +40,7 @@ getLocalHash() { getRemoteHash(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_HASH:=N/A}";; - "web" ) echo "${GITHUB_WEB_HASH:=N/A}";; + "web" ) echo "${GITHUB_WEB_HASH:=N/A}";; "FTL" ) echo "${GITHUB_FTL_HASH:=N/A}";; esac } @@ -48,7 +48,7 @@ getRemoteHash(){ getRemoteVersion(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_VERSION:=N/A}";; - "web" ) echo "${GITHUB_WEB_VERSION:=N/A}";; + "web" ) echo "${GITHUB_WEB_VERSION:=N/A}";; "FTL" ) echo "${GITHUB_FTL_VERSION:=N/A}";; esac } @@ -56,7 +56,7 @@ getRemoteVersion(){ getLocalBranch(){ case ${1} in "Pi-hole" ) echo "${CORE_BRANCH:=N/A}";; - "web" ) echo "${WEB_BRANCH:=N/A}";; + "web" ) echo "${WEB_BRANCH:=N/A}";; "FTL" ) echo "${FTL_BRANCH:=N/A}";; esac } @@ -114,11 +114,11 @@ defaultOutput() { helpFunc() { echo "Usage: pihole -v [repo | option] [option] Example: 'pihole -v -p -l' -Show Pi-hole, Admin Console & FTL versions +Show Pi-hole, Web Console & FTL versions Repositories: -p, --pihole Only retrieve info regarding Pi-hole repository - -a, --admin Only retrieve info regarding web repository + -w, --web Only retrieve info regarding web repository -f, --ftl Only retrieve info regarding FTL repository Options: @@ -131,7 +131,7 @@ Options: case "${1}" in "-p" | "--pihole" ) shift; versionOutput "Pi-hole" "$@";; - "-a" | "--admin" ) shift; versionOutput "web" "$@";; + "-w" | "--web" ) shift; versionOutput "web" "$@";; "-f" | "--ftl" ) shift; versionOutput "FTL" "$@";; "-h" | "--help" ) helpFunc;; * ) defaultOutput "$@";; diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index b2740724..4fe8f83a 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -35,7 +35,7 @@ _pihole() { COMPREPLY=( $(compgen -W "${opts_update}" -- ${cur}) ) ;; "version") - opts_version="\--admin \--current \--ftl \--hash \--latest \--pihole" + opts_version="\--web \--current \--ftl \--hash \--latest \--pihole" COMPREPLY=( $(compgen -W "${opts_version}" -- ${cur}) ) ;; "core"|"admin"|"ftl") From c90a27c5096d53f4be21044fdd71cb380806bc3f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 9 Dec 2023 23:06:50 +0100 Subject: [PATCH 168/200] Add "-ni" to all sqlite3 invocations Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 34 ++++++++-------- advanced/Scripts/list.sh | 20 +++++----- advanced/Scripts/piholeARPTable.sh | 4 +- advanced/Scripts/piholeDebug.sh | 12 +++--- advanced/Scripts/piholeLogFlush.sh | 2 +- gravity.sh | 39 +++++++++---------- 6 files changed, 55 insertions(+), 56 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index e36d9b1e..7c4deaa7 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -19,13 +19,13 @@ upgrade_gravityDB(){ auditFile="${piholeDir}/auditlog.list" # Get database version - version="$(pihole-FTL sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" + version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" if [[ "$version" == "1" ]]; then # This migration script upgrades the gravity.db file by # adding the domain_audit table echo -e " ${INFO} Upgrading gravity database from version 1 to 2" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/1_to_2.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/1_to_2.sql" version=2 # Store audit domains in database table @@ -40,28 +40,28 @@ upgrade_gravityDB(){ # renaming the regex table to regex_blacklist, and # creating a new regex_whitelist table + corresponding linking table and views echo -e " ${INFO} Upgrading gravity database from version 2 to 3" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/2_to_3.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/2_to_3.sql" version=3 fi if [[ "$version" == "3" ]]; then # This migration script unifies the formally separated domain # lists into a single table with a UNIQUE domain constraint echo -e " ${INFO} Upgrading gravity database from version 3 to 4" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/3_to_4.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/3_to_4.sql" version=4 fi if [[ "$version" == "4" ]]; then # This migration script upgrades the gravity and list views # implementing necessary changes for per-client blocking echo -e " ${INFO} Upgrading gravity database from version 4 to 5" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/4_to_5.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/4_to_5.sql" version=5 fi if [[ "$version" == "5" ]]; then # This migration script upgrades the adlist view # to return an ID used in gravity.sh echo -e " ${INFO} Upgrading gravity database from version 5 to 6" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/5_to_6.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/5_to_6.sql" version=6 fi if [[ "$version" == "6" ]]; then @@ -69,7 +69,7 @@ upgrade_gravityDB(){ # which is automatically associated to all clients not # having their own group assignments echo -e " ${INFO} Upgrading gravity database from version 6 to 7" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/6_to_7.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/6_to_7.sql" version=7 fi if [[ "$version" == "7" ]]; then @@ -77,21 +77,21 @@ upgrade_gravityDB(){ # to ensure uniqueness on the group name # We also add date_added and date_modified columns echo -e " ${INFO} Upgrading gravity database from version 7 to 8" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/7_to_8.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/7_to_8.sql" version=8 fi if [[ "$version" == "8" ]]; then # This migration fixes some issues that were introduced # in the previous migration script. echo -e " ${INFO} Upgrading gravity database from version 8 to 9" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/8_to_9.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/8_to_9.sql" version=9 fi if [[ "$version" == "9" ]]; then # This migration drops unused tables and creates triggers to remove # obsolete groups assignments when the linked items are deleted echo -e " ${INFO} Upgrading gravity database from version 9 to 10" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/9_to_10.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/9_to_10.sql" version=10 fi if [[ "$version" == "10" ]]; then @@ -101,44 +101,44 @@ upgrade_gravityDB(){ # to keep the copying process generic (needs the same columns in both the # source and the destination databases). echo -e " ${INFO} Upgrading gravity database from version 10 to 11" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/10_to_11.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/10_to_11.sql" version=11 fi if [[ "$version" == "11" ]]; then # Rename group 0 from "Unassociated" to "Default" echo -e " ${INFO} Upgrading gravity database from version 11 to 12" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/11_to_12.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/11_to_12.sql" version=12 fi if [[ "$version" == "12" ]]; then # Add column date_updated to adlist table echo -e " ${INFO} Upgrading gravity database from version 12 to 13" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/12_to_13.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/12_to_13.sql" version=13 fi if [[ "$version" == "13" ]]; then # Add columns number and status to adlist table echo -e " ${INFO} Upgrading gravity database from version 13 to 14" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/13_to_14.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/13_to_14.sql" version=14 fi if [[ "$version" == "14" ]]; then # Changes the vw_adlist created in 5_to_6 echo -e " ${INFO} Upgrading gravity database from version 14 to 15" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/14_to_15.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/14_to_15.sql" version=15 fi if [[ "$version" == "15" ]]; then # Add column abp_entries to adlist table echo -e " ${INFO} Upgrading gravity database from version 15 to 16" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/15_to_16.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/15_to_16.sql" version=16 fi if [[ "$version" == "16" ]]; then # Add antigravity table # Add column type to adlist table (to support adlist types) echo -e " ${INFO} Upgrading gravity database from version 16 to 17" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/16_to_17.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/16_to_17.sql" version=17 fi } diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index b76a7ef7..76558e58 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -150,18 +150,18 @@ AddDomain() { domain="$1" # Is the domain in the list we want to add it to? - num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")" + num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")" requestedListname="$(GetListnameFromTypeId "${typeId}")" if [[ "${num}" -ne 0 ]]; then - existingTypeId="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")" + existingTypeId="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")" if [[ "${existingTypeId}" == "${typeId}" ]]; then if [[ "${verbose}" == true ]]; then echo -e " ${INFO} ${1} already exists in ${requestedListname}, no need to add!" fi else existingListname="$(GetListnameFromTypeId "${existingTypeId}")" - pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';" if [[ "${verbose}" == true ]]; then echo -e " ${INFO} ${1} already exists in ${existingListname}, it has been moved to ${requestedListname}!" fi @@ -177,10 +177,10 @@ AddDomain() { # Insert only the domain here. The enabled and date_added fields will be filled # with their default values (enabled = true, date_added = current timestamp) if [[ -z "${comment}" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});" else # also add comment when variable has been set through the "--comment" option - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');" fi } @@ -189,7 +189,7 @@ RemoveDomain() { domain="$1" # Is the domain in the list we want to remove it from? - num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")" + num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")" requestedListname="$(GetListnameFromTypeId "${typeId}")" @@ -206,14 +206,14 @@ RemoveDomain() { fi reload=true # Remove it from the current list - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};" } Displaylist() { local count num_pipes domain enabled status nicedate requestedListname requestedListname="$(GetListnameFromTypeId "${typeId}")" - data="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)" + data="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)" if [[ -z $data ]]; then echo -e "Not showing empty list" @@ -251,10 +251,10 @@ Displaylist() { } NukeList() { - count=$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};") + count=$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};") listname="$(GetListnameFromTypeId "${typeId}")" if [ "$count" -gt 0 ];then - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};" echo " ${TICK} Removed ${count} domain(s) from the ${listname}" else echo " ${INFO} ${listname} already empty. Nothing to do!" diff --git a/advanced/Scripts/piholeARPTable.sh b/advanced/Scripts/piholeARPTable.sh index 5daa025d..b92dd124 100755 --- a/advanced/Scripts/piholeARPTable.sh +++ b/advanced/Scripts/piholeARPTable.sh @@ -39,7 +39,7 @@ flushARP(){ # Truncate network_addresses table in pihole-FTL.db # This needs to be done before we can truncate the network table due to # foreign key constraints - if ! output=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then + if ! output=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then echo -e "${OVER} ${CROSS} Failed to truncate network_addresses table" echo " Database location: ${DBFILE}" echo " Output: ${output}" @@ -47,7 +47,7 @@ flushARP(){ fi # Truncate network table in pihole-FTL.db - if ! output=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM network" 2>&1); then + if ! output=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM network" 2>&1); then echo -e "${OVER} ${CROSS} Failed to truncate network table" echo " Database location: ${DBFILE}" echo " Output: ${output}" diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 931e95a4..aa0e61c4 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -718,7 +718,7 @@ dig_at() { # This helps emulate queries to different domains that a user might query # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains local random_url - random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") + random_url=$(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") # Fallback if no non-ABP style domains were found if [ -z "${random_url}" ]; then random_url="flurry.com" @@ -1064,7 +1064,7 @@ show_db_entries() { IFS=$'\r\n' local entries=() mapfile -t entries < <(\ - pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" \ + pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" \ -cmd ".headers on" \ -cmd ".mode column" \ -cmd ".width ${widths}" \ @@ -1089,7 +1089,7 @@ show_FTL_db_entries() { IFS=$'\r\n' local entries=() mapfile -t entries < <(\ - pihole-FTL sqlite3 "${PIHOLE_FTL_DB_FILE}" \ + pihole-FTL sqlite3 -ni "${PIHOLE_FTL_DB_FILE}" \ -cmd ".headers on" \ -cmd ".mode column" \ -cmd ".width ${widths}" \ @@ -1155,7 +1155,7 @@ analyze_gravity_list() { fi show_db_entries "Info table" "SELECT property,value FROM info" "20 40" - gravity_updated_raw="$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT value FROM info where property = 'updated'")" + gravity_updated_raw="$(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT value FROM info where property = 'updated'")" gravity_updated="$(date -d @"${gravity_updated_raw}")" log_write " Last gravity run finished at: ${COL_CYAN}${gravity_updated}${COL_NC}" log_write "" @@ -1163,7 +1163,7 @@ analyze_gravity_list() { OLD_IFS="$IFS" IFS=$'\r\n' local gravity_sample=() - mapfile -t gravity_sample < <(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10") + mapfile -t gravity_sample < <(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10") log_write " ${COL_CYAN}----- First 10 Gravity Domains -----${COL_NC}" for line in "${gravity_sample[@]}"; do @@ -1195,7 +1195,7 @@ database_integrity_check(){ log_write "${INFO} Checking foreign key constraints of ${database} ... (this can take several minutes)" unset result - result="$(pihole-FTL sqlite3 "${database}" -cmd ".headers on" -cmd ".mode column" "PRAGMA foreign_key_check" 2>&1 & spinner)" + result="$(pihole-FTL sqlite3 -ni "${database}" -cmd ".headers on" -cmd ".mode column" "PRAGMA foreign_key_check" 2>&1 & spinner)" if [[ -z ${result} ]]; then log_write "${TICK} No foreign key errors in ${database}" else diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 14542e4b..4d97fec5 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -63,7 +63,7 @@ else fi fi # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history) - deleted=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1") + deleted=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1") # Restart pihole-FTL to force reloading history sudo pihole restartdns diff --git a/gravity.sh b/gravity.sh index 8dc8edd8..f51103ff 100755 --- a/gravity.sh +++ b/gravity.sh @@ -48,7 +48,6 @@ gravityDBcopy="${piholeGitDir}/advanced/Templates/gravity_copy.sql" domainsExtension="domains" curl_connect_timeout=10 - # Check gravity temp directory if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then echo -e " ${COL_LIGHT_RED}Gravity temporary directory does not exist or is not a writeable directory, falling back to /tmp. ${COL_NC}" @@ -64,7 +63,7 @@ gravityOLDfile="${gravityDIR}/gravity_old.db" # Generate new SQLite3 file from schema template generate_gravity_database() { - if ! pihole-FTL sqlite3 "${gravityDBfile}" < "${gravityDBschema}"; then + if ! pihole-FTL sqlite3 -ni "${gravityDBfile}" < "${gravityDBschema}"; then echo -e " ${CROSS} Unable to create ${gravityDBfile}" return 1 fi @@ -79,7 +78,7 @@ gravity_build_tree() { echo -ne " ${INFO} ${str}..." # The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once - output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -118,7 +117,7 @@ gravity_swap_databases() { # Update timestamp when the gravity table was last updated successfully update_gravity_timestamp() { - output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -163,7 +162,7 @@ database_table_from_file() { # Get MAX(id) from domainlist when INSERTing into this table if [[ "${table}" == "domainlist" ]]; then - rowid="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")" + rowid="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")" if [[ -z "$rowid" ]]; then rowid=0 fi @@ -193,7 +192,7 @@ database_table_from_file() { # Store domains in database table specified by ${table} # Use printf as .mode and .import need to be on separate lines # see https://unix.stackexchange.com/a/445615/83260 - output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 -ni "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -213,7 +212,7 @@ database_table_from_file() { # Check if a column with name ${2} exists in gravity table with name ${1} gravity_column_exists() { - output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) if [[ "${output}" == "1" ]]; then return 0 # Bash 0 is success fi @@ -228,7 +227,7 @@ database_adlist_number() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${2}" "${3}" "${1}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${2}" "${3}" "${1}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -244,7 +243,7 @@ database_adlist_status() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -362,9 +361,9 @@ gravity_DownloadBlocklists() { # Retrieve source URLs from gravity database # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true) - mapfile -t sources <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" - mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)" - mapfile -t sourceTypes <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2> /dev/null)" + mapfile -t sources <<< "$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" + mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)" + mapfile -t sourceTypes <<< "$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2> /dev/null)" # Parse source domains from $sources mapfile -t sourceDomains <<< "$( @@ -393,7 +392,7 @@ gravity_DownloadBlocklists() { str="Preparing new gravity database" echo -ne " ${INFO} ${str}..." rm "${gravityTEMPfile}" > /dev/null 2>&1 - output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -413,7 +412,7 @@ gravity_DownloadBlocklists() { copyGravity="${copyGravity//"${gravityDBfile_default}"/"${gravityDBfile}"}" fi - output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -663,12 +662,12 @@ gravity_Table_Count() { local table="${1}" local str="${2}" local num - num="$(pihole-FTL sqlite3 "${gravityTEMPfile}" "SELECT COUNT(*) FROM ${table};")" + num="$(pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM ${table};")" if [[ "${table}" == "gravity" ]]; then local unique - unique="$(pihole-FTL sqlite3 "${gravityTEMPfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" + unique="$(pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" echo -e " ${INFO} Number of ${str}: ${num} (${COL_BOLD}${unique} unique domains${COL_NC})" - pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" + pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" else echo -e " ${INFO} Number of ${str}: ${num}" fi @@ -749,7 +748,7 @@ database_recovery() { local str="Checking integrity of existing gravity database (this can take a while)" local option="${1}" echo -ne " ${INFO} ${str}..." - result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)" + result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)" if [[ ${result} = "ok" ]]; then echo -e "${OVER} ${TICK} ${str} - no errors found" @@ -757,7 +756,7 @@ database_recovery() { str="Checking foreign keys of existing gravity database (this can take a while)" echo -ne " ${INFO} ${str}..." unset result - result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)" + result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)" if [[ -z ${result} ]]; then echo -e "${OVER} ${TICK} ${str} - no errors found" if [[ "${option}" != "force" ]]; then @@ -776,7 +775,7 @@ database_recovery() { echo -ne " ${INFO} ${str}..." # We have to remove any possibly existing recovery database or this will fail rm -f "${gravityDBfile}.recovered" > /dev/null 2>&1 - if result="$(pihole-FTL sqlite3 "${gravityDBfile}" ".recover" | pihole-FTL sqlite3 "${gravityDBfile}.recovered" 2>&1)"; then + if result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" ".recover" | pihole-FTL sqlite3 -ni "${gravityDBfile}.recovered" 2>&1)"; then echo -e "${OVER} ${TICK} ${str} - success" mv "${gravityDBfile}" "${gravityDBfile}.old" mv "${gravityDBfile}.recovered" "${gravityDBfile}" From fe4d934a406cf4fa45a72668170868bd57448537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 22:31:36 +0100 Subject: [PATCH 169/200] Simplify pihole -v MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/version.sh | 140 ++++++++---------------------------- pihole | 5 +- 2 files changed, 32 insertions(+), 113 deletions(-) diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index 2983c04e..ff7f0815 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -8,6 +8,10 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. +# Ignore warning about `local` being undefinded in POSIX +# shellcheck disable=SC3043 +# https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions + # Source the versions file poupulated by updatechecker.sh cachedVersions="/etc/pihole/versions" @@ -21,118 +25,34 @@ else . "$cachedVersions" fi -getLocalVersion() { - case ${1} in - "Pi-hole" ) echo "${CORE_VERSION:=N/A}";; - "web" ) echo "${WEB_VERSION:=N/A}";; - "FTL" ) echo "${FTL_VERSION:=N/A}";; - esac -} - -getLocalHash() { - case ${1} in - "Pi-hole" ) echo "${CORE_HASH:=N/A}";; - "web" ) echo "${WEB_HASH:=N/A}";; - "FTL" ) echo "${FTL_HASH:=N/A}";; - esac -} - -getRemoteHash(){ - case ${1} in - "Pi-hole" ) echo "${GITHUB_CORE_HASH:=N/A}";; - "web" ) echo "${GITHUB_WEB_HASH:=N/A}";; - "FTL" ) echo "${GITHUB_FTL_HASH:=N/A}";; - esac -} - -getRemoteVersion(){ - case ${1} in - "Pi-hole" ) echo "${GITHUB_CORE_VERSION:=N/A}";; - "web" ) echo "${GITHUB_WEB_VERSION:=N/A}";; - "FTL" ) echo "${GITHUB_FTL_VERSION:=N/A}";; - esac -} - -getLocalBranch(){ - case ${1} in - "Pi-hole" ) echo "${CORE_BRANCH:=N/A}";; - "web" ) echo "${WEB_BRANCH:=N/A}";; - "FTL" ) echo "${FTL_BRANCH:=N/A}";; - esac -} - -versionOutput() { - - [ "$2" = "-c" ] || [ "$2" = "--current" ] || [ -z "$2" ] && current=$(getLocalVersion "${1}") && branch=$(getLocalBranch "${1}") - [ "$2" = "-l" ] || [ "$2" = "--latest" ] || [ -z "$2" ] && latest=$(getRemoteVersion "${1}") - if [ "$2" = "--hash" ]; then - [ "$3" = "-c" ] || [ "$3" = "--current" ] || [ -z "$3" ] && curHash=$(getLocalHash "${1}") && branch=$(getLocalBranch "${1}") - [ "$3" = "-l" ] || [ "$3" = "--latest" ] || [ -z "$3" ] && latHash=$(getRemoteHash "${1}") && branch=$(getLocalBranch "${1}") - fi - - # We do not want to show the branch name when we are on master, - # blank out the variable in this case - if [ "$branch" = "master" ]; then - branch="" +main() { + local details + details=$1 + + if [ "${details}" = true ]; then + echo "Core" + echo " Version is ${CORE_VERSION:=N/A} (Latest: ${GITHUB_CORE_VERSION:=N/A})" + echo " Branch is ${CORE_BRANCH:=N/A}" + echo " Hash is ${CORE_HASH:=N/A} (Latest: ${GITHUB_CORE_HASH:=N/A})" + echo "Web" + echo " Version is ${WEB_VERSION:=N/A} (Latest: ${GITHUB_WEB_VERSION:=N/A})" + echo " Branch is ${WEB_BRANCH:=N/A}" + echo " Hash is ${WEB_HASH:=N/A} (Latest: ${GITHUB_WEB_HASH:=N/A})" + echo "FTL" + echo " Version is ${FTL_VERSION:=N/A} (Latest: ${GITHUB_FTL_VERSION:=N/A})" + echo " Branch is ${FTL_BRANCH:=N/A}" + echo " Hash is ${FTL_HASH:=N/A} (Latest: ${GITHUB_FTL_HASH:=N/A})" else - branch="$branch " + echo "Core version is ${CORE_VERSION:=N/A} (Latest: ${GITHUB_CORE_VERSION:=N/A})" + echo "Web version is ${WEB_VERSION:=N/A} (Latest: ${GITHUB_WEB_VERSION:=N/A})" + echo "FTL version is ${FTL_VERSION:=N/A} (Latest: ${GITHUB_FTL_VERSION:=N/A})" fi - - if [ -n "$current" ] && [ -n "$latest" ]; then - output="${1} version is $branch$current (Latest: $latest)" - elif [ -n "$current" ] && [ -z "$latest" ]; then - output="Current ${1} version is $branch$current" - elif [ -z "$current" ] && [ -n "$latest" ]; then - output="Latest ${1} version is $latest" - elif [ -n "$curHash" ] && [ -n "$latHash" ]; then - output="Local ${1} hash is $curHash (Remote: $latHash)" - elif [ -n "$curHash" ] && [ -z "$latHash" ]; then - output="Current local ${1} hash is $curHash" - elif [ -z "$curHash" ] && [ -n "$latHash" ]; then - output="Latest remote ${1} hash is $latHash" - elif [ -z "$curHash" ] && [ -z "$latHash" ]; then - output="Hashes for ${1} not available" - else - errorOutput - return 1 - fi - - [ -n "$output" ] && echo " $output" } -errorOutput() { - echo " Invalid Option! Try 'pihole -v --help' for more information." - exit 1 -} - -defaultOutput() { - versionOutput "Pi-hole" "$@" - versionOutput "web" "$@" - versionOutput "FTL" "$@" -} - -helpFunc() { - echo "Usage: pihole -v [repo | option] [option] -Example: 'pihole -v -p -l' -Show Pi-hole, Web Console & FTL versions - -Repositories: - -p, --pihole Only retrieve info regarding Pi-hole repository - -w, --web Only retrieve info regarding web repository - -f, --ftl Only retrieve info regarding FTL repository - -Options: - -c, --current Return the current version - -l, --latest Return the latest version - --hash Return the GitHub hash from your local repositories - -h, --help Show this help dialog" - exit 0 -} - -case "${1}" in - "-p" | "--pihole" ) shift; versionOutput "Pi-hole" "$@";; - "-w" | "--web" ) shift; versionOutput "web" "$@";; - "-f" | "--ftl" ) shift; versionOutput "FTL" "$@";; - "-h" | "--help" ) helpFunc;; - * ) defaultOutput "$@";; +# Process options (if present) +case "$1" in + "-vv" ) details=true;; + * ) details=false;; esac + +main "${details}" diff --git a/pihole b/pihole index 47da4ddd..ef93146c 100755 --- a/pihole +++ b/pihole @@ -140,7 +140,6 @@ uninstallFunc() { } versionFunc() { - shift exec "${PI_HOLE_SCRIPT_DIR}"/version.sh "$@" } @@ -508,7 +507,7 @@ Options: -up, updatePihole Update Pi-hole subsystems Add '--check-only' to exit script before update is performed. -v, version Show installed versions of Pi-hole, Web Interface & FTL - Add '-h' for more info on version usage + Use -vv for detailed information. uninstall Uninstall Pi-hole from your system status Display the running status of Pi-hole subsystems enable Enable Pi-hole subsystems @@ -531,7 +530,7 @@ fi need_root=1 case "${1}" in "-h" | "help" | "--help" ) helpFunc;; - "-v" | "version" ) versionFunc "$@";; + "-v" | "-vv" | "version" ) versionFunc "$@";; "-c" | "chronometer" ) chronometerFunc "$@";; "-q" | "query" ) queryFunc "$@";; "status" ) statusFunc "$2";; From b333e30162a07287bcf313ec9da01e1644c72d52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 22:55:01 +0100 Subject: [PATCH 170/200] Remove version options from bash completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/bash-completion/pihole | 4 ---- 1 file changed, 4 deletions(-) diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 4fe8f83a..89e02d2f 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -34,10 +34,6 @@ _pihole() { opts_update="--check-only" COMPREPLY=( $(compgen -W "${opts_update}" -- ${cur}) ) ;; - "version") - opts_version="\--web \--current \--ftl \--hash \--latest \--pihole" - COMPREPLY=( $(compgen -W "${opts_version}" -- ${cur}) ) - ;; "core"|"admin"|"ftl") if [[ "$prev2" == "checkout" ]]; then opts_checkout="master dev" From c3c31a1a60dcf68ae3b08a2c474485fdbd3c9cb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 23:09:58 +0100 Subject: [PATCH 171/200] Print version details automatically if not on master MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/version.sh | 16 ++++++++-------- pihole | 5 ++--- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index ff7f0815..2422ea6d 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -27,7 +27,13 @@ fi main() { local details - details=$1 + details=false + + # Automatically show detailed information if + # at least one of the components is not on master branch + if [ ! "${CORE_BRANCH}" = "master" ] || [ ! "${WEB_BRANCH}" = "master" ] || [ ! "${FTL_BRANCH}" = "master" ] ; then + details=true + fi if [ "${details}" = true ]; then echo "Core" @@ -49,10 +55,4 @@ main() { fi } -# Process options (if present) -case "$1" in - "-vv" ) details=true;; - * ) details=false;; -esac - -main "${details}" +main diff --git a/pihole b/pihole index ef93146c..7c84771c 100755 --- a/pihole +++ b/pihole @@ -140,7 +140,7 @@ uninstallFunc() { } versionFunc() { - exec "${PI_HOLE_SCRIPT_DIR}"/version.sh "$@" + exec "${PI_HOLE_SCRIPT_DIR}"/version.sh } restartDNS() { @@ -507,7 +507,6 @@ Options: -up, updatePihole Update Pi-hole subsystems Add '--check-only' to exit script before update is performed. -v, version Show installed versions of Pi-hole, Web Interface & FTL - Use -vv for detailed information. uninstall Uninstall Pi-hole from your system status Display the running status of Pi-hole subsystems enable Enable Pi-hole subsystems @@ -530,7 +529,7 @@ fi need_root=1 case "${1}" in "-h" | "help" | "--help" ) helpFunc;; - "-v" | "-vv" | "version" ) versionFunc "$@";; + "-v" | "version" ) versionFunc;; "-c" | "chronometer" ) chronometerFunc "$@";; "-q" | "query" ) queryFunc "$@";; "status" ) statusFunc "$2";; From 0cfcdc4b50e16612bf7572f290c1960c0f39e873 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 16 Dec 2023 10:08:32 +0000 Subject: [PATCH 172/200] Bump github/codeql-action from 2 to 3 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2 to 3. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v2...v3) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6544db61..fc821194 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -29,12 +29,12 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: 'python' - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 From 0066c6fbffa889525751478468b1cfba77272e20 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Dec 2023 05:29:11 +0100 Subject: [PATCH 173/200] Make IDs of anti-/gravity lists available in vw_(anti)gravity Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 6 +++++ .../database_migration/gravity/17_to_18.sql | 25 +++++++++++++++++++ advanced/Templates/gravity.db.sql | 6 ++--- gravity.sh | 2 +- 4 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 advanced/Scripts/database_migration/gravity/17_to_18.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 7c4deaa7..b263b40d 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -141,4 +141,10 @@ upgrade_gravityDB(){ pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/16_to_17.sql" version=17 fi + if [[ "$version" == "17" ]]; then + # Add adlist.id to vw_gravity and vw_antigravity + echo -e " ${INFO} Upgrading gravity database from version 17 to 18" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/17_to_18.sql" + version=18 + fi } diff --git a/advanced/Scripts/database_migration/gravity/17_to_18.sql b/advanced/Scripts/database_migration/gravity/17_to_18.sql new file mode 100644 index 00000000..00171a9a --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/17_to_18.sql @@ -0,0 +1,25 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +DROP VIEW vw_gravity; +CREATE VIEW vw_gravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id + FROM gravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = gravity.adlist_id + LEFT JOIN adlist ON adlist.id = gravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1); + +DROP VIEW vw_antigravity; +CREATE VIEW vw_antigravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id + FROM antigravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id + LEFT JOIN adlist ON adlist.id = antigravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; + +UPDATE info SET value = 18 WHERE property = 'version'; + +COMMIT; diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 46f26ba7..097b0a78 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -65,7 +65,7 @@ CREATE TABLE info value TEXT NOT NULL ); -INSERT INTO "info" VALUES('version','17'); +INSERT INTO "info" VALUES('version','18'); CREATE TABLE domain_audit ( @@ -144,14 +144,14 @@ CREATE VIEW vw_regex_blacklist AS SELECT domain, domainlist.id AS id, domainlist AND domainlist.type = 3 ORDER BY domainlist.id; -CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id +CREATE VIEW vw_gravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id FROM gravity LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = gravity.adlist_id LEFT JOIN adlist ON adlist.id = gravity.adlist_id LEFT JOIN "group" ON "group".id = adlist_by_group.group_id WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1); -CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id +CREATE VIEW vw_antigravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id FROM antigravity LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id LEFT JOIN adlist ON adlist.id = antigravity.adlist_id diff --git a/gravity.sh b/gravity.sh index f51103ff..ebe7e740 100755 --- a/gravity.sh +++ b/gravity.sh @@ -78,7 +78,7 @@ gravity_build_tree() { echo -ne " ${INFO} ${str}..." # The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once - output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) + output=$( { time pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } ) status="$?" if [[ "${status}" -ne 0 ]]; then From 8e8c7ecad212d45548dc89f408d2545d332a37a5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Dec 2023 05:41:43 +0100 Subject: [PATCH 174/200] Run gravity upgrade on checkout/update Signed-off-by: DL6ER --- automated install/basic-install.sh | 5 +++++ gravity.sh | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f5bf15e6..5863e818 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2225,6 +2225,11 @@ main() { # but before starting or restarting the ftl service disable_resolved_stublistener + # Check if gravity database needs to be upgraded. If so, do it without rebuilding + # gravity altogether. This may be a very long running task needlessly blocking + # the update process. + /opt/pihole/gravity.sh --upgrade + printf " %b Restarting services...\\n" "${INFO}" # Start services diff --git a/gravity.sh b/gravity.sh index ebe7e740..216a67a5 100755 --- a/gravity.sh +++ b/gravity.sh @@ -78,7 +78,7 @@ gravity_build_tree() { echo -ne " ${INFO} ${str}..." # The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once - output=$( { time pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -831,6 +831,7 @@ for var in "$@"; do case "${var}" in "-f" | "--force" ) forceDelete=true;; "-r" | "--repair" ) repairSelector "$3";; + "-u" | "--upgrade" ) upgrade_gravityDB "${gravityDBfile}" "${piholeDir}"; exit 0;; "-h" | "--help" ) helpFunc;; esac done From 1b87ec067ef8d19df0cc6af9f4aa30ba69696d3c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Dec 2023 06:12:18 +0100 Subject: [PATCH 175/200] Exit early if the database does not exist (e.g. in some CI tests) Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity-db.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index b263b40d..7b356b3a 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -18,6 +18,11 @@ upgrade_gravityDB(){ piholeDir="${2}" auditFile="${piholeDir}/auditlog.list" + # Exit early if the database does not exist (e.g. in CI tests) + if [[ ! -f "${database}" ]]; then + return + fi + # Get database version version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" From 0c6363572b2ddde8c44f3bd5d06dfff92f929380 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Dec 2023 22:00:49 +0100 Subject: [PATCH 176/200] Append the interface to the gateway address if it is a link-local address Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index aa0e61c4..a78f94ee 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -547,17 +547,24 @@ ping_gateway() { ping_ipv4_or_ipv6 "${protocol}" # Check if we are using IPv4 or IPv6 # Find the default gateways using IPv4 or IPv6 - local gateway + local gateway gateway_addr gateway_iface log_write "${INFO} Default IPv${protocol} gateway(s):" while IFS= read -r gateway; do - log_write " ${gateway}" - done < <(ip -"${protocol}" route | grep default | cut -d ' ' -f 3) + log_write " $(cut -d ' ' -f 3 <<< "${gateway}")%$(cut -d ' ' -f 5 <<< "${gateway}")" + done < <(ip -"${protocol}" route | grep default) - gateway=$(ip -"${protocol}" route | grep default | cut -d ' ' -f 3 | head -n 1) + gateway_addr=$(ip -"${protocol}" route | grep default | cut -d ' ' -f 3 | head -n 1) + gateway_iface=$(ip -"${protocol}" route | grep default | cut -d ' ' -f 5 | head -n 1) # If there was at least one gateway - if [ -n "${gateway}" ]; then + if [ -n "${gateway_addr}" ]; then + # Append the interface to the gateway address if it is a link-local address + if [[ "${gateway_addr}" =~ ^fe80 ]]; then + gateway="${gateway_addr}%${gateway_iface}" + else + gateway="${gateway_addr}" + fi # Let the user know we will ping the gateway for a response log_write " * Pinging first gateway ${gateway}..." # Try to quietly ping the gateway 3 times, with a timeout of 3 seconds, using numeric output only, From 00340136bdc517cc71d4458fb5ccd8d784724767 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 26 Dec 2023 21:35:24 +0100 Subject: [PATCH 177/200] Extend %iface logic to the dig test. Also fix intentation in this function Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 39 +++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index a78f94ee..2c3ebb14 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -764,24 +764,29 @@ dig_at() { # Removes CIDR and everything thereafter (e.g., scope properties) addresses="$(ip address show dev "${iface}" | sed "/${sed_selector} /!d;s/^.*${sed_selector} //g;s/\/.*$//g;")" if [ -n "${addresses}" ]; then - while IFS= read -r local_address ; do + while IFS= read -r local_address ; do + # If ${local_address} is an IPv6 link-local address, append the interface name to it + if [[ "${local_address}" =~ ^fe80 ]]; then + local_address="${local_address}%${iface}" + fi + # Check if Pi-hole can use itself to block a domain - if local_dig="$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${local_address}" "${record_type}")"; then - # If it can, show success - if [[ "${local_dig}" == *"status: NOERROR"* ]]; then - local_dig="NOERROR" - elif [[ "${local_dig}" == *"status: NXDOMAIN"* ]]; then - local_dig="NXDOMAIN" - else - # Extract the first entry in the answer section from dig's output, - # replacing any multiple spaces and tabs with a single space - local_dig="$(echo "${local_dig}" | grep -A1 "ANSWER SECTION" | grep -v "ANSWER SECTION" | tr -s " \t" " ")" - fi - log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} on ${COL_CYAN}${iface}${COL_NC} (${COL_CYAN}${local_address}${COL_NC})" - else - # Otherwise, show a failure - log_write "${CROSS} ${COL_RED}Failed to resolve${COL_NC} ${random_url} on ${COL_RED}${iface}${COL_NC} (${COL_RED}${local_address}${COL_NC})" - fi + if local_dig="$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${local_address}" "${record_type}")"; then + # If it can, show success + if [[ "${local_dig}" == *"status: NOERROR"* ]]; then + local_dig="NOERROR" + elif [[ "${local_dig}" == *"status: NXDOMAIN"* ]]; then + local_dig="NXDOMAIN" + else + # Extract the first entry in the answer section from dig's output, + # replacing any multiple spaces and tabs with a single space + local_dig="$(echo "${local_dig}" | grep -A1 "ANSWER SECTION" | grep -v "ANSWER SECTION" | tr -s " \t" " ")" + fi + log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} on ${COL_CYAN}${iface}${COL_NC} (${COL_CYAN}${local_address}${COL_NC})" + else + # Otherwise, show a failure + log_write "${CROSS} ${COL_RED}Failed to resolve${COL_NC} ${random_url} on ${COL_RED}${iface}${COL_NC} (${COL_RED}${local_address}${COL_NC})" + fi done <<< "${addresses}" else log_write "${TICK} No IPv${protocol} address available on ${COL_CYAN}${iface}${COL_NC}" From d2828310f240aed098e5b3d787f5d63f47575a7a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 28 Dec 2023 11:12:20 +0100 Subject: [PATCH 178/200] Modify DELETE triggers to delete BEFORE instead of AFTER to prevent possible foreign key constraint violations Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 7 +++++ .../database_migration/gravity/18_to_19.sql | 27 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 advanced/Scripts/database_migration/gravity/18_to_19.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 7b356b3a..e99f1df2 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -152,4 +152,11 @@ upgrade_gravityDB(){ pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/17_to_18.sql" version=18 fi + if [[ "$version" == "18" ]]; then + # Modify DELETE triggers to delete BEFORE instead of AFTER to prevent + # foreign key constraint violations + echo -e " ${INFO} Upgrading gravity database from version 18 to 19" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/18_to_19.sql" + version=19 + fi } diff --git a/advanced/Scripts/database_migration/gravity/18_to_19.sql b/advanced/Scripts/database_migration/gravity/18_to_19.sql new file mode 100644 index 00000000..c85a4d57 --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/18_to_19.sql @@ -0,0 +1,27 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +DROP TRIGGER tr_domainlist_delete; +CREATE TRIGGER tr_domainlist_delete BEFORE DELETE ON domainlist + BEGIN + DELETE FROM domainlist_by_group WHERE domainlist_id = OLD.id; + END; + +DROP TRIGGER tr_adlist_delete; +CREATE TRIGGER tr_adlist_delete BEFORE DELETE ON adlist + BEGIN + DELETE FROM adlist_by_group WHERE adlist_id = OLD.id; + END; + +DROP TRIGGER tr_client_delete; +CREATE TRIGGER tr_client_delete BEFORE DELETE ON client + BEGIN + DELETE FROM client_by_group WHERE client_id = OLD.id; + END; + +UPDATE info SET value = 19 WHERE property = 'version'; + +COMMIT; From 9c3578856f1c47887d7af5f1bc972ce44f5316ec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Jan 2024 10:02:39 +0000 Subject: [PATCH 179/200] Bump pytest from 7.4.3 to 7.4.4 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.3 to 7.4.4. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.3...7.4.4) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index bfc6d027..0873d097 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.3 +pytest == 7.4.4 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.11.4 From 220c0675ef67adea60161fdd38f409736bcaaac0 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 7 Jan 2024 21:03:48 +0000 Subject: [PATCH 180/200] As we do in other tests, specify `development-v6` branch of FTL is downloaded for the tests. We need to change this to `develoment` once v6 is released Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index c53070dc..ab301a6e 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -239,6 +239,7 @@ def test_FTL_detect_aarch64_no_errors(host): mock_command("uname", {"-m": ("aarch64", "0")}, host) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -272,6 +273,7 @@ def test_FTL_detect_armv6_no_errors(host): ) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -305,6 +307,7 @@ def test_FTL_detect_armv7l_no_errors(host): ) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -338,6 +341,7 @@ def test_FTL_detect_armv7_no_errors(host): ) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -371,6 +375,7 @@ def test_FTL_detect_armv8a_no_errors(host): ) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -393,6 +398,7 @@ def test_FTL_detect_x86_64_no_errors(host): """ detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -415,6 +421,7 @@ def test_FTL_detect_unknown_no_errors(host): mock_command("uname", {"-m": ("mips", "0")}, host) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -442,6 +449,7 @@ def test_FTL_download_aarch64_no_errors(host): ) download_binary = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user FTLinstall "pihole-FTL-aarch64-linux-gnu" @@ -458,6 +466,7 @@ def test_FTL_development_binary_installed_and_responsive_no_errors(host): """ host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) From df9c60e3511f7871c1392f19bebbef1112be64ac Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 8 Jan 2024 18:26:59 +0000 Subject: [PATCH 181/200] Reduce code duplication in FTL arch detect tests. Use parametrize to run the same test with different parameters for each arch we need to test. Also include detection of unusupported in this test. Create FTL_BRANCH constant to be used in functions (less places to remember to change it) Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 253 +++++------------------------ 1 file changed, 37 insertions(+), 216 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index ab301a6e..0930f0af 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -12,6 +12,8 @@ from .conftest import ( run_script, ) +FTL_BRANCH = "development-v6" + def test_supported_package_manager(host): """ @@ -80,11 +82,7 @@ def test_installPihole_fresh_install_readableFiles(host): host.run("command -v dnf > /dev/null && dnf install -y man") host.run("command -v yum > /dev/null && yum install -y man") # Workaround to get FTLv6 installed until it reaches master branch - host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - """ - ) + host.run('echo "' + FTL_BRANCH + '" > /etc/pihole/ftlbranch') install = host.run( """ export TERM=xterm @@ -231,151 +229,37 @@ def test_update_package_cache_failure_no_errors(host): assert "Error: Unable to update package cache." in updateCache.stdout -def test_FTL_detect_aarch64_no_errors(host): - """ - confirms only aarch64 package is downloaded for FTL engine - """ - # mock uname to return aarch64 platform - mock_command("uname", {"-m": ("aarch64", "0")}, host) - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected AArch64 (64 Bit ARM) architecture" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv6_no_errors(host): - """ - confirms only armv6 package is downloaded for FTL engine - """ - # mock uname to return armv6 platform - mock_command("uname", {"-m": ("armv6", "0")}, host) - # mock readelf to respond with armv6l CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv6", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv6", "0"), - }, - host, - ) - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv6 architecture" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv7l_no_errors(host): - """ - confirms only armv7l package is downloaded for FTL engine - """ - # mock uname to return armv7l platform - mock_command("uname", {"-m": ("armv7l", "0")}, host) - # mock readelf to respond with armv7l CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv7l", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv7l", "0"), - }, - host, - ) - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv7 (or newer) architecture") - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv7_no_errors(host): - """ - confirms only armv7 package is downloaded for FTL engine - """ - # mock uname to return armv7 platform - mock_command("uname", {"-m": ("armv7", "0")}, host) - # mock readelf to respond with armv7 CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv7", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv7", "0"), - }, - host, - ) - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv7 (or newer) architecture") - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv8a_no_errors(host): +@pytest.mark.parametrize( + "arch,detected_string,supported", + [ + ("aarch64", "AArch64 (64 Bit ARM)", True), + ("armv6", "ARMv6", True), + ("armv7l", "ARMv7 (or newer)", True), + ("armv7", "ARMv7 (or newer)", True), + ("armv8a", "ARMv7 (or newer)", True), + ("x86_64", "x86_64", True), + ("riscv64", "riscv64", True), + ("mips", "mips", False), + ], +) +def test_FTL_detect_no_errors(host, arch, detected_string, supported): """ - confirms only armv8a package is downloaded for FTL engine + confirms only correct package is downloaded for FTL engine """ - # mock uname to return armv8a platform - mock_command("uname", {"-m": ("armv8a", "0")}, host) - # mock readelf to respond with armv8a CPU architecture + # mock uname to return passed platform + mock_command("uname", {"-m": (arch, "0")}, host) + # mock readelf to respond with passed CPU architecture mock_command_2( "readelf", { - "-A /bin/sh": ("Tag_CPU_arch: armv8a", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv8a", "0"), + "-A /bin/sh": ("Tag_CPU_arch: " + arch, "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: " + arch, "0"), }, host, ) + host.run('echo "' + FTL_BRANCH + '" > /etc/pihole/ftlbranch') detectPlatform = host.run( """ - echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -384,93 +268,30 @@ def test_FTL_detect_armv8a_no_errors(host): FTLdetect "${binary}" "${theRest}" """ ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv7 (or newer) architecture (armv8a)" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_x86_64_no_errors(host): - """ - confirms only x86_64 package is downloaded for FTL engine - """ - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected x86_64 architecture" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_unknown_no_errors(host): - """confirms only generic package is downloaded for FTL engine""" - # mock uname to return generic platform - mock_command("uname", {"-m": ("mips", "0")}, host) - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = "Not able to detect architecture (unknown: mips)" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_download_aarch64_no_errors(host): - """ - confirms only aarch64 package is downloaded for FTL engine - """ - # mock dialog answers and ensure installer dependencies - mock_command("dialog", {"*": ("", "0")}, host) - host.run( - """ - source /opt/pihole/basic-install.sh - package_manager_detect - install_dependent_packages ${INSTALLER_DEPS[@]} - """ - ) - download_binary = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - FTLinstall "pihole-FTL-aarch64-linux-gnu" - """ - ) - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in download_binary.stdout - assert "error" not in download_binary.stdout.lower() + if supported: + expected_stdout = info_box + " FTL Checks..." + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Detected " + detected_string + " architecture" + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Downloading and Installing FTL" + assert expected_stdout in detectPlatform.stdout + else: + expected_stdout = ( + "Not able to detect architecture (unknown: " + detected_string + ")" + ) + assert expected_stdout in detectPlatform.stdout def test_FTL_development_binary_installed_and_responsive_no_errors(host): """ confirms FTL development binary is copied and functional in installed location """ + host.run('echo "' + FTL_BRANCH + '" > /etc/pihole/ftlbranch') host.run( """ - echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) - echo "development-v6" > /etc/pihole/ftlbranch binary="pihole-FTL${funcOutput##*pihole-FTL}" theRest="${funcOutput%pihole-FTL*}" FTLdetect "${binary}" "${theRest}" From debab10792c7bee8cd6ba18b6c3798c4f03c8a65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 7 Jan 2024 21:37:58 +0100 Subject: [PATCH 182/200] Use 204 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/api.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index fe73a608..18a48ce7 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -113,9 +113,8 @@ DeleteSession() { deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in - "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; + "204") printf "%b" "Session successfully deleted.\n";; "401") printf "%b" "Logout attempt without a valid session. Unauthorized!\n";; - "410") printf "%b" "Session successfully deleted.\n";; esac; fi From b662fd6f0101193a37b79377b077ffa37c33dfc2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 13 Jan 2024 10:43:13 +0000 Subject: [PATCH 183/200] Bump tox from 4.11.4 to 4.12.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.11.4 to 4.12.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.11.4...4.12.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 0873d097..59fd13cc 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.4 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.11.4 +tox == 4.12.0 From 47f06dfd71824f199af2a9b757f6ac06f9724ef4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 15 Jan 2024 20:53:34 +0100 Subject: [PATCH 184/200] Do not store remote version in versions file if on custom branch. It's always wrong here Signed-off-by: DL6ER --- advanced/Scripts/updatecheck.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 66f1a7ab..d2a338a7 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -26,7 +26,12 @@ function get_local_hash() { } function get_remote_version() { - curl -s "https://api.github.com/repos/pi-hole/${1}/releases/latest" 2> /dev/null | jq --raw-output .tag_name || return 1 + # if ${2} is = "master" we need to use the "latest" endpoint, otherwise, we simply return null + if [[ "${2}" == "master" ]]; then + curl -s "https://api.github.com/repos/pi-hole/${1}/releases/latest" 2> /dev/null | jq --raw-output .tag_name || return 1 + else + echo "null" + fi } @@ -73,7 +78,7 @@ addOrEditKeyValPair "${VERSION_FILE}" "CORE_BRANCH" "${CORE_BRANCH}" CORE_HASH="$(get_local_hash /etc/.pihole)" addOrEditKeyValPair "${VERSION_FILE}" "CORE_HASH" "${CORE_HASH}" -GITHUB_CORE_VERSION="$(get_remote_version pi-hole)" +GITHUB_CORE_VERSION="$(get_remote_version pi-hole "${CORE_BRANCH}")" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_CORE_VERSION" "${GITHUB_CORE_VERSION}" GITHUB_CORE_HASH="$(get_remote_hash pi-hole "${CORE_BRANCH}")" @@ -91,7 +96,7 @@ addOrEditKeyValPair "${VERSION_FILE}" "WEB_BRANCH" "${WEB_BRANCH}" WEB_HASH="$(get_local_hash /var/www/html/admin)" addOrEditKeyValPair "${VERSION_FILE}" "WEB_HASH" "${WEB_HASH}" -GITHUB_WEB_VERSION="$(get_remote_version web)" +GITHUB_WEB_VERSION="$(get_remote_version web "${WEB_BRANCH}")" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_VERSION" "${GITHUB_WEB_VERSION}" GITHUB_WEB_HASH="$(get_remote_hash web "${WEB_BRANCH}")" @@ -108,7 +113,7 @@ addOrEditKeyValPair "${VERSION_FILE}" "FTL_BRANCH" "${FTL_BRANCH}" FTL_HASH="$(pihole-FTL --hash)" addOrEditKeyValPair "${VERSION_FILE}" "FTL_HASH" "${FTL_HASH}" -GITHUB_FTL_VERSION="$(get_remote_version FTL)" +GITHUB_FTL_VERSION="$(get_remote_version FTL "${FTL_BRANCH}")" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_FTL_VERSION" "${GITHUB_FTL_VERSION}" GITHUB_FTL_HASH="$(get_remote_hash FTL "${FTL_BRANCH}")" From 60de50bb73f3df5f78c68726c3cd31ba1d5193ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Jan 2024 10:49:27 +0000 Subject: [PATCH 185/200] Bump tox from 4.12.0 to 4.12.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.12.0 to 4.12.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.12.0...4.12.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 59fd13cc..dea4ace9 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.4 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.12.0 +tox == 4.12.1 From 935a4ce0b3262fa3b3d468f018771a5001ffe145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 28 Jan 2024 16:56:57 +0100 Subject: [PATCH 186/200] Also remove Fedora 37 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 3 +-- test/_fedora_37.Dockerfile | 18 ------------------ test/tox.fedora_37.ini | 8 -------- 3 files changed, 1 insertion(+), 28 deletions(-) delete mode 100644 test/_fedora_37.Dockerfile delete mode 100644 test/tox.fedora_37.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9d90b2e7..eadfc930 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,8 +64,7 @@ jobs: ubuntu_23, centos_8, centos_9, - fedora_37, - fedora_38, + fedora_38, fedora_39, ] env: diff --git a/test/_fedora_37.Dockerfile b/test/_fedora_37.Dockerfile deleted file mode 100644 index b4f939ba..00000000 --- a/test/_fedora_37.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM fedora:37 -RUN dnf install -y git initscripts - -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole - -RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole -ADD . $GITDIR -RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR - -RUN true && \ - chmod +x $SCRIPTDIR/* - -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net - -#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/tox.fedora_37.ini b/test/tox.fedora_37.ini deleted file mode 100644 index 9c8752cc..00000000 --- a/test/tox.fedora_37.ini +++ /dev/null @@ -1,8 +0,0 @@ -[tox] -envlist = py3 - -[testenv] -allowlist_externals = docker -deps = -rrequirements.txt -commands = docker buildx build --load --progress plain -f _fedora_37.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py From 31a8f150b2e3a5411871f1163c7cc7a433e7ecc5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 30 Jan 2024 19:09:14 +0100 Subject: [PATCH 187/200] Add checking for availability of ftl.pi-hole.net when using FTL from a custom branch. If the server is down (or the user is offline, cannot resolve the domain, etc.), we fail early and hard instead of possibly corrupting the installation Signed-off-by: DL6ER --- advanced/Scripts/piholeCheckout.sh | 22 +++++++++++----- advanced/Scripts/update.sh | 7 ++++- automated install/basic-install.sh | 41 +++++++++++++++++++++--------- 3 files changed, 51 insertions(+), 19 deletions(-) diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index 39d39b1c..41fd8606 100755 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -164,7 +164,9 @@ checkout() { path="${2}/${binary}" oldbranch="$(pihole-FTL -b)" - if check_download_exists "$path"; then + check_download_exists "$path" + local ret=$? + if [ $ret -eq 0 ]; then echo " ${TICK} Branch ${2} exists" echo "${2}" > /etc/pihole/ftlbranch chmod 644 /etc/pihole/ftlbranch @@ -175,11 +177,19 @@ checkout() { # Update local and remote versions via updatechecker /opt/pihole/updatecheck.sh else - echo " ${CROSS} Requested branch \"${2}\" is not available" - ftlbranches=( $(git ls-remote https://github.com/pi-hole/ftl | grep 'heads' | sed 's/refs\/heads\///;s/ //g' | awk '{print $2}') ) - echo -e " ${INFO} Available branches for FTL are:" - for e in "${ftlbranches[@]}"; do echo " - $e"; done - exit 1 + if [[ $ret -eq 1 ]]; then + echo " ${CROSS} Requested branch \"${2}\" is not available" + ftlbranches=( $(git ls-remote https://github.com/pi-hole/ftl | grep 'heads' | sed 's/refs\/heads\///;s/ //g' | awk '{print $2}') ) + echo -e " ${INFO} Available branches for FTL are:" + for e in "${ftlbranches[@]}"; do echo " - $e"; done + exit 1 + elif [[ $ret -eq 2 ]]; then + printf " %b Unable to download from ftl.pi-hole.net. Please check your Internet connection and try again later.\\n" "${CROSS}" + exit 1 + else + printf " %b Unknown error. Please contact Pi-hole Support\\n" "${CROSS}" + exit 1 + fi fi else diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index 9dae66df..8a35ef2e 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -144,7 +144,7 @@ main() { local binary binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL) - if FTLcheckUpdate "${binary}" > /dev/null; then + if FTLcheckUpdate "${binary}"; then FTL_update=true echo -e " ${INFO} FTL:\\t\\t${COL_YELLOW}update available${COL_NC}" else @@ -155,8 +155,13 @@ main() { 2) echo -e " ${INFO} FTL:\\t\\t${COL_LIGHT_RED}Branch is not available.${COL_NC}\\n\\t\\t\\tUse ${COL_LIGHT_GREEN}pihole checkout ftl [branchname]${COL_NC} to switch to a valid branch." ;; + 3) + echo -e " ${INFO} FTL:\\t\\t${COL_LIGHT_RED}Something has gone wrong, cannot reach download server${COL_NC}" + exit 1 + ;; *) echo -e " ${INFO} FTL:\\t\\t${COL_LIGHT_RED}Something has gone wrong, contact support${COL_NC}" + exit 1 esac FTL_update=false fi diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f5bf15e6..af7ca151 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1687,12 +1687,19 @@ update_dialogs() { } check_download_exists() { + # Check if the download exists and we can reach the server status=$(curl --head --silent "https://ftl.pi-hole.net/${1}" | head -n 1) - if grep -q "404" <<< "$status"; then - return 1 - else + + # Check the status code + if grep -q "200" <<< "$status"; then return 0 + elif grep -q "404" <<< "$status"; then + return 1 fi + + # Other error or no status code at all, e.g., no Internet, server not + # available/reachable, ... + return 2 } fully_fetch_repo() { @@ -1957,10 +1964,8 @@ get_binary_name() { } FTLcheckUpdate() { - #In the next section we check to see if FTL is already installed (in case of pihole -r). - #If the installed version matches the latest version, then check the installed sha1sum of the binary vs the remote sha1sum. If they do not match, then download - printf " %b Checking for existing FTL binary...\\n" "${INFO}" - + # In the next section we check to see if FTL is already installed (in case of pihole -r). + # If the installed version matches the latest version, then check the installed sha1sum of the binary vs the remote sha1sum. If they do not match, then download local ftlLoc ftlLoc=$(command -v pihole-FTL 2>/dev/null) @@ -1979,14 +1984,24 @@ FTLcheckUpdate() { local localSha1 if [[ ! "${ftlBranch}" == "master" ]]; then - #Check whether or not the binary for this FTL branch actually exists. If not, then there is no update! + # Check whether or not the binary for this FTL branch actually exists. If not, then there is no update! local path path="${ftlBranch}/${binary}" # shellcheck disable=SC1090 - if ! check_download_exists "$path"; then - printf " %b Branch \"%s\" is not available.\\n" "${INFO}" "${ftlBranch}" - printf " %b Use %bpihole checkout ftl [branchname]%b to switch to a valid branch.\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${COL_NC}" - return 2 + check_download_exists "$path" + local ret=$? + if [ $ret -ne 0 ]; then + if [[ $ret -eq 1 ]]; then + printf " %b Branch \"%s\" is not available.\\n" "${INFO}" "${ftlBranch}" + printf " %b Use %bpihole checkout ftl [branchname]%b to switch to a valid branch.\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${COL_NC}" + return 2 + elif [[ $ret -eq 2 ]]; then + printf " %b Unable to download from ftl.pi-hole.net. Please check your Internet connection and try again later.\\n" "${CROSS}" + return 3 + else + printf " %b Unknown error. Please contact Pi-hole Support\\n" "${CROSS}" + return 4 + fi fi if [[ ${ftlLoc} ]]; then @@ -2011,12 +2026,14 @@ FTLcheckUpdate() { FTLversion=$(/usr/bin/pihole-FTL tag) local FTLlatesttag + # Get the latest version from the GitHub API if ! FTLlatesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep --color=never -i Location: | awk -F / '{print $NF}' | tr -d '[:cntrl:]'); then # There was an issue while retrieving the latest version printf " %b Failed to retrieve latest FTL release metadata" "${CROSS}" return 3 fi + # Check if the installed version matches the latest version if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then return 0 else From 6e946f76d6e0166d6f2d0e86ed7ab01686faf6bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 3 Feb 2024 10:21:48 +0000 Subject: [PATCH 188/200] Bump pytest from 7.4.4 to 8.0.0 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.4 to 8.0.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.4...8.0.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index dea4ace9..2b8dfb1b 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.4 +pytest == 8.0.0 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.12.1 From b5ab8ac1980442f19d474f630f47da6dd851f151 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 11 Feb 2024 16:54:22 +0100 Subject: [PATCH 189/200] Change UNIQUEness constraint from (address) to (address, type) in the adlist table. This will allow certain adlists to be associated to different groups. A possible scenario is an adlist meant to block a specific service (e.g. Twitter, Youtube, etc.). It can then either be used to ensure these services are really blocked on the devices of group A but will never be blocked on devices of group B. Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 097b0a78..42060443 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -27,7 +27,7 @@ CREATE TABLE domainlist CREATE TABLE adlist ( id INTEGER PRIMARY KEY AUTOINCREMENT, - address TEXT UNIQUE NOT NULL, + address TEXT NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), @@ -37,7 +37,8 @@ CREATE TABLE adlist invalid_domains INTEGER NOT NULL DEFAULT 0, status INTEGER NOT NULL DEFAULT 0, abp_entries INTEGER NOT NULL DEFAULT 0, - type INTEGER NOT NULL DEFAULT 0 + type INTEGER NOT NULL DEFAULT 0, + UNIQUE(address, type) ); CREATE TABLE adlist_by_group From bcb712b6e3f408b0080768b9c2c77c2a4c7c6dbd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 11 Feb 2024 21:07:15 +0100 Subject: [PATCH 190/200] Fix accidential double -ni from one of the last PRs Signed-off-by: DL6ER --- gravity.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gravity.sh b/gravity.sh index 592aa801..4d785d8a 100755 --- a/gravity.sh +++ b/gravity.sh @@ -114,7 +114,7 @@ gravity_swap_databases() { # Update timestamp when the gravity table was last updated successfully update_gravity_timestamp() { - output=$({ printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 -ni -ni "${gravityTEMPfile}"; } 2>&1) + output=$({ printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -352,8 +352,8 @@ gravity_DownloadBlocklists() { # Retrieve source URLs from gravity database # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true) - mapfile -t sources <<<"$(pihole-FTL sqlite3 -ni -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2>/dev/null)" - mapfile -t sourceIDs <<<"$(pihole-FTL sqlite3 -ni -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2>/dev/null)" + mapfile -t sources <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2>/dev/null)" + mapfile -t sourceIDs <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2>/dev/null)" mapfile -t sourceTypes <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2>/dev/null)" # Parse source domains from $sources @@ -667,12 +667,12 @@ gravity_Table_Count() { local table="${1}" local str="${2}" local num - num="$(pihole-FTL sqlite3 -ni -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM ${table};")" + num="$(pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM ${table};")" if [[ "${table}" == "gravity" ]]; then local unique - unique="$(pihole-FTL sqlite3 -ni -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" + unique="$(pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" echo -e " ${INFO} Number of ${str}: ${num} (${COL_BOLD}${unique} unique domains${COL_NC})" - pihole-FTL sqlite3 -ni -ni "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" + pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" else echo -e " ${INFO} Number of ${str}: ${num}" fi From 75fadb9b55fbb1454bbddd60d0ed99924200d2d4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 13 Feb 2024 08:55:26 +0100 Subject: [PATCH 191/200] Adlists need to be grouped by both address and type to differentiate between anti-/gravity lists Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 581d2069..df7db893 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -48,7 +48,7 @@ GenerateOutput() { lists_data=$(printf %s "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]') # construct a new json for the gravity results where each object contains the adlist URL and the related domains - gravity_data=$(printf %s "${data}" | jq '.search.gravity | group_by(.address) | map({ address: (.[0].address), domains: [.[] | .domain] })') + gravity_data=$(printf %s "${data}" | jq '.search.gravity | group_by(.address,.type) | map({ address: (.[0].address), domains: [.[] | .domain] })') # number of objects in each json num_gravity=$(printf %s "${gravity_data}" | jq length) From 9ff43040ec52d9bc1adefa2f4a69775835dfc67c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 13 Feb 2024 09:58:23 +0100 Subject: [PATCH 192/200] Add list type in pihole -q Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index df7db893..493c75ea 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -41,14 +41,14 @@ Options: GenerateOutput() { local data gravity_data lists_data num_gravity num_lists search_type_str - local gravity_data_csv lists_data_csv line current_domain + local gravity_data_csv lists_data_csv line current_domain url type color data="${1}" # construct a new json for the list results where each object contains the domain and the related type lists_data=$(printf %s "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]') # construct a new json for the gravity results where each object contains the adlist URL and the related domains - gravity_data=$(printf %s "${data}" | jq '.search.gravity | group_by(.address,.type) | map({ address: (.[0].address), domains: [.[] | .domain] })') + gravity_data=$(printf %s "${data}" | jq '.search.gravity | group_by(.address,.type) | map({ address: (.[0].address), type: (.[0].type), domains: [.[] | .domain] })') # number of objects in each json num_gravity=$(printf %s "${gravity_data}" | jq length) @@ -78,15 +78,27 @@ GenerateOutput() { if [ "${num_gravity}" -gt 0 ]; then # Convert the data to a csv, each line is a "URL,domain,domain,...." string # not using jq's @csv here as it quotes each value individually - gravity_data_csv=$(printf %s "${gravity_data}" | jq --raw-output '.[] | [.address, .domains[]] | join(",")') + gravity_data_csv=$(printf %s "${gravity_data}" | jq --raw-output '.[] | [.address, .type, .domains[]] | join(",")') # Generate line-by-line output for each csv line echo "${gravity_data_csv}" | while read -r line; do + # Get first part of the line, the URL + url=${line%%,*} + + # cut off URL, leaving "type,domain,domain,...." + line=${line#*,} + type=${line%%,*} + # type == "block" -> red, type == "allow" -> green + if [ "${type}" = "block" ]; then + color="${COL_RED}" + else + color="${COL_GREEN}" + fi # print adlist URL - printf "%s\n\n" " - ${COL_BLUE}${line%%,*}${COL_NC}" + printf "%s (%s)\n\n" " - ${COL_BLUE}${url}${COL_NC}" "${color}${type}${COL_NC}" - # cut off URL, leaving "domain,domain,...." + # cut off type, leaving "domain,domain,...." line=${line#*,} # print each domain and remove it from the string until nothing is left while [ ${#line} -gt 0 ]; do From b322f1e98bf1ec64b705add4f089cda22ae8e5e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 17 Feb 2024 10:50:41 +0000 Subject: [PATCH 193/200] Bump tox from 4.12.1 to 4.13.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.12.1 to 4.13.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.12.1...4.13.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 2b8dfb1b..a6826838 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.0.0 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.12.1 +tox == 4.13.0 From 3ed29f494b93be8b03bc9739ad5f03a7cf578a25 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 24 Feb 2024 12:45:22 +0000 Subject: [PATCH 194/200] Bump pytest from 8.0.0 to 8.0.1 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.0.0 to 8.0.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.0.0...8.0.1) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index a6826838..3e341719 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.0.0 +pytest == 8.0.1 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.13.0 From 07e6c0d250febddf706e5b414466b88a6c89b288 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 24 Feb 2024 12:49:07 +0000 Subject: [PATCH 195/200] Bump pytest-testinfra from 10.0.0 to 10.1.0 in /test Bumps [pytest-testinfra](https://github.com/pytest-dev/pytest-testinfra) from 10.0.0 to 10.1.0. - [Release notes](https://github.com/pytest-dev/pytest-testinfra/releases) - [Changelog](https://github.com/pytest-dev/pytest-testinfra/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-testinfra/compare/10.0.0...10.1.0) --- updated-dependencies: - dependency-name: pytest-testinfra dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 3e341719..2a559b05 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 8.0.1 pytest-xdist == 3.5.0 -pytest-testinfra == 10.0.0 +pytest-testinfra == 10.1.0 tox == 4.13.0 From fd1372df3e72509e16c92bfcb9b4489d9b232fc6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Mar 2024 10:01:31 +0000 Subject: [PATCH 196/200] Bump pytest from 8.0.1 to 8.0.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.0.1 to 8.0.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.0.1...8.0.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 2a559b05..ffb05813 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.0.1 +pytest == 8.0.2 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 tox == 4.13.0 From bfd8b572cbc20a294a2886f1cfb4e2315d10ca98 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Mar 2024 10:07:37 +0000 Subject: [PATCH 197/200] Bump tox from 4.13.0 to 4.14.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.13.0 to 4.14.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.13.0...4.14.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index ffb05813..d4415e2b 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.0.2 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 -tox == 4.13.0 +tox == 4.14.1 From c02401b81e43febae7365f960c2d530afc714e24 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 9 Mar 2024 11:17:33 +0100 Subject: [PATCH 198/200] Highlight "### CHANGED" strings in the debug log of pihole.toml Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index f5a57278..7c558127 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -865,8 +865,6 @@ make_array_from_file() { local limit=${2} # A local iterator for testing if we are at the limit above local i=0 - # Set the array to be empty so we can start fresh when the function is used - local file_content=() # If the file is a directory if [[ -d "${filename}" ]]; then # do nothing since it cannot be parsed @@ -878,11 +876,14 @@ make_array_from_file() { new_line=$(echo "${line}" | sed -e 's/^\s*#.*$//' -e '/^$/d') # If the line still has content (a non-zero value) if [[ -n "${new_line}" ]]; then - # Put it into the array - file_content+=("${new_line}") - else - # Otherwise, it's a blank line or comment, so do nothing - : + + # If the string contains "### CHANGED", highlight this part in red + if [[ "${new_line}" == *"### CHANGED"* ]]; then + new_line="${new_line//### CHANGED/${COL_RED}### CHANGED${COL_NC}}" + fi + + # Finally, write this line to the log + log_write " ${new_line}" fi # Increment the iterator +1 i=$((i+1)) @@ -894,12 +895,6 @@ make_array_from_file() { break fi done < "${filename}" - # Now the we have made an array of the file's content - for each_line in "${file_content[@]}"; do - # Print each line - # At some point, we may want to check the file line-by-line, so that's the reason for an array - log_write " ${each_line}" - done fi } From 2fd0de4743b134ad1a0be6bea119301d16b98ef5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 10 Mar 2024 08:43:37 +0100 Subject: [PATCH 199/200] Verify that we actually downloaded a valid checksum before comparing it to the local one. This covers situations where downloading the checksum from remote might have failed Signed-off-by: DL6ER --- automated install/basic-install.sh | 38 +++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 24abb7e7..eca7d8b0 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2002,9 +2002,11 @@ FTLcheckUpdate() { local localSha1 if [[ ! "${ftlBranch}" == "master" ]]; then - # Check whether or not the binary for this FTL branch actually exists. If not, then there is no update! + # This is not the master branch local path path="${ftlBranch}/${binary}" + + # Check whether or not the binary for this FTL branch actually exists. If not, then there is no update! # shellcheck disable=SC1090 check_download_exists "$path" local ret=$? @@ -2023,12 +2025,20 @@ FTLcheckUpdate() { fi if [[ ${ftlLoc} ]]; then - # We already have a pihole-FTL binary downloaded. - # Alt branches don't have a tagged version against them, so just confirm the checksum of the local vs remote to decide whether we download or not + # We already have a pihole-FTL binary installed, check if it's the + # same as the remote one + # Alt branches don't have a tagged version against them, so just + # confirm the checksum of the local vs remote to decide whether we + # download or not remoteSha1=$(curl -sSL --fail "https://ftl.pi-hole.net/${ftlBranch}/${binary}.sha1" | cut -d ' ' -f 1) - localSha1=$(sha1sum "$(command -v pihole-FTL)" | cut -d ' ' -f 1) + localSha1=$(sha1sum "${ftlLoc}" | cut -d ' ' -f 1) - if [[ "${remoteSha1}" != "${localSha1}" ]]; then + # Check we downloaded a valid checksum (no 404 or other error like + # no DNS resolution) + if [[ ! "${remoteSha1}" =~ ^[a-f0-9]{40}$ ]]; then + printf " %b Remote checksum not available, trying to download binary from ftl.pi-hole.net.\\n" "${CROSS}" + return 0 + elif [[ "${remoteSha1}" != "${localSha1}" ]]; then printf " %b Checksums do not match, downloading from ftl.pi-hole.net.\\n" "${INFO}" return 0 else @@ -2039,7 +2049,10 @@ FTLcheckUpdate() { return 0 fi else + # This is the master branch if [[ ${ftlLoc} ]]; then + # We already have a pihole-FTL binary installed, check if it's the + # same as the remote one local FTLversion FTLversion=$(/usr/bin/pihole-FTL tag) local FTLlatesttag @@ -2053,15 +2066,24 @@ FTLcheckUpdate() { # Check if the installed version matches the latest version if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then + # If the installed version does not match the latest version, then download return 0 else + # If the installed version matches the latest version, then + # check the installed sha1sum of the binary vs the remote + # sha1sum. If they do not match, then download printf " %b Latest FTL Binary already installed (%s). Confirming Checksum...\\n" "${INFO}" "${FTLlatesttag}" remoteSha1=$(curl -sSL --fail "https://github.com/pi-hole/FTL/releases/download/${FTLversion%$'\r'}/${binary}.sha1" | cut -d ' ' -f 1) - localSha1=$(sha1sum "$(command -v pihole-FTL)" | cut -d ' ' -f 1) + localSha1=$(sha1sum "${ftlLoc}" | cut -d ' ' -f 1) - if [[ "${remoteSha1}" != "${localSha1}" ]]; then - printf " %b Corruption detected...\\n" "${INFO}" + # Check we downloaded a valid checksum (no 404 or other error like + # no DNS resolution) + if [[ ! "${remoteSha1}" =~ ^[a-f0-9]{40}$ ]]; then + printf " %b Remote checksum not available, trying to redownload binary...\\n" "${CROSS}" + return 0 + elif [[ "${remoteSha1}" != "${localSha1}" ]]; then + printf " %b Corruption detected, redownloading binary...\\n" "${CROSS}" return 0 else printf " %b Checksum correct. No need to download!\\n" "${INFO}" From 82a83c497dd772cf6f044ec50b4471af102bff37 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 10 Mar 2024 21:18:13 +0100 Subject: [PATCH 200/200] Reduce code-duplication Signed-off-by: DL6ER --- automated install/basic-install.sh | 59 +++++++++++++----------------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index eca7d8b0..d057cb82 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2030,21 +2030,9 @@ FTLcheckUpdate() { # Alt branches don't have a tagged version against them, so just # confirm the checksum of the local vs remote to decide whether we # download or not - remoteSha1=$(curl -sSL --fail "https://ftl.pi-hole.net/${ftlBranch}/${binary}.sha1" | cut -d ' ' -f 1) - localSha1=$(sha1sum "${ftlLoc}" | cut -d ' ' -f 1) - - # Check we downloaded a valid checksum (no 404 or other error like - # no DNS resolution) - if [[ ! "${remoteSha1}" =~ ^[a-f0-9]{40}$ ]]; then - printf " %b Remote checksum not available, trying to download binary from ftl.pi-hole.net.\\n" "${CROSS}" - return 0 - elif [[ "${remoteSha1}" != "${localSha1}" ]]; then - printf " %b Checksums do not match, downloading from ftl.pi-hole.net.\\n" "${INFO}" - return 0 - else - printf " %b Checksum of installed binary matches remote. No need to download!\\n" "${INFO}" - return 1 - fi + printf " %b FTL binary already installed. Confirming Checksum...\\n" "${INFO}" + checkSumFile="https://ftl.pi-hole.net/${ftlBranch}/${binary}.sha1" + # Continue further down... else return 0 fi @@ -2066,34 +2054,39 @@ FTLcheckUpdate() { # Check if the installed version matches the latest version if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then - # If the installed version does not match the latest version, then download + # If the installed version does not match the latest version, + # then download return 0 else # If the installed version matches the latest version, then # check the installed sha1sum of the binary vs the remote # sha1sum. If they do not match, then download - printf " %b Latest FTL Binary already installed (%s). Confirming Checksum...\\n" "${INFO}" "${FTLlatesttag}" - - remoteSha1=$(curl -sSL --fail "https://github.com/pi-hole/FTL/releases/download/${FTLversion%$'\r'}/${binary}.sha1" | cut -d ' ' -f 1) - localSha1=$(sha1sum "${ftlLoc}" | cut -d ' ' -f 1) - - # Check we downloaded a valid checksum (no 404 or other error like - # no DNS resolution) - if [[ ! "${remoteSha1}" =~ ^[a-f0-9]{40}$ ]]; then - printf " %b Remote checksum not available, trying to redownload binary...\\n" "${CROSS}" - return 0 - elif [[ "${remoteSha1}" != "${localSha1}" ]]; then - printf " %b Corruption detected, redownloading binary...\\n" "${CROSS}" - return 0 - else - printf " %b Checksum correct. No need to download!\\n" "${INFO}" - return 1 - fi + printf " %b Latest FTL binary already installed (%s). Confirming Checksum...\\n" "${INFO}" "${FTLlatesttag}" + checkSumFile="https://github.com/pi-hole/FTL/releases/download/${FTLversion%$'\r'}/${binary}.sha1" + # Continue further down... fi else return 0 fi fi + + # If we reach this point, we need to check the checksum of the local vs + # remote to decide whether we download or not + remoteSha1=$(curl -sSL --fail "${checkSumFile}" | cut -d ' ' -f 1) + localSha1=$(sha1sum "${ftlLoc}" | cut -d ' ' -f 1) + + # Check we downloaded a valid checksum (no 404 or other error like + # no DNS resolution) + if [[ ! "${remoteSha1}" =~ ^[a-f0-9]{40}$ ]]; then + printf " %b Remote checksum not available, trying to redownload binary...\\n" "${CROSS}" + return 0 + elif [[ "${remoteSha1}" != "${localSha1}" ]]; then + printf " %b Corruption detected, redownloading binary...\\n" "${CROSS}" + return 0 + fi + + printf " %b Checksum correct. No need to download!\\n" "${INFO}" + return 1 } # Detect suitable FTL binary platform