From 2c8dcd86e570df2c1b1372b330b095d749f804d9 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Thu, 9 Nov 2017 20:47:15 +0000 Subject: [PATCH 001/163] remove package_check to avoid situations like #1760 Signed-off-by: Adam Warner --- automated install/uninstall.sh | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 2f4f4f9f..5628702f 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -62,19 +62,12 @@ if [ -x "$(command -v rpm)" ]; then package_check() { rpm -qa | grep ^$1- > /dev/null } - package_cleanup() { - ${SUDO} ${PKG_MANAGER} -y autoremove - } elif [ -x "$(command -v apt-get)" ]; then # Debian Family PKG_REMOVE="${PKG_MANAGER} -y remove --purge" package_check() { dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -c "ok installed" } - package_cleanup() { - ${SUDO} ${PKG_MANAGER} -y autoremove - ${SUDO} ${PKG_MANAGER} -y autoclean - } else echo -e " ${CROSS} OS distribution not supported" exit 1 @@ -103,14 +96,9 @@ removeAndPurge() { done # Remove dnsmasq config files - ${SUDO} rm -f /etc/dnsmasq.conf /etc/dnsmasq.conf.orig /etc/dnsmasq.d/01-pihole.conf &> /dev/null + ${SUDO} rm -f /etc/dnsmasq.conf /etc/dnsmasq.conf.orig /etc/dnsmasq.d/*-pihole*.conf &> /dev/null echo -e " ${TICK} Removing dnsmasq config files" - # Take care of any additional package cleaning - echo -ne " ${INFO} Removing & cleaning remaining dependencies..." - package_cleanup &> /dev/null - echo -e "${OVER} ${TICK} Removed & cleaned up remaining dependencies" - # Call removeNoPurge to remove Pi-hole specific files removeNoPurge } From 36945a67d03d7ea28b706ed4b5b0b57b2ed0c93a Mon Sep 17 00:00:00 2001 From: Ryan Knapper Date: Wed, 7 Feb 2018 17:04:10 -0500 Subject: [PATCH 002/163] Prevent redundant entries in to adlists.list Grep ${args[3]} and only add if grep -c -eq 0 Signed-off-by: Ryan Knapper --- advanced/Scripts/webpage.sh | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index d47eaff6..bd69d8ad 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -182,10 +182,6 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423 add_dnsmasq_setting "interface" "${PIHOLE_INTERFACE}" fi - if [[ "${CONDITIONAL_FORWARDING}" == true ]]; then - add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_DOMAIN}/${CONDITIONAL_FORWARDING_IP}" - add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_REVERSE}/${CONDITIONAL_FORWARDING_IP}" - fi } @@ -215,17 +211,6 @@ SetDNSServers() { else change_setting "DNSSEC" "false" fi - if [[ "${args[6]}" == "conditional_forwarding" ]]; then - change_setting "CONDITIONAL_FORWARDING" "true" - change_setting "CONDITIONAL_FORWARDING_IP" "${args[7]}" - change_setting "CONDITIONAL_FORWARDING_DOMAIN" "${args[8]}" - change_setting "CONDITIONAL_FORWARDING_REVERSE" "${args[9]}" - else - change_setting "CONDITIONAL_FORWARDING" "false" - delete_setting "CONDITIONAL_FORWARDING_IP" - delete_setting "CONDITIONAL_FORWARDING_DOMAIN" - delete_setting "CONDITIONAL_FORWARDING_REVERSE" - fi ProcessDNSSettings @@ -269,7 +254,7 @@ ProcessDHCPSettings() { fi if [[ "${PIHOLE_DOMAIN}" == "" ]]; then - PIHOLE_DOMAIN="local" + PIHOLE_DOMAIN="lan" change_setting "PIHOLE_DOMAIN" "${PIHOLE_DOMAIN}" fi @@ -361,7 +346,9 @@ CustomizeAdLists() { elif [[ "${args[2]}" == "disable" ]]; then sed -i "\\@${args[3]}@s/^http/#http/g" "${list}" elif [[ "${args[2]}" == "add" ]]; then - echo "${args[3]}" >> ${list} + if [[ $(grep -c "${args[3]}" "${list}") -eq 0 ]] ; then + echo "${args[3]}" >> ${list} + fi elif [[ "${args[2]}" == "del" ]]; then var=$(echo "${args[3]}" | sed 's/\//\\\//g') sed -i "/${var}/Id" "${list}" From f2b3752f3dea7bd7e7e9ab6e06fba372bebeb454 Mon Sep 17 00:00:00 2001 From: Ryan Knapper Date: Wed, 7 Feb 2018 17:08:55 -0500 Subject: [PATCH 003/163] lan to local Reduced differences. Signed-off-by: Ryan Knapper --- advanced/Scripts/webpage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index bd69d8ad..6e13d24e 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -254,7 +254,7 @@ ProcessDHCPSettings() { fi if [[ "${PIHOLE_DOMAIN}" == "" ]]; then - PIHOLE_DOMAIN="lan" + PIHOLE_DOMAIN="local" change_setting "PIHOLE_DOMAIN" "${PIHOLE_DOMAIN}" fi From 0541d8f1c550e9c55acfdcbbbd6b225885644ac1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 24 Feb 2018 11:13:33 +0100 Subject: [PATCH 004/163] Try to determine if the user is running a 32bit OS on a 64bit system. If so, download the 32bit binary as we cannot expect the 64bit libraries to be present. Signed-off-by: DL6ER --- advanced/Scripts/piholeCheckout.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index 9e97c69c..63198b5a 100644 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -68,9 +68,20 @@ FTLinstall() { } get_binary_name() { + # This gives the machine architecture which may be different from the OS architecture... local machine machine=$(uname -m) + # This gives the architecture of packages dpkg installs (for example, "i386") + local dpkgarch + dpkgarch=$(dpkg --print-architecture 2> /dev/null) + + # Special case: This is a 32 bit OS, installed on a 64 bit machine + # -> change machine architecture to download the 32 bit executable + if [[ "${machine}" == "x86_64" && "${dpkgarch}" == "i386" ]]; then + machine="i686" + fi + local str str="Detecting architecture" echo -ne " ${INFO} ${str}..." From 5ecfc58e5fb016d1f8fd031325d8f7c98aa8cb4a Mon Sep 17 00:00:00 2001 From: Ryan Knapper Date: Wed, 28 Feb 2018 13:06:07 -0500 Subject: [PATCH 005/163] Require exact match Updated to require an exact match to reduce false-positives, as suggested by DL6ER. Signed-off-by: Ryan Knapper --- advanced/Scripts/webpage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 6e13d24e..3c76e173 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -346,7 +346,7 @@ CustomizeAdLists() { elif [[ "${args[2]}" == "disable" ]]; then sed -i "\\@${args[3]}@s/^http/#http/g" "${list}" elif [[ "${args[2]}" == "add" ]]; then - if [[ $(grep -c "${args[3]}" "${list}") -eq 0 ]] ; then + if [[ $(grep -c "^${args[3]}$" "${list}") -eq 0 ]] ; then echo "${args[3]}" >> ${list} fi elif [[ "${args[2]}" == "del" ]]; then From 143e75d213a0cbcf9b2f179fe4e4a095c248c8e1 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sun, 25 Mar 2018 09:39:29 -0500 Subject: [PATCH 006/163] fix empty ports on some systems Signed-off-by: Jacob Salmela --- advanced/Scripts/piholeDebug.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index b668af94..633fb7a9 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -652,7 +652,7 @@ check_required_ports() { # Sort the addresses and remove duplicates while IFS= read -r line; do ports_in_use+=( "$line" ) - done < <( lsof -i -P -n | awk -F' ' '/LISTEN/ {print $9, $1}' | sort -n | uniq | cut -d':' -f2 ) + done < <( lsof -i -P -n | awk -F' ' '/LISTEN/ {print $1, $9}' | sort -n | tr -d '[*]\r' | uniq | awk '{print $2, $1}' ) # Now that we have the values stored, for i in "${!ports_in_use[@]}"; do From c3f391dc5ac45209f8e963f681bea8787657fbf9 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 31 Mar 2018 14:08:48 -0500 Subject: [PATCH 007/163] maybe fixes #2028 by deleting null bytes Signed-off-by: Jacob Salmela --- gravity.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 395ea548..b040b69b 100755 --- a/gravity.sh +++ b/gravity.sh @@ -384,7 +384,7 @@ gravity_ConsolidateDownloadedBlocklists() { tr -d '\r' < "${i}" | tr '[:upper:]' '[:lower:]' >> "${piholeDir}/${matterAndLight}" # Ensure that the first line of a new list is on a new line - lastLine=$(tail -1 "${piholeDir}/${matterAndLight}") + lastLine=$(tail -1 "${piholeDir}/${matterAndLight}" | tr -d '\0') if [[ "${#lastLine}" -gt 0 ]]; then echo "" >> "${piholeDir}/${matterAndLight}" fi @@ -505,7 +505,7 @@ gravity_ParseBlacklistDomains() { # Empty $accretionDisc if it already exists, otherwise, create it : > "${piholeDir}/${accretionDisc}" - + if [[ -f "${piholeDir}/${whitelistMatter}" ]]; then gravity_ParseDomainsIntoHosts "${piholeDir}/${whitelistMatter}" "${piholeDir}/${accretionDisc}" else From 187848660c15243802942b05c78dd572b3054d28 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 31 Mar 2018 16:12:29 -0500 Subject: [PATCH 008/163] try another fix for #2028 Signed-off-by: Jacob Salmela --- gravity.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index b040b69b..677f1956 100755 --- a/gravity.sh +++ b/gravity.sh @@ -382,9 +382,8 @@ gravity_ConsolidateDownloadedBlocklists() { if [[ -r "${i}" ]]; then # Remove windows CRs from file, convert list to lower case, and append into $matterAndLight tr -d '\r' < "${i}" | tr '[:upper:]' '[:lower:]' >> "${piholeDir}/${matterAndLight}" - # Ensure that the first line of a new list is on a new line - lastLine=$(tail -1 "${piholeDir}/${matterAndLight}" | tr -d '\0') + IFS= read -r -d '' lastLine <"${piholeDir}/${matterAndLight}" || [[ $lastLine ]] if [[ "${#lastLine}" -gt 0 ]]; then echo "" >> "${piholeDir}/${matterAndLight}" fi From 1a275ba18458363f66da62ff6ab0481cf472e3a7 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sun, 1 Apr 2018 06:40:48 -0500 Subject: [PATCH 009/163] debug user locale; improve function to parse variables and files Signed-off-by: Jacob Salmela --- advanced/Scripts/piholeDebug.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index a16457a0..33114794 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -493,6 +493,12 @@ parse_setup_vars() { fi } +parse_locale() { + echo_current_diagnostic "Locale" + local pihole_locale="$(locale)" + parse_file "${pihole_locale}" +} + does_ip_match_setup_vars() { # Check for IPv4 or 6 local protocol="${1}" @@ -879,8 +885,11 @@ parse_file() { # Put the current Internal Field Separator into another variable so it can be restored later OLD_IFS="$IFS" # Get the lines that are in the file(s) and store them in an array for parsing later - IFS=$'\r\n' command eval 'file_info=( $(cat "${filename}") )' - + if [[ -f "$filename" ]]; then + IFS=$'\r\n' command eval 'file_info=( $(cat "${filename}") )' + else + read -a file_info <<< $filename + fi # Set a named variable for better readability local file_lines # For each line in the file, @@ -1165,6 +1174,7 @@ parse_setup_vars check_x_headers analyze_gravity_list show_content_of_pihole_files +parse_locale analyze_pihole_log copy_to_debug_log upload_to_tricorder From 250b445eeee8691d9b5a3bfee30f48d46e87853f Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Tue, 10 Apr 2018 21:37:04 -0700 Subject: [PATCH 010/163] Split declaration and population for stickler. Signed-off-by: Dan Schaper --- advanced/Scripts/piholeDebug.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 33114794..f34b8646 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -494,8 +494,9 @@ parse_setup_vars() { } parse_locale() { + local pihole_locale echo_current_diagnostic "Locale" - local pihole_locale="$(locale)" + pihole_locale="$(locale)" parse_file "${pihole_locale}" } From 5ffc3561eda57963e5b88b74816184869574f66e Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Wed, 11 Apr 2018 20:35:51 -0500 Subject: [PATCH 011/163] implement dschapers suggestions--better command, less subshells, and finer formatting Signed-off-by: Jacob Salmela --- advanced/Scripts/piholeDebug.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 633fb7a9..44e71a25 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -652,15 +652,22 @@ check_required_ports() { # Sort the addresses and remove duplicates while IFS= read -r line; do ports_in_use+=( "$line" ) - done < <( lsof -i -P -n | awk -F' ' '/LISTEN/ {print $1, $9}' | sort -n | tr -d '[*]\r' | uniq | awk '{print $2, $1}' ) + done < <( lsof -iTCP -sTCP:LISTEN -P -n +c 10 ) # Now that we have the values stored, for i in "${!ports_in_use[@]}"; do # loop through them and assign some local variables - local port_number - port_number="$(echo "${ports_in_use[$i]}" | awk '{print $1}')" local service_name - service_name=$(echo "${ports_in_use[$i]}" | awk '{print $2}') + service_name=$(echo "${ports_in_use[$i]}" | awk '{print $1}') + local protocol_type + protocol_type=$(echo "${ports_in_use[$i]}" | awk '{print $5}') + local port_number + port_number="$(echo "${ports_in_use[$i]}" | awk '{print $9}')" + + # Skip the line if it's the titles of the columns the lsof command produces + if [[ "${service_name}" == COMMAND ]]; then + continue + fi # Use a case statement to determine if the right services are using the right ports case "${port_number}" in 53) compare_port_to_service_assigned "${resolver}" @@ -670,7 +677,7 @@ check_required_ports() { 4711) compare_port_to_service_assigned "${ftl}" ;; # If it's not a default port that Pi-hole needs, just print it out for the user to see - *) log_write "[${port_number}] is in use by ${service_name}"; + *) log_write "${port_number} ${service_name} (${protocol_type})"; esac done } From 8ad37af70ee884c3d863755b32f87dea35681ae5 Mon Sep 17 00:00:00 2001 From: bcambl Date: Tue, 1 May 2018 00:28:55 -0600 Subject: [PATCH 012/163] CentOS install re-worked to include PHP7 via Remi repository Signed-off-by: bcambl --- automated install/basic-install.sh | 73 ++++++++++++++++++++++++++---- 1 file changed, 65 insertions(+), 8 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index e7c5cd55..753f1211 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -217,14 +217,71 @@ elif command -v rpm &> /dev/null; then PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l" INSTALLER_DEPS=(dialog git iproute net-tools newt procps-ng) PIHOLE_DEPS=(bc bind-utils cronie curl findutils nmap-ncat sudo unzip wget libidn2 psmisc) - PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php php-common php-cli php-pdo) - # EPEL (https://fedoraproject.org/wiki/EPEL) is required for lighttpd on CentOS - if grep -qi 'centos' /etc/redhat-release; then - INSTALLER_DEPS=("${INSTALLER_DEPS[@]}" "epel-release"); - fi - LIGHTTPD_USER="lighttpd" - LIGHTTPD_GROUP="lighttpd" - LIGHTTPD_CFG="lighttpd.conf.fedora" + PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo) + LIGHTTPD_USER="lighttpd" + LIGHTTPD_GROUP="lighttpd" + LIGHTTPD_CFG="lighttpd.conf.fedora" + # If the host OS is Fedora, + if grep -qi 'fedora' /etc/redhat-release; then + # all required packages should be available by default with the latest fedora release + : # continue + # or if host OS is CentOS, + elif grep -qi 'centos' /etc/redhat-release; then + # Pi-Hole currently supports CentOS 7+ with PHP7+ + SUPPORTED_CENTOS_VERSION=7 + SUPPORTED_CENTOS_PHP_VERSION=7 + # Check current CentOS major release version + CURRENT_CENTOS_VERSION=$(rpm -q --queryformat '%{VERSION}' centos-release) + # Check if CentOS version is supported + if [[ $CURRENT_CENTOS_VERSION -lt $SUPPORTED_CENTOS_VERSION ]]; then + echo -e " ${CROSS} CentOS $CURRENT_CENTOS_VERSION is not suported." + echo -e " Please update to CentOS release $SUPPORTED_CENTOS_VERSION or later" + # exit the installer + exit + fi + # on CentOS we need to add the EPEL repository to gain access to Fedora packages + EPEL_PKG="epel-release" + rpm -q ${EPEL_PKG} &> /dev/null || rc=$? + if [[ $rc -ne 0 ]]; then + echo -e " ${INFO} Enabling EPEL package repository (https://fedoraproject.org/wiki/EPEL)" + "${PKG_INSTALL[@]}" ${EPEL_PKG} &> /dev/null + echo -e " ${TICK} Installed ${EPEL_PKG}" + fi + + # The default php on CentOS 7.x is 5.4 which is EOL + # Check if the version of PHP available via installed repositories is >= to PHP 7 + AVAILABLE_PHP_VERSION=$(${PKG_MANAGER} info php | grep -i version | grep -o '[0-9]\+' | head -1) + if [[ $AVAILABLE_PHP_VERSION -ge $SUPPORTED_CENTOS_PHP_VERSION ]]; then + # Since PHP 7 is available by default, install via default PHP package names + : # do nothing as PHP is current + else + REMI_PKG="remi-release" + REMI_REPO="remi-php72" + rpm -q ${REMI_PKG} &> /dev/null || rc=$? + if [[ $rc -ne 0 ]]; then + # The PHP version available via default repositories is older than version 7 + if ! whiptail --defaultno --title "PHP 7 Update (recommended)" --yesno "PHP 7.x is recommended for both security and language features.\\nWould you like to install PHP7 via Remi's RPM repository?\\n\\nSee: https://rpms.remirepo.net for more information" ${r} ${c}; then + # User decided to NOT update PHP from REMI, attempt to install the default available PHP version + echo -e " ${INFO} User opt-out of PHP 7 upgrade on CentOS. Deprecated PHP may be in use." + : # continue with unsupported php version + else + echo -e " ${INFO} Enabling Remi's RPM repository (https://rpms.remirepo.net)" + "${PKG_INSTALL[@]}" "https://rpms.remirepo.net/enterprise/${REMI_PKG}-$(rpm -E '%{rhel}').rpm" &> /dev/null + # enable the PHP 7 repository via yum-config-manager (provided by yum-utils) + "${PKG_INSTALL[@]}" "yum-utils" &> /dev/null + yum-config-manager --enable ${REMI_REPO} &> /dev/null + echo -e " ${TICK} Remi's RPM repository has been enabled for PHP7" + + fi + fi + fi + else + # If not a supported version of Fedora or CentOS, + echo -e " ${CROSS} Unsupported RPM based distribution" + # exit the installer + exit + fi + # If neither apt-get or rmp/dnf are found else From ec3802c180385edf2d092a87619c76062440a314 Mon Sep 17 00:00:00 2001 From: bcambl Date: Thu, 3 May 2018 23:41:42 -0600 Subject: [PATCH 013/163] add 'which' dependency (missing on Fedora minimal) Signed-off-by: bcambl --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 753f1211..878f2785 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -215,7 +215,7 @@ elif command -v rpm &> /dev/null; then UPDATE_PKG_CACHE=":" PKG_INSTALL=(${PKG_MANAGER} install -y) PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l" - INSTALLER_DEPS=(dialog git iproute net-tools newt procps-ng) + INSTALLER_DEPS=(dialog git iproute net-tools newt procps-ng which) PIHOLE_DEPS=(bc bind-utils cronie curl findutils nmap-ncat sudo unzip wget libidn2 psmisc) PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo) LIGHTTPD_USER="lighttpd" From d3dda443cd03aa6a9ad3b1c87d3d12a68c9bb60d Mon Sep 17 00:00:00 2001 From: bcambl Date: Fri, 4 May 2018 00:02:47 -0600 Subject: [PATCH 014/163] flip uninstall compatability check Signed-off-by: bcambl --- automated install/uninstall.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 2f4c58a1..b58c3ecb 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -53,24 +53,24 @@ if [[ "${INSTALL_WEB_SERVER}" == true ]]; then fi # Compatability -if [ -x "$(command -v rpm)" ]; then - # Fedora Family - PKG_REMOVE="${PKG_MANAGER} remove -y" +if [ -x "$(command -v apt-get)" ]; then + # Debian Family + PKG_REMOVE="${PKG_MANAGER} -y remove --purge" package_check() { - rpm -qa | grep ^$1- > /dev/null + dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -c "ok installed" } package_cleanup() { ${SUDO} ${PKG_MANAGER} -y autoremove + ${SUDO} ${PKG_MANAGER} -y autoclean } -elif [ -x "$(command -v apt-get)" ]; then - # Debian Family - PKG_REMOVE="${PKG_MANAGER} -y remove --purge" +elif [ -x "$(command -v rpm)" ]; then + # Fedora Family + PKG_REMOVE="${PKG_MANAGER} remove -y" package_check() { - dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -c "ok installed" + rpm -qa | grep ^$1- > /dev/null } package_cleanup() { ${SUDO} ${PKG_MANAGER} -y autoremove - ${SUDO} ${PKG_MANAGER} -y autoclean } else echo -e " ${CROSS} OS distribution not supported" From 31951dae4c8dbefb7606251dfa800fcefe6614df Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Fri, 11 May 2018 14:31:42 +1000 Subject: [PATCH 015/163] Update index.php Avoiding calling empty() on a function allows this to work under PHP5. Making the check for blocklist generation in this way instead is compatible with both PHP5 and PHP7. Signed-off-by: Rob Gill --- advanced/index.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/advanced/index.php b/advanced/index.php index d097fe0f..041939a7 100644 --- a/advanced/index.php +++ b/advanced/index.php @@ -102,7 +102,8 @@ if ($serverName === "pi.hole") { $bpAskAdmin = !empty($svEmail) ? '' : ""; // Determine if at least one block list has been generated -if (empty(glob("/etc/pihole/list.0.*.domains"))) +$blocklistglob = glob("/etc/pihole/list.0.*.domains"); +if ($blocklistglob = "") die("[ERROR] There are no domain lists generated lists within /etc/pihole/! Please update gravity by running pihole -g, or repair Pi-hole using pihole -r."); // Set location of adlists file From 5e99baf7b9b9ae81ec3d07dd44919732d8dc31fc Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Fri, 11 May 2018 14:52:30 +1000 Subject: [PATCH 016/163] Update index.php thanks stickler-ci ....... Signed-off-by: Rob Gill --- advanced/index.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/advanced/index.php b/advanced/index.php index 041939a7..0f22908f 100644 --- a/advanced/index.php +++ b/advanced/index.php @@ -103,8 +103,9 @@ $bpAskAdmin = !empty($svEmail) ? '/etc/pihole/! Please update gravity by running pihole -g, or repair Pi-hole using pihole -r."); +} // Set location of adlists file if (is_file("/etc/pihole/adlists.list")) { From 9379487942ec0f9f733ac18de7b9a215503190ca Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Sat, 12 May 2018 10:49:01 +1000 Subject: [PATCH 017/163] changes as requested changes as requested Signed-off-by: Rob Gill --- advanced/index.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/index.php b/advanced/index.php index 0f22908f..5503d604 100644 --- a/advanced/index.php +++ b/advanced/index.php @@ -103,7 +103,7 @@ $bpAskAdmin = !empty($svEmail) ? '/etc/pihole/! Please update gravity by running pihole -g, or repair Pi-hole using pihole -r."); } From 382c19024f6d7927a3684e9a77f7ae3e4252fdf6 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Sat, 12 May 2018 10:53:44 +1000 Subject: [PATCH 018/163] oh stickler bot... accidentally a space Signed-off-by: Rob Gill --- advanced/index.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/index.php b/advanced/index.php index 5503d604..1575bafc 100644 --- a/advanced/index.php +++ b/advanced/index.php @@ -103,7 +103,7 @@ $bpAskAdmin = !empty($svEmail) ? '/etc/pihole/! Please update gravity by running pihole -g, or repair Pi-hole using pihole -r."); } From c1ecfbfe63961850f5d8f311fe9be2db08321de4 Mon Sep 17 00:00:00 2001 From: bcambl Date: Sat, 12 May 2018 19:39:17 -0600 Subject: [PATCH 019/163] linting: Double quote to prevent globbing and word splitting Signed-off-by: bcambl --- automated install/uninstall.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index b58c3ecb..2bcf66bb 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -60,17 +60,17 @@ if [ -x "$(command -v apt-get)" ]; then dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -c "ok installed" } package_cleanup() { - ${SUDO} ${PKG_MANAGER} -y autoremove - ${SUDO} ${PKG_MANAGER} -y autoclean + "${SUDO} ${PKG_MANAGER}" -y autoremove + "${SUDO} ${PKG_MANAGER}" -y autoclean } elif [ -x "$(command -v rpm)" ]; then # Fedora Family PKG_REMOVE="${PKG_MANAGER} remove -y" package_check() { - rpm -qa | grep ^$1- > /dev/null + rpm -qa | grep "^$1-" > /dev/null } package_cleanup() { - ${SUDO} ${PKG_MANAGER} -y autoremove + "${SUDO} ${PKG_MANAGER}" -y autoremove } else echo -e " ${CROSS} OS distribution not supported" From adf2275018ea9e20b11c87d175d32530c35e30ba Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Mon, 14 May 2018 19:38:12 +1000 Subject: [PATCH 020/163] unbind resolved on ubuntu 18.04 Stop systemd-resolved from interfering with dnsmasq/ftl Signed-off-by: Rob Gill --- automated install/basic-install.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 3100ce58..d4984061 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2304,6 +2304,25 @@ main() { fi fi + # resolved and dnsmasq can't share port 53. + # resolved needs to remain in place for installer to download needed files + # so this change needs to be made after installation is complete, but before resarting dnsmasq/ftl + + # Check if running ubuntu 18.04 bionic beaver, which ships with resolved active on port 53 + # (This check may need to be broadened for other systems running resolved?) + if ( lsb_release -a | grep 'Ubuntu 18.04' &> /dev/null ); then + # Running ubuntu 18.04, so check if resolved is running, + if (systemctl is-enabled systemd-resolved | grep -c 'enabled' || true); then + # if resolveconf is running unbind it from port 53 + # Note that this breaks dns functionality on host until dnsmasq/ftl are up and running + echo -e "Unbinding resolved from port 53" + # Make a backup of the original /etc/systemd/resolveconf.d + # (This will need to be restored on uninstallation) + sed -i.orig 's/#DNSStubListener=yes/DNSStubListener=no/g' /etc/systemd/resolved.conf + systemctl restart systemd-resolved + fi + fi + # Enable FTL start_service pihole-FTL enable_service pihole-FTL From 3d3e7a330cf8809bd53254eb5a5a660fdb3f8053 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Mon, 14 May 2018 19:44:23 +1000 Subject: [PATCH 021/163] restore resolvd.conf If dnsmasq is removed, resolved will need to be restored. Signed-off-by: Rob Gill --- automated install/uninstall.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 2f4c58a1..68eeec45 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -165,6 +165,13 @@ removeNoPurge() { ${SUDO} rm -f /etc/sudoers.d/pihole &> /dev/null echo -e " ${TICK} Removed config files" + # Restore resolved + if [[ -e /etc/systemd/resolved.conf.orig ]]; then + systemctl disable systemd-resolved + cp /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf + systemctl enable systemd-resolved + fi + # Remove FTL if command -v pihole-FTL &> /dev/null; then echo -ne " ${INFO} Removing pihole-FTL..." From 97809277df1ebe3f8a6546c9c4f0de5e75508c33 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Mon, 14 May 2018 19:50:58 +1000 Subject: [PATCH 022/163] Update uninstall.sh Signed-off-by: Rob Gill > --- automated install/uninstall.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 68eeec45..b339ed2c 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -168,7 +168,7 @@ removeNoPurge() { # Restore resolved if [[ -e /etc/systemd/resolved.conf.orig ]]; then systemctl disable systemd-resolved - cp /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf + ${SUDO} cp /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf systemctl enable systemd-resolved fi From 2f24e5ceb732d75d1c70967c9a3abe19a73f3fab Mon Sep 17 00:00:00 2001 From: RamSet Date: Mon, 14 May 2018 12:21:20 -0600 Subject: [PATCH 023/163] Minor correction for double instance of the word "found". Signed-off-by: RamSet --- pihole | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pihole b/pihole index cb75861b..b3f532af 100755 --- a/pihole +++ b/pihole @@ -232,7 +232,7 @@ Options: # Handle notices if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then - echo -e " ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} found within block lists" + echo -e " ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} within the block lists" exit 0 elif [[ -z "${results[*]}" ]]; then # Result found in WL/BL/Wildcards From b89a78ce1787dd917e604cbc6372473aacd36d4c Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Tue, 15 May 2018 19:42:18 +1000 Subject: [PATCH 024/163] message text Signed-off-by: Rob Gill > --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index d4984061..f5d17b04 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2315,7 +2315,7 @@ main() { if (systemctl is-enabled systemd-resolved | grep -c 'enabled' || true); then # if resolveconf is running unbind it from port 53 # Note that this breaks dns functionality on host until dnsmasq/ftl are up and running - echo -e "Unbinding resolved from port 53" + echo -e "Disabling systemd-resolved DNSStubListener" # Make a backup of the original /etc/systemd/resolveconf.d # (This will need to be restored on uninstallation) sed -i.orig 's/#DNSStubListener=yes/DNSStubListener=no/g' /etc/systemd/resolved.conf From 25d0e125e58a3a921911be0257a9651a9606e459 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Tue, 15 May 2018 20:23:36 +1000 Subject: [PATCH 025/163] relocate as function The check for systemd-resolved DNSStubListener, and disabling as necessary is a new function, called just prior to start_service pihole-FTL. The check for ubuntu bionic 18.04 specifically is removed. The check if resolved is enabled is made with check_service_active() An additional check that the dnsstublistener is enabled is made. Signed-off-by: Rob Gill --- automated install/basic-install.sh | 39 ++++++++++++++++-------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f5d17b04..155e9f90 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1265,6 +1265,25 @@ check_service_active() { fi } +# Systemd-resolved's DNSStubListener and dnsmasq can't share port 53. +# Resolved needs to remain in place for installer to download needed files +# so this change needs to be made after installation is complete, but before resarting dnsmasq/ftl +disable_resolved_stublistener() { + # Check if Systemd-resolved's DNSSTub listener is enabled and active on port 53 + if check_service_active "systemd-resolved"; then + # Check if DNSStubListener is enabled + if ( grep '#DNSStubListener=yes' /etc/systemd/resolved.conf &> /dev/null ); then + # Disable the DNSStubListener to unbind it from port 53 + # Note that this breaks dns functionality on host until dnsmasq/ftl are up and running + echo -e "Disabling systemd-resolved DNSStubListener" + # Make a backup of the original /etc/systemd/resolved.conf + # (This will need to be restored on uninstallation) + sed -i.orig 's/#DNSStubListener=yes/DNSStubListener=no/g' /etc/systemd/resolved.conf + systemctl restart systemd-resolved + fi + fi +} + update_package_cache() { # Running apt-get update/upgrade with minimal output can cause some issues with # requiring user input (e.g password for phpmyadmin see #218) @@ -2304,24 +2323,8 @@ main() { fi fi - # resolved and dnsmasq can't share port 53. - # resolved needs to remain in place for installer to download needed files - # so this change needs to be made after installation is complete, but before resarting dnsmasq/ftl - - # Check if running ubuntu 18.04 bionic beaver, which ships with resolved active on port 53 - # (This check may need to be broadened for other systems running resolved?) - if ( lsb_release -a | grep 'Ubuntu 18.04' &> /dev/null ); then - # Running ubuntu 18.04, so check if resolved is running, - if (systemctl is-enabled systemd-resolved | grep -c 'enabled' || true); then - # if resolveconf is running unbind it from port 53 - # Note that this breaks dns functionality on host until dnsmasq/ftl are up and running - echo -e "Disabling systemd-resolved DNSStubListener" - # Make a backup of the original /etc/systemd/resolveconf.d - # (This will need to be restored on uninstallation) - sed -i.orig 's/#DNSStubListener=yes/DNSStubListener=no/g' /etc/systemd/resolved.conf - systemctl restart systemd-resolved - fi - fi + # Check for and if necessary disable systemd-resolved-DNSStubListener + disable_resolved_stublistener # Enable FTL start_service pihole-FTL From a431c829cbbecfbcfb6036047776fc6f4df73cd9 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Wed, 16 May 2018 07:11:23 +1000 Subject: [PATCH 026/163] Regex & case fix grep & sed regexes match commented or uncommented Signed-off-by: Rob Gill --- automated install/basic-install.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 155e9f90..d9efa280 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1269,16 +1269,16 @@ check_service_active() { # Resolved needs to remain in place for installer to download needed files # so this change needs to be made after installation is complete, but before resarting dnsmasq/ftl disable_resolved_stublistener() { - # Check if Systemd-resolved's DNSSTub listener is enabled and active on port 53 + # Check if Systemd-resolved's DNSStubListener is enabled and active on port 53 if check_service_active "systemd-resolved"; then # Check if DNSStubListener is enabled - if ( grep '#DNSStubListener=yes' /etc/systemd/resolved.conf &> /dev/null ); then + if ( grep -E '#?DNSStubListener=yes' /etc/systemd/resolved.conf &> /dev/null ); then # Disable the DNSStubListener to unbind it from port 53 # Note that this breaks dns functionality on host until dnsmasq/ftl are up and running echo -e "Disabling systemd-resolved DNSStubListener" # Make a backup of the original /etc/systemd/resolved.conf # (This will need to be restored on uninstallation) - sed -i.orig 's/#DNSStubListener=yes/DNSStubListener=no/g' /etc/systemd/resolved.conf + sed -r -i.orig 's/#?DNSStubListener=yes/DNSStubListener=no/g' /etc/systemd/resolved.conf systemctl restart systemd-resolved fi fi From c400b914e5d46765a8613c1745ad67778b753a77 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Wed, 16 May 2018 07:28:32 +1000 Subject: [PATCH 027/163] Update basic-install.sh Force reloading of relsolved config where available Signed-off-by: Rob Gill --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index d9efa280..47462723 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1279,7 +1279,7 @@ disable_resolved_stublistener() { # Make a backup of the original /etc/systemd/resolved.conf # (This will need to be restored on uninstallation) sed -r -i.orig 's/#?DNSStubListener=yes/DNSStubListener=no/g' /etc/systemd/resolved.conf - systemctl restart systemd-resolved + systemctl reload-or-restart systemd-resolved fi fi } From 15f0ba839f5f4979d3cc43397dfc4090910cc198 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Wed, 16 May 2018 07:35:22 +1000 Subject: [PATCH 028/163] reload resloved reload resolved config if possible, restart otherwise Signed-off-by: Rob Gill --- automated install/uninstall.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index b339ed2c..70f8eeb3 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -167,9 +167,8 @@ removeNoPurge() { # Restore resolved if [[ -e /etc/systemd/resolved.conf.orig ]]; then - systemctl disable systemd-resolved ${SUDO} cp /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf - systemctl enable systemd-resolved + systemctl reload-or-restart systemd-resolved fi # Remove FTL From 8cfe89604af7d699b75b29fbede39f789636ffb8 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Thu, 17 May 2018 07:44:07 +1000 Subject: [PATCH 029/163] user-facing messages Signed-off-by: Rob Gill --- automated install/basic-install.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 47462723..07a4adc1 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1266,21 +1266,28 @@ check_service_active() { } # Systemd-resolved's DNSStubListener and dnsmasq can't share port 53. -# Resolved needs to remain in place for installer to download needed files +# Resolved needs to remain in place for installer to download needed files, # so this change needs to be made after installation is complete, but before resarting dnsmasq/ftl disable_resolved_stublistener() { + echo -en " ${INFO} Testing if systemd-resolved is enabled" # Check if Systemd-resolved's DNSStubListener is enabled and active on port 53 if check_service_active "systemd-resolved"; then # Check if DNSStubListener is enabled + echo -en " ${OVER} ${INFO} Testing if systemd-resolved DNSStub-Listener is active" if ( grep -E '#?DNSStubListener=yes' /etc/systemd/resolved.conf &> /dev/null ); then # Disable the DNSStubListener to unbind it from port 53 # Note that this breaks dns functionality on host until dnsmasq/ftl are up and running - echo -e "Disabling systemd-resolved DNSStubListener" + echo -en "${OVER} ${TICK} Disabling systemd-resolved DNSStubListener" # Make a backup of the original /etc/systemd/resolved.conf # (This will need to be restored on uninstallation) sed -r -i.orig 's/#?DNSStubListener=yes/DNSStubListener=no/g' /etc/systemd/resolved.conf + echo -e " and restarting systemd-resolved" systemctl reload-or-restart systemd-resolved + else + echo -e "${OVER} ${INFO} Systemd-resolved does not need to be restarted" fi + else + echo -e "${OVER} ${INFO} Systemd-resolved is not enabled" fi } From 20ccb7b5589bc7f1b2f225995f4c9f0c3bf385aa Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Mon, 21 May 2018 08:16:53 +1000 Subject: [PATCH 030/163] move & clarify comments --- automated install/basic-install.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 07a4adc1..3a8fe928 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1266,8 +1266,6 @@ check_service_active() { } # Systemd-resolved's DNSStubListener and dnsmasq can't share port 53. -# Resolved needs to remain in place for installer to download needed files, -# so this change needs to be made after installation is complete, but before resarting dnsmasq/ftl disable_resolved_stublistener() { echo -en " ${INFO} Testing if systemd-resolved is enabled" # Check if Systemd-resolved's DNSStubListener is enabled and active on port 53 @@ -2316,8 +2314,11 @@ main() { fi fi - echo -e " ${INFO} Restarting services..." - # Start services + # Check for and disable systemd-resolved-DNSStubListener before reloading resolved + # DNSStubListener needs to remain in place for installer to download needed files, + # so this change needs to be made after installation is complete, + # but before starting or resarting the dnsmasq or ftl services + disable_resolved_stublistener # If the Web server was installed, if [[ "${INSTALL_WEB_SERVER}" == true ]]; then @@ -2330,8 +2331,8 @@ main() { fi fi - # Check for and if necessary disable systemd-resolved-DNSStubListener - disable_resolved_stublistener + echo -e " ${INFO} Restarting services..." + # Start services # Enable FTL start_service pihole-FTL From 1911c3690d2970cb03741266dd44adf12be0e61c Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Tue, 29 May 2018 17:59:39 +1000 Subject: [PATCH 031/163] Style/Tabs to spaces uninstall.sh contains a mix of tabs and spaces for indentation in different parts of the file. Everywhere that used tabs has been converted to spaces, compatible with the indentation style used in basic_install.sh No code has been altered, only the use of tabs and spaces in indention. Signed-off-by: Rob Gill --- automated install/uninstall.sh | 216 ++++++++++++++++----------------- 1 file changed, 108 insertions(+), 108 deletions(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 2f4c58a1..a077ff44 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -11,29 +11,29 @@ source "/opt/pihole/COL_TABLE" while true; do - read -rp " ${QST} Are you sure you would like to remove ${COL_WHITE}Pi-hole${COL_NC}? [y/N] " yn - case ${yn} in - [Yy]* ) break;; - [Nn]* ) echo -e "\n ${COL_LIGHT_GREEN}Uninstall has been cancelled${COL_NC}"; exit 0;; + read -rp " ${QST} Are you sure you would like to remove ${COL_WHITE}Pi-hole${COL_NC}? [y/N] " yn + case ${yn} in + [Yy]* ) break;; + [Nn]* ) echo -e "\n ${COL_LIGHT_GREEN}Uninstall has been cancelled${COL_NC}"; exit 0;; * ) echo -e "\n ${COL_LIGHT_GREEN}Uninstall has been cancelled${COL_NC}"; exit 0;; - esac + esac done # Must be root to uninstall str="Root user check" if [[ ${EUID} -eq 0 ]]; then - echo -e " ${TICK} ${str}" + echo -e " ${TICK} ${str}" else - # Check if sudo is actually installed - # If it isn't, exit because the uninstall can not complete - if [ -x "$(command -v sudo)" ]; then - export SUDO="sudo" - else + # Check if sudo is actually installed + # If it isn't, exit because the uninstall can not complete + if [ -x "$(command -v sudo)" ]; then + export SUDO="sudo" + else echo -e " ${CROSS} ${str} Script called with non-root privileges The Pi-hole requires elevated privleges to uninstall" - exit 1 - fi + exit 1 + fi fi readonly PI_HOLE_FILES_DIR="/etc/.pihole" @@ -54,115 +54,115 @@ fi # Compatability if [ -x "$(command -v rpm)" ]; then - # Fedora Family - PKG_REMOVE="${PKG_MANAGER} remove -y" - package_check() { - rpm -qa | grep ^$1- > /dev/null - } - package_cleanup() { - ${SUDO} ${PKG_MANAGER} -y autoremove - } + # Fedora Family + PKG_REMOVE="${PKG_MANAGER} remove -y" + package_check() { + rpm -qa | grep ^$1- > /dev/null + } + package_cleanup() { + ${SUDO} ${PKG_MANAGER} -y autoremove + } elif [ -x "$(command -v apt-get)" ]; then - # Debian Family - PKG_REMOVE="${PKG_MANAGER} -y remove --purge" - package_check() { - dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -c "ok installed" - } - package_cleanup() { - ${SUDO} ${PKG_MANAGER} -y autoremove - ${SUDO} ${PKG_MANAGER} -y autoclean - } + # Debian Family + PKG_REMOVE="${PKG_MANAGER} -y remove --purge" + package_check() { + dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -c "ok installed" + } + package_cleanup() { + ${SUDO} ${PKG_MANAGER} -y autoremove + ${SUDO} ${PKG_MANAGER} -y autoclean + } else echo -e " ${CROSS} OS distribution not supported" - exit 1 + exit 1 fi removeAndPurge() { - # Purge dependencies + # Purge dependencies echo "" - for i in "${DEPS[@]}"; do - package_check ${i} > /dev/null - if [[ "$?" -eq 0 ]]; then - while true; do - read -rp " ${QST} Do you wish to remove ${COL_WHITE}${i}${COL_NC} from your system? [Y/N] " yn - case ${yn} in - [Yy]* ) + for i in "${DEPS[@]}"; do + package_check ${i} > /dev/null + if [[ "$?" -eq 0 ]]; then + while true; do + read -rp " ${QST} Do you wish to remove ${COL_WHITE}${i}${COL_NC} from your system? [Y/N] " yn + case ${yn} in + [Yy]* ) echo -ne " ${INFO} Removing ${i}..."; ${SUDO} ${PKG_REMOVE} "${i}" &> /dev/null; echo -e "${OVER} ${INFO} Removed ${i}"; break;; - [Nn]* ) echo -e " ${INFO} Skipped ${i}"; break;; - esac - done - else - echo -e " ${INFO} Package ${i} not installed" - fi - done - - # Remove dnsmasq config files - ${SUDO} rm -f /etc/dnsmasq.conf /etc/dnsmasq.conf.orig /etc/dnsmasq.d/01-pihole.conf &> /dev/null + [Nn]* ) echo -e " ${INFO} Skipped ${i}"; break;; + esac + done + else + echo -e " ${INFO} Package ${i} not installed" + fi + done + + # Remove dnsmasq config files + ${SUDO} rm -f /etc/dnsmasq.conf /etc/dnsmasq.conf.orig /etc/dnsmasq.d/01-pihole.conf &> /dev/null echo -e " ${TICK} Removing dnsmasq config files" - # Take care of any additional package cleaning - echo -ne " ${INFO} Removing & cleaning remaining dependencies..." - package_cleanup &> /dev/null + # Take care of any additional package cleaning + echo -ne " ${INFO} Removing & cleaning remaining dependencies..." + package_cleanup &> /dev/null echo -e "${OVER} ${TICK} Removed & cleaned up remaining dependencies" - # Call removeNoPurge to remove Pi-hole specific files - removeNoPurge + # Call removeNoPurge to remove Pi-hole specific files + removeNoPurge } removeNoPurge() { - # Only web directories/files that are created by Pi-hole should be removed - echo -ne " ${INFO} Removing Web Interface..." - ${SUDO} rm -rf /var/www/html/admin &> /dev/null - ${SUDO} rm -rf /var/www/html/pihole &> /dev/null - ${SUDO} rm -f /var/www/html/index.lighttpd.orig &> /dev/null - - # If the web directory is empty after removing these files, then the parent html folder can be removed. - if [ -d "/var/www/html" ]; then - if [[ ! "$(ls -A /var/www/html)" ]]; then - ${SUDO} rm -rf /var/www/html &> /dev/null - fi - fi + # Only web directories/files that are created by Pi-hole should be removed + echo -ne " ${INFO} Removing Web Interface..." + ${SUDO} rm -rf /var/www/html/admin &> /dev/null + ${SUDO} rm -rf /var/www/html/pihole &> /dev/null + ${SUDO} rm -f /var/www/html/index.lighttpd.orig &> /dev/null + + # If the web directory is empty after removing these files, then the parent html folder can be removed. + if [ -d "/var/www/html" ]; then + if [[ ! "$(ls -A /var/www/html)" ]]; then + ${SUDO} rm -rf /var/www/html &> /dev/null + fi + fi echo -e "${OVER} ${TICK} Removed Web Interface" - # Attempt to preserve backwards compatibility with older versions - # to guarantee no additional changes were made to /etc/crontab after - # the installation of pihole, /etc/crontab.pihole should be permanently - # preserved. - if [[ -f /etc/crontab.orig ]]; then - ${SUDO} mv /etc/crontab /etc/crontab.pihole - ${SUDO} mv /etc/crontab.orig /etc/crontab - ${SUDO} service cron restart + # Attempt to preserve backwards compatibility with older versions + # to guarantee no additional changes were made to /etc/crontab after + # the installation of pihole, /etc/crontab.pihole should be permanently + # preserved. + if [[ -f /etc/crontab.orig ]]; then + ${SUDO} mv /etc/crontab /etc/crontab.pihole + ${SUDO} mv /etc/crontab.orig /etc/crontab + ${SUDO} service cron restart echo -e " ${TICK} Restored the default system cron" - fi + fi - # Attempt to preserve backwards compatibility with older versions - if [[ -f /etc/cron.d/pihole ]];then - ${SUDO} rm -f /etc/cron.d/pihole &> /dev/null + # Attempt to preserve backwards compatibility with older versions + if [[ -f /etc/cron.d/pihole ]];then + ${SUDO} rm -f /etc/cron.d/pihole &> /dev/null echo -e " ${TICK} Removed /etc/cron.d/pihole" - fi - - package_check lighttpd > /dev/null - if [[ $? -eq 1 ]]; then - ${SUDO} rm -rf /etc/lighttpd/ &> /dev/null - echo -e " ${TICK} Removed lighttpd" - else - if [ -f /etc/lighttpd/lighttpd.conf.orig ]; then - ${SUDO} mv /etc/lighttpd/lighttpd.conf.orig /etc/lighttpd/lighttpd.conf - fi - fi - - ${SUDO} rm -f /etc/dnsmasq.d/adList.conf &> /dev/null - ${SUDO} rm -f /etc/dnsmasq.d/01-pihole.conf &> /dev/null - ${SUDO} rm -rf /var/log/*pihole* &> /dev/null - ${SUDO} rm -rf /etc/pihole/ &> /dev/null - ${SUDO} rm -rf /etc/.pihole/ &> /dev/null - ${SUDO} rm -rf /opt/pihole/ &> /dev/null - ${SUDO} rm -f /usr/local/bin/pihole &> /dev/null - ${SUDO} rm -f /etc/bash_completion.d/pihole &> /dev/null - ${SUDO} rm -f /etc/sudoers.d/pihole &> /dev/null + fi + + package_check lighttpd > /dev/null + if [[ $? -eq 1 ]]; then + ${SUDO} rm -rf /etc/lighttpd/ &> /dev/null + echo -e " ${TICK} Removed lighttpd" + else + if [ -f /etc/lighttpd/lighttpd.conf.orig ]; then + ${SUDO} mv /etc/lighttpd/lighttpd.conf.orig /etc/lighttpd/lighttpd.conf + fi + fi + + ${SUDO} rm -f /etc/dnsmasq.d/adList.conf &> /dev/null + ${SUDO} rm -f /etc/dnsmasq.d/01-pihole.conf &> /dev/null + ${SUDO} rm -rf /var/log/*pihole* &> /dev/null + ${SUDO} rm -rf /etc/pihole/ &> /dev/null + ${SUDO} rm -rf /etc/.pihole/ &> /dev/null + ${SUDO} rm -rf /opt/pihole/ &> /dev/null + ${SUDO} rm -f /usr/local/bin/pihole &> /dev/null + ${SUDO} rm -f /etc/bash_completion.d/pihole &> /dev/null + ${SUDO} rm -f /etc/sudoers.d/pihole &> /dev/null echo -e " ${TICK} Removed config files" # Remove FTL @@ -180,15 +180,15 @@ removeNoPurge() { echo -e "${OVER} ${TICK} Removed pihole-FTL" fi - # If the pihole user exists, then remove - if id "pihole" &> /dev/null; then - ${SUDO} userdel -r pihole 2> /dev/null + # If the pihole user exists, then remove + if id "pihole" &> /dev/null; then + ${SUDO} userdel -r pihole 2> /dev/null if [[ "$?" -eq 0 ]]; then echo -e " ${TICK} Removed 'pihole' user" else echo -e " ${CROSS} Unable to remove 'pihole' user" fi - fi + fi echo -e "\n We're sorry to see you go, but thanks for checking out Pi-hole! If you need help, reach out to us on Github, Discourse, Reddit or Twitter @@ -211,10 +211,10 @@ while true; do echo -n "${i} " done echo "${COL_NC}" - read -rp " ${QST} Do you wish to go through each dependency for removal? (Choosing No will leave all dependencies installed) [Y/n] " yn - case ${yn} in - [Yy]* ) removeAndPurge; break;; - [Nn]* ) removeNoPurge; break;; + read -rp " ${QST} Do you wish to go through each dependency for removal? (Choosing No will leave all dependencies installed) [Y/n] " yn + case ${yn} in + [Yy]* ) removeAndPurge; break;; + [Nn]* ) removeNoPurge; break;; * ) removeAndPurge; break;; - esac + esac done From ebeab06710cb8953bd14ed80d026e1c2b78f167b Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Tue, 29 May 2018 18:59:55 +1000 Subject: [PATCH 032/163] Appease Stickler-bot So... originally no changes were made to the code, but Stickler-bot was unimpressed, so I've followed its suggestions. Signed-off-by: Rob Gill --- automated install/uninstall.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index a077ff44..7d4782ba 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -57,10 +57,10 @@ if [ -x "$(command -v rpm)" ]; then # Fedora Family PKG_REMOVE="${PKG_MANAGER} remove -y" package_check() { - rpm -qa | grep ^$1- > /dev/null + rpm -qa | grep "^$1-" > /dev/null } package_cleanup() { - ${SUDO} ${PKG_MANAGER} -y autoremove + "${SUDO}" "${PKG_MANAGER}" -y autoremove } elif [ -x "$(command -v apt-get)" ]; then # Debian Family @@ -69,8 +69,8 @@ elif [ -x "$(command -v apt-get)" ]; then dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -c "ok installed" } package_cleanup() { - ${SUDO} ${PKG_MANAGER} -y autoremove - ${SUDO} ${PKG_MANAGER} -y autoclean + "${SUDO}" "${PKG_MANAGER}" -y autoremove + "${SUDO}" "${PKG_MANAGER}" -y autoclean } else echo -e " ${CROSS} OS distribution not supported" @@ -81,8 +81,7 @@ removeAndPurge() { # Purge dependencies echo "" for i in "${DEPS[@]}"; do - package_check ${i} > /dev/null - if [[ "$?" -eq 0 ]]; then + if package_check "${i}" > /dev/null; then while true; do read -rp " ${QST} Do you wish to remove ${COL_WHITE}${i}${COL_NC} from your system? [Y/N] " yn case ${yn} in From e6893bc419165e5a9d091b72dbe118e761eb84fd Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Tue, 29 May 2018 19:03:00 +1000 Subject: [PATCH 033/163] Update uninstall.sh Signed-off-by: Rob Gill --- automated install/uninstall.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 7d4782ba..d6bde208 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -14,8 +14,8 @@ while true; do read -rp " ${QST} Are you sure you would like to remove ${COL_WHITE}Pi-hole${COL_NC}? [y/N] " yn case ${yn} in [Yy]* ) break;; - [Nn]* ) echo -e "\n ${COL_LIGHT_GREEN}Uninstall has been cancelled${COL_NC}"; exit 0;; - * ) echo -e "\n ${COL_LIGHT_GREEN}Uninstall has been cancelled${COL_NC}"; exit 0;; + [Nn]* ) echo -e "${OVER} ${COL_LIGHT_GREEN}Uninstall has been cancelled${COL_NC}"; exit 0;; + * ) echo -e "${OVER} ${COL_LIGHT_GREEN}Uninstall has been cancelled${COL_NC}"; exit 0;; esac done From ff71379a8e5e88f35eda0c24b5d4c2c67bb3c778 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Thu, 31 May 2018 13:14:18 +1000 Subject: [PATCH 034/163] Convert from two to four space indetation Signed-Off-By: Rob Gill --- automated install/uninstall.sh | 312 ++++++++++++++++----------------- 1 file changed, 155 insertions(+), 157 deletions(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index d6bde208..00c28a3a 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -11,29 +11,29 @@ source "/opt/pihole/COL_TABLE" while true; do - read -rp " ${QST} Are you sure you would like to remove ${COL_WHITE}Pi-hole${COL_NC}? [y/N] " yn - case ${yn} in - [Yy]* ) break;; - [Nn]* ) echo -e "${OVER} ${COL_LIGHT_GREEN}Uninstall has been cancelled${COL_NC}"; exit 0;; - * ) echo -e "${OVER} ${COL_LIGHT_GREEN}Uninstall has been cancelled${COL_NC}"; exit 0;; - esac + read -rp " ${QST} Are you sure you would like to remove ${COL_WHITE}Pi-hole${COL_NC}? [y/N] " yn + case ${yn} in + [Yy]* ) break;; + [Nn]* ) echo -e "${OVER} ${COL_LIGHT_GREEN}Uninstall has been cancelled${COL_NC}"; exit 0;; + * ) echo -e "${OVER} ${COL_LIGHT_GREEN}Uninstall has been cancelled${COL_NC}"; exit 0;; + esac done # Must be root to uninstall str="Root user check" if [[ ${EUID} -eq 0 ]]; then - echo -e " ${TICK} ${str}" + echo -e " ${TICK} ${str}" else - # Check if sudo is actually installed - # If it isn't, exit because the uninstall can not complete - if [ -x "$(command -v sudo)" ]; then - export SUDO="sudo" - else - echo -e " ${CROSS} ${str} - Script called with non-root privileges - The Pi-hole requires elevated privleges to uninstall" - exit 1 - fi + # Check if sudo is actually installed + # If it isn't, exit because the uninstall can not complete + if [ -x "$(command -v sudo)" ]; then + export SUDO="sudo" + else + echo -e " ${CROSS} ${str} + Script called with non-root privileges + The Pi-hole requires elevated privleges to uninstall" + exit 1 + fi fi readonly PI_HOLE_FILES_DIR="/etc/.pihole" @@ -48,172 +48,170 @@ distro_check # Install packages used by the Pi-hole DEPS=("${INSTALLER_DEPS[@]}" "${PIHOLE_DEPS[@]}") if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - # Install the Web dependencies - DEPS+=("${PIHOLE_WEB_DEPS[@]}") + # Install the Web dependencies + DEPS+=("${PIHOLE_WEB_DEPS[@]}") fi # Compatability if [ -x "$(command -v rpm)" ]; then - # Fedora Family - PKG_REMOVE="${PKG_MANAGER} remove -y" - package_check() { - rpm -qa | grep "^$1-" > /dev/null - } - package_cleanup() { - "${SUDO}" "${PKG_MANAGER}" -y autoremove - } + # Fedora Family + PKG_REMOVE="${PKG_MANAGER} remove -y" + package_check() { + rpm -qa | grep "^$1-" > /dev/null + } + package_cleanup() { + "${SUDO}" "${PKG_MANAGER}" -y autoremove + } elif [ -x "$(command -v apt-get)" ]; then - # Debian Family - PKG_REMOVE="${PKG_MANAGER} -y remove --purge" - package_check() { - dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -c "ok installed" - } - package_cleanup() { - "${SUDO}" "${PKG_MANAGER}" -y autoremove - "${SUDO}" "${PKG_MANAGER}" -y autoclean - } + # Debian Family + PKG_REMOVE="${PKG_MANAGER} -y remove --purge" + package_check() { + dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -c "ok installed" + } + package_cleanup() { + "${SUDO}" "${PKG_MANAGER}" -y autoremove + "${SUDO}" "${PKG_MANAGER}" -y autoclean + } else - echo -e " ${CROSS} OS distribution not supported" - exit 1 + echo -e " ${CROSS} OS distribution not supported" + exit 1 fi removeAndPurge() { - # Purge dependencies - echo "" - for i in "${DEPS[@]}"; do - if package_check "${i}" > /dev/null; then - while true; do - read -rp " ${QST} Do you wish to remove ${COL_WHITE}${i}${COL_NC} from your system? [Y/N] " yn - case ${yn} in - [Yy]* ) - echo -ne " ${INFO} Removing ${i}..."; - ${SUDO} ${PKG_REMOVE} "${i}" &> /dev/null; - echo -e "${OVER} ${INFO} Removed ${i}"; - break;; - [Nn]* ) echo -e " ${INFO} Skipped ${i}"; break;; - esac - done - else - echo -e " ${INFO} Package ${i} not installed" - fi - done - - # Remove dnsmasq config files - ${SUDO} rm -f /etc/dnsmasq.conf /etc/dnsmasq.conf.orig /etc/dnsmasq.d/01-pihole.conf &> /dev/null - echo -e " ${TICK} Removing dnsmasq config files" - - # Take care of any additional package cleaning - echo -ne " ${INFO} Removing & cleaning remaining dependencies..." - package_cleanup &> /dev/null - echo -e "${OVER} ${TICK} Removed & cleaned up remaining dependencies" - - # Call removeNoPurge to remove Pi-hole specific files - removeNoPurge + # Purge dependencies + echo "" + for i in "${DEPS[@]}"; do + if package_check "${i}" > /dev/null; then + while true; do + read -rp " ${QST} Do you wish to remove ${COL_WHITE}${i}${COL_NC} from your system? [Y/N] " yn + case ${yn} in + [Yy]* ) + echo -ne " ${INFO} Removing ${i}..."; + ${SUDO} ${PKG_REMOVE} "${i}" &> /dev/null; + echo -e "${OVER} ${INFO} Removed ${i}"; + break;; + [Nn]* ) echo -e " ${INFO} Skipped ${i}"; break;; + esac + done + else + echo -e " ${INFO} Package ${i} not installed" + fi + done + + # Remove dnsmasq config files + ${SUDO} rm -f /etc/dnsmasq.conf /etc/dnsmasq.conf.orig /etc/dnsmasq.d/01-pihole.conf &> /dev/null + echo -e " ${TICK} Removing dnsmasq config files" + + # Take care of any additional package cleaning + echo -ne " ${INFO} Removing & cleaning remaining dependencies..." + package_cleanup &> /dev/null + echo -e "${OVER} ${TICK} Removed & cleaned up remaining dependencies" + + # Call removeNoPurge to remove Pi-hole specific files + removeNoPurge } removeNoPurge() { - # Only web directories/files that are created by Pi-hole should be removed - echo -ne " ${INFO} Removing Web Interface..." - ${SUDO} rm -rf /var/www/html/admin &> /dev/null - ${SUDO} rm -rf /var/www/html/pihole &> /dev/null - ${SUDO} rm -f /var/www/html/index.lighttpd.orig &> /dev/null - - # If the web directory is empty after removing these files, then the parent html folder can be removed. - if [ -d "/var/www/html" ]; then - if [[ ! "$(ls -A /var/www/html)" ]]; then - ${SUDO} rm -rf /var/www/html &> /dev/null + # Only web directories/files that are created by Pi-hole should be removed + echo -ne " ${INFO} Removing Web Interface..." + ${SUDO} rm -rf /var/www/html/admin &> /dev/null + ${SUDO} rm -rf /var/www/html/pihole &> /dev/null + ${SUDO} rm -f /var/www/html/index.lighttpd.orig &> /dev/null + + # If the web directory is empty after removing these files, then the parent html folder can be removed. + if [ -d "/var/www/html" ]; then + if [[ ! "$(ls -A /var/www/html)" ]]; then + ${SUDO} rm -rf /var/www/html &> /dev/null + fi fi - fi - echo -e "${OVER} ${TICK} Removed Web Interface" - - # Attempt to preserve backwards compatibility with older versions - # to guarantee no additional changes were made to /etc/crontab after - # the installation of pihole, /etc/crontab.pihole should be permanently - # preserved. - if [[ -f /etc/crontab.orig ]]; then - ${SUDO} mv /etc/crontab /etc/crontab.pihole - ${SUDO} mv /etc/crontab.orig /etc/crontab - ${SUDO} service cron restart - echo -e " ${TICK} Restored the default system cron" - fi - - # Attempt to preserve backwards compatibility with older versions - if [[ -f /etc/cron.d/pihole ]];then - ${SUDO} rm -f /etc/cron.d/pihole &> /dev/null - echo -e " ${TICK} Removed /etc/cron.d/pihole" - fi - - package_check lighttpd > /dev/null - if [[ $? -eq 1 ]]; then - ${SUDO} rm -rf /etc/lighttpd/ &> /dev/null - echo -e " ${TICK} Removed lighttpd" - else - if [ -f /etc/lighttpd/lighttpd.conf.orig ]; then - ${SUDO} mv /etc/lighttpd/lighttpd.conf.orig /etc/lighttpd/lighttpd.conf + echo -e "${OVER} ${TICK} Removed Web Interface" + + # Attempt to preserve backwards compatibility with older versions + # to guarantee no additional changes were made to /etc/crontab after + # the installation of pihole, /etc/crontab.pihole should be permanently + # preserved. + if [[ -f /etc/crontab.orig ]]; then + ${SUDO} mv /etc/crontab /etc/crontab.pihole + ${SUDO} mv /etc/crontab.orig /etc/crontab + ${SUDO} service cron restart + echo -e " ${TICK} Restored the default system cron" + fi + + # Attempt to preserve backwards compatibility with older versions + if [[ -f /etc/cron.d/pihole ]];then + ${SUDO} rm -f /etc/cron.d/pihole &> /dev/null + echo -e " ${TICK} Removed /etc/cron.d/pihole" fi - fi - - ${SUDO} rm -f /etc/dnsmasq.d/adList.conf &> /dev/null - ${SUDO} rm -f /etc/dnsmasq.d/01-pihole.conf &> /dev/null - ${SUDO} rm -rf /var/log/*pihole* &> /dev/null - ${SUDO} rm -rf /etc/pihole/ &> /dev/null - ${SUDO} rm -rf /etc/.pihole/ &> /dev/null - ${SUDO} rm -rf /opt/pihole/ &> /dev/null - ${SUDO} rm -f /usr/local/bin/pihole &> /dev/null - ${SUDO} rm -f /etc/bash_completion.d/pihole &> /dev/null - ${SUDO} rm -f /etc/sudoers.d/pihole &> /dev/null - echo -e " ${TICK} Removed config files" - - # Remove FTL - if command -v pihole-FTL &> /dev/null; then - echo -ne " ${INFO} Removing pihole-FTL..." - - if [[ -x "$(command -v systemctl)" ]]; then - systemctl stop pihole-FTL + + package_check lighttpd > /dev/null + if [[ $? -eq 1 ]]; then + ${SUDO} rm -rf /etc/lighttpd/ &> /dev/null + echo -e " ${TICK} Removed lighttpd" else - service pihole-FTL stop + if [ -f /etc/lighttpd/lighttpd.conf.orig ]; then + ${SUDO} mv /etc/lighttpd/lighttpd.conf.orig /etc/lighttpd/lighttpd.conf + fi fi - ${SUDO} rm -f /etc/init.d/pihole-FTL - ${SUDO} rm -f /usr/bin/pihole-FTL - echo -e "${OVER} ${TICK} Removed pihole-FTL" - fi + ${SUDO} rm -f /etc/dnsmasq.d/adList.conf &> /dev/null + ${SUDO} rm -f /etc/dnsmasq.d/01-pihole.conf &> /dev/null + ${SUDO} rm -rf /var/log/*pihole* &> /dev/null + ${SUDO} rm -rf /etc/pihole/ &> /dev/null + ${SUDO} rm -rf /etc/.pihole/ &> /dev/null + ${SUDO} rm -rf /opt/pihole/ &> /dev/null + ${SUDO} rm -f /usr/local/bin/pihole &> /dev/null + ${SUDO} rm -f /etc/bash_completion.d/pihole &> /dev/null + ${SUDO} rm -f /etc/sudoers.d/pihole &> /dev/null + echo -e " ${TICK} Removed config files" + + # Remove FTL + if command -v pihole-FTL &> /dev/null; then + echo -ne " ${INFO} Removing pihole-FTL..." + if [[ -x "$(command -v systemctl)" ]]; then + systemctl stop pihole-FTL + else + service pihole-FTL stop + fi + ${SUDO} rm -f /etc/init.d/pihole-FTL + ${SUDO} rm -f /usr/bin/pihole-FTL + echo -e "${OVER} ${TICK} Removed pihole-FTL" + fi - # If the pihole user exists, then remove - if id "pihole" &> /dev/null; then - ${SUDO} userdel -r pihole 2> /dev/null - if [[ "$?" -eq 0 ]]; then - echo -e " ${TICK} Removed 'pihole' user" - else - echo -e " ${CROSS} Unable to remove 'pihole' user" + # If the pihole user exists, then remove + if id "pihole" &> /dev/null; then + ${SUDO} userdel -r pihole 2> /dev/null + if [[ "$?" -eq 0 ]]; then + echo -e " ${TICK} Removed 'pihole' user" + else + echo -e " ${CROSS} Unable to remove 'pihole' user" + fi fi - fi - echo -e "\n We're sorry to see you go, but thanks for checking out Pi-hole! - If you need help, reach out to us on Github, Discourse, Reddit or Twitter - Reinstall at any time: ${COL_WHITE}curl -sSL https://install.pi-hole.net | bash${COL_NC} + echo -e "\n We're sorry to see you go, but thanks for checking out Pi-hole! + If you need help, reach out to us on Github, Discourse, Reddit or Twitter + Reinstall at any time: ${COL_WHITE}curl -sSL https://install.pi-hole.net | bash${COL_NC} - ${COL_LIGHT_RED}Please reset the DNS on your router/clients to restore internet connectivity - ${COL_LIGHT_GREEN}Uninstallation Complete! ${COL_NC}" + ${COL_LIGHT_RED}Please reset the DNS on your router/clients to restore internet connectivity + ${COL_LIGHT_GREEN}Uninstallation Complete! ${COL_NC}" } ######### SCRIPT ########### if command -v vcgencmd &> /dev/null; then - echo -e " ${INFO} All dependencies are safe to remove on Raspbian" + echo -e " ${INFO} All dependencies are safe to remove on Raspbian" else - echo -e " ${INFO} Be sure to confirm if any dependencies should not be removed" + echo -e " ${INFO} Be sure to confirm if any dependencies should not be removed" fi while true; do - echo -e " ${INFO} ${COL_YELLOW}The following dependencies may have been added by the Pi-hole install:" - echo -n " " - for i in "${DEPS[@]}"; do - echo -n "${i} " - done - echo "${COL_NC}" - read -rp " ${QST} Do you wish to go through each dependency for removal? (Choosing No will leave all dependencies installed) [Y/n] " yn - case ${yn} in - [Yy]* ) removeAndPurge; break;; - [Nn]* ) removeNoPurge; break;; - * ) removeAndPurge; break;; - esac + echo -e " ${INFO} ${COL_YELLOW}The following dependencies may have been added by the Pi-hole install:" + echo -n " " + for i in "${DEPS[@]}"; do + echo -n "${i} " + done + echo "${COL_NC}" + read -rp " ${QST} Do you wish to go through each dependency for removal? (Choosing No will leave all dependencies installed) [Y/n] " yn + case ${yn} in + [Yy]* ) removeAndPurge; break;; + [Nn]* ) removeNoPurge; break;; + * ) removeAndPurge; break;; + esac done From 552138e85188f5f3f013f322040db3be634d3420 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Thu, 31 May 2018 13:24:09 +1000 Subject: [PATCH 035/163] Appease stickler-bot. Signed-off-by: Rob Gill --- automated install/uninstall.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 00c28a3a..b5a7964b 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -87,7 +87,7 @@ removeAndPurge() { case ${yn} in [Yy]* ) echo -ne " ${INFO} Removing ${i}..."; - ${SUDO} ${PKG_REMOVE} "${i}" &> /dev/null; + ${SUDO} "${PKG_REMOVE} ${i}" &> /dev/null; echo -e "${OVER} ${INFO} Removed ${i}"; break;; [Nn]* ) echo -e " ${INFO} Skipped ${i}"; break;; @@ -179,15 +179,14 @@ removeNoPurge() { # If the pihole user exists, then remove if id "pihole" &> /dev/null; then - ${SUDO} userdel -r pihole 2> /dev/null - if [[ "$?" -eq 0 ]]; then + if ${SUDO} userdel -r pihole 2> /dev/null; then echo -e " ${TICK} Removed 'pihole' user" else echo -e " ${CROSS} Unable to remove 'pihole' user" fi fi - echo -e "\n We're sorry to see you go, but thanks for checking out Pi-hole! + echo -e "\\n We're sorry to see you go, but thanks for checking out Pi-hole! If you need help, reach out to us on Github, Discourse, Reddit or Twitter Reinstall at any time: ${COL_WHITE}curl -sSL https://install.pi-hole.net | bash${COL_NC} From f2bedddce46d87ab2202926cf45d5cd3aaed93f6 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Thu, 31 May 2018 13:37:52 +1000 Subject: [PATCH 036/163] Changes from #2167 converted to spaces Signed-Off-By: Rob Gill --- automated install/uninstall.sh | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index b5a7964b..51e85946 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -53,24 +53,17 @@ if [[ "${INSTALL_WEB_SERVER}" == true ]]; then fi # Compatability -if [ -x "$(command -v rpm)" ]; then - # Fedora Family - PKG_REMOVE="${PKG_MANAGER} remove -y" - package_check() { - rpm -qa | grep "^$1-" > /dev/null - } - package_cleanup() { - "${SUDO}" "${PKG_MANAGER}" -y autoremove - } -elif [ -x "$(command -v apt-get)" ]; then +if [ -x "$(command -v apt-get)" ]; then # Debian Family PKG_REMOVE="${PKG_MANAGER} -y remove --purge" package_check() { dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -c "ok installed" } - package_cleanup() { - "${SUDO}" "${PKG_MANAGER}" -y autoremove - "${SUDO}" "${PKG_MANAGER}" -y autoclean +elif [ -x "$(command -v rpm)" ]; then + # Fedora Family + PKG_REMOVE="${PKG_MANAGER} remove -y" + package_check() { + rpm -qa | grep "^$1-" > /dev/null } else echo -e " ${CROSS} OS distribution not supported" @@ -81,13 +74,14 @@ removeAndPurge() { # Purge dependencies echo "" for i in "${DEPS[@]}"; do - if package_check "${i}" > /dev/null; then + package_check ${i} > /dev/null + if [[ "$?" -eq 0 ]]; then while true; do read -rp " ${QST} Do you wish to remove ${COL_WHITE}${i}${COL_NC} from your system? [Y/N] " yn case ${yn} in [Yy]* ) echo -ne " ${INFO} Removing ${i}..."; - ${SUDO} "${PKG_REMOVE} ${i}" &> /dev/null; + ${SUDO} ${PKG_REMOVE} "${i}" &> /dev/null; echo -e "${OVER} ${INFO} Removed ${i}"; break;; [Nn]* ) echo -e " ${INFO} Skipped ${i}"; break;; @@ -95,18 +89,13 @@ removeAndPurge() { done else echo -e " ${INFO} Package ${i} not installed" - fi + fi done # Remove dnsmasq config files - ${SUDO} rm -f /etc/dnsmasq.conf /etc/dnsmasq.conf.orig /etc/dnsmasq.d/01-pihole.conf &> /dev/null + ${SUDO} rm -f /etc/dnsmasq.conf /etc/dnsmasq.conf.orig /etc/dnsmasq.d/*-pihole*.conf &> /dev/null echo -e " ${TICK} Removing dnsmasq config files" - # Take care of any additional package cleaning - echo -ne " ${INFO} Removing & cleaning remaining dependencies..." - package_cleanup &> /dev/null - echo -e "${OVER} ${TICK} Removed & cleaned up remaining dependencies" - # Call removeNoPurge to remove Pi-hole specific files removeNoPurge } From f0dfa4d53d375f4e4741f2cbb28730e3609a89fa Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Thu, 31 May 2018 13:43:49 +1000 Subject: [PATCH 037/163] Stickler-bot... Signed-Off-By: Rob Gill --- automated install/uninstall.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 51e85946..b8797dbb 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -74,14 +74,13 @@ removeAndPurge() { # Purge dependencies echo "" for i in "${DEPS[@]}"; do - package_check ${i} > /dev/null - if [[ "$?" -eq 0 ]]; then + if package_check "${i}" > /dev/null; then while true; do read -rp " ${QST} Do you wish to remove ${COL_WHITE}${i}${COL_NC} from your system? [Y/N] " yn case ${yn} in [Yy]* ) echo -ne " ${INFO} Removing ${i}..."; - ${SUDO} ${PKG_REMOVE} "${i}" &> /dev/null; + ${SUDO} "${PKG_REMOVE} ${i}" &> /dev/null; echo -e "${OVER} ${INFO} Removed ${i}"; break;; [Nn]* ) echo -e " ${INFO} Skipped ${i}"; break;; From b60a9fa371147d96bb10382aa58c4e47207017fa Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Fri, 1 Jun 2018 07:14:54 +1000 Subject: [PATCH 038/163] merge (#2) * remove package_check to avoid situations like #1760 Signed-off-by: Adam Warner * Prevent redundant entries in to adlists.list Grep ${args[3]} and only add if grep -c -eq 0 Signed-off-by: Ryan Knapper * lan to local Reduced differences. Signed-off-by: Ryan Knapper * Require exact match Updated to require an exact match to reduce false-positives, as suggested by DL6ER. Signed-off-by: Ryan Knapper * fix empty ports on some systems Signed-off-by: Jacob Salmela * debug user locale; improve function to parse variables and files Signed-off-by: Jacob Salmela * Split declaration and population for stickler. Signed-off-by: Dan Schaper * implement dschapers suggestions--better command, less subshells, and finer formatting Signed-off-by: Jacob Salmela * flip uninstall compatability check Signed-off-by: bcambl * Update index.php Avoiding calling empty() on a function allows this to work under PHP5. Making the check for blocklist generation in this way instead is compatible with both PHP5 and PHP7. Signed-off-by: Rob Gill * Update index.php thanks stickler-ci ....... Signed-off-by: Rob Gill * changes as requested changes as requested Signed-off-by: Rob Gill * oh stickler bot... accidentally a space Signed-off-by: Rob Gill * linting: Double quote to prevent globbing and word splitting Signed-off-by: bcambl * unbind resolved on ubuntu 18.04 Stop systemd-resolved from interfering with dnsmasq/ftl Signed-off-by: Rob Gill * restore resolvd.conf If dnsmasq is removed, resolved will need to be restored. Signed-off-by: Rob Gill * Update uninstall.sh Signed-off-by: Rob Gill > * Minor correction for double instance of the word "found". Signed-off-by: RamSet * message text Signed-off-by: Rob Gill > * relocate as function The check for systemd-resolved DNSStubListener, and disabling as necessary is a new function, called just prior to start_service pihole-FTL. The check for ubuntu bionic 18.04 specifically is removed. The check if resolved is enabled is made with check_service_active() An additional check that the dnsstublistener is enabled is made. Signed-off-by: Rob Gill * Regex & case fix grep & sed regexes match commented or uncommented Signed-off-by: Rob Gill * Update basic-install.sh Force reloading of relsolved config where available Signed-off-by: Rob Gill * reload resloved reload resolved config if possible, restart otherwise Signed-off-by: Rob Gill * user-facing messages Signed-off-by: Rob Gill * move & clarify comments --- advanced/Scripts/piholeDebug.sh | 32 ++++++++++++++++++++++------ advanced/Scripts/webpage.sh | 19 +++-------------- advanced/index.php | 4 +++- automated install/basic-install.sh | 34 ++++++++++++++++++++++++++++-- automated install/uninstall.sh | 1 + pihole | 2 +- 6 files changed, 65 insertions(+), 27 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 94ff3fb3..a7fa4c2a 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -493,6 +493,13 @@ parse_setup_vars() { fi } +parse_locale() { + local pihole_locale + echo_current_diagnostic "Locale" + pihole_locale="$(locale)" + parse_file "${pihole_locale}" +} + does_ip_match_setup_vars() { # Check for IPv4 or 6 local protocol="${1}" @@ -652,15 +659,22 @@ check_required_ports() { # Sort the addresses and remove duplicates while IFS= read -r line; do ports_in_use+=( "$line" ) - done < <( lsof -i -P -n | awk -F' ' '/LISTEN/ {print $9, $1}' | sort -n | uniq | cut -d':' -f2 ) + done < <( lsof -iTCP -sTCP:LISTEN -P -n +c 10 ) # Now that we have the values stored, for i in "${!ports_in_use[@]}"; do # loop through them and assign some local variables - local port_number - port_number="$(echo "${ports_in_use[$i]}" | awk '{print $1}')" local service_name - service_name=$(echo "${ports_in_use[$i]}" | awk '{print $2}') + service_name=$(echo "${ports_in_use[$i]}" | awk '{print $1}') + local protocol_type + protocol_type=$(echo "${ports_in_use[$i]}" | awk '{print $5}') + local port_number + port_number="$(echo "${ports_in_use[$i]}" | awk '{print $9}')" + + # Skip the line if it's the titles of the columns the lsof command produces + if [[ "${service_name}" == COMMAND ]]; then + continue + fi # Use a case statement to determine if the right services are using the right ports case "${port_number}" in 53) compare_port_to_service_assigned "${resolver}" @@ -670,7 +684,7 @@ check_required_ports() { 4711) compare_port_to_service_assigned "${ftl}" ;; # If it's not a default port that Pi-hole needs, just print it out for the user to see - *) log_write "[${port_number}] is in use by ${service_name}"; + *) log_write "${port_number} ${service_name} (${protocol_type})"; esac done } @@ -879,8 +893,11 @@ parse_file() { # Put the current Internal Field Separator into another variable so it can be restored later OLD_IFS="$IFS" # Get the lines that are in the file(s) and store them in an array for parsing later - IFS=$'\r\n' command eval 'file_info=( $(cat "${filename}") )' - + if [[ -f "$filename" ]]; then + IFS=$'\r\n' command eval 'file_info=( $(cat "${filename}") )' + else + read -a file_info <<< $filename + fi # Set a named variable for better readability local file_lines # For each line in the file, @@ -1193,6 +1210,7 @@ parse_setup_vars check_x_headers analyze_gravity_list show_content_of_pihole_files +parse_locale analyze_pihole_log copy_to_debug_log upload_to_tricorder diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 8a85839f..48161604 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -204,10 +204,6 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423 add_dnsmasq_setting "interface" "${PIHOLE_INTERFACE}" fi - if [[ "${CONDITIONAL_FORWARDING}" == true ]]; then - add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_DOMAIN}/${CONDITIONAL_FORWARDING_IP}" - add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_REVERSE}/${CONDITIONAL_FORWARDING_IP}" - fi } @@ -237,17 +233,6 @@ SetDNSServers() { else change_setting "DNSSEC" "false" fi - if [[ "${args[6]}" == "conditional_forwarding" ]]; then - change_setting "CONDITIONAL_FORWARDING" "true" - change_setting "CONDITIONAL_FORWARDING_IP" "${args[7]}" - change_setting "CONDITIONAL_FORWARDING_DOMAIN" "${args[8]}" - change_setting "CONDITIONAL_FORWARDING_REVERSE" "${args[9]}" - else - change_setting "CONDITIONAL_FORWARDING" "false" - delete_setting "CONDITIONAL_FORWARDING_IP" - delete_setting "CONDITIONAL_FORWARDING_DOMAIN" - delete_setting "CONDITIONAL_FORWARDING_REVERSE" - fi ProcessDNSSettings @@ -383,7 +368,9 @@ CustomizeAdLists() { elif [[ "${args[2]}" == "disable" ]]; then sed -i "\\@${args[3]}@s/^http/#http/g" "${list}" elif [[ "${args[2]}" == "add" ]]; then - echo "${args[3]}" >> ${list} + if [[ $(grep -c "^${args[3]}$" "${list}") -eq 0 ]] ; then + echo "${args[3]}" >> ${list} + fi elif [[ "${args[2]}" == "del" ]]; then var=$(echo "${args[3]}" | sed 's/\//\\\//g') sed -i "/${var}/Id" "${list}" diff --git a/advanced/index.php b/advanced/index.php index d097fe0f..1575bafc 100644 --- a/advanced/index.php +++ b/advanced/index.php @@ -102,8 +102,10 @@ if ($serverName === "pi.hole") { $bpAskAdmin = !empty($svEmail) ? '' : ""; // Determine if at least one block list has been generated -if (empty(glob("/etc/pihole/list.0.*.domains"))) +$blocklistglob = glob("/etc/pihole/list.0.*.domains"); +if ($blocklistglob === array()) { die("[ERROR] There are no domain lists generated lists within /etc/pihole/! Please update gravity by running pihole -g, or repair Pi-hole using pihole -r."); +} // Set location of adlists file if (is_file("/etc/pihole/adlists.list")) { diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 3100ce58..3a8fe928 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1265,6 +1265,30 @@ check_service_active() { fi } +# Systemd-resolved's DNSStubListener and dnsmasq can't share port 53. +disable_resolved_stublistener() { + echo -en " ${INFO} Testing if systemd-resolved is enabled" + # Check if Systemd-resolved's DNSStubListener is enabled and active on port 53 + if check_service_active "systemd-resolved"; then + # Check if DNSStubListener is enabled + echo -en " ${OVER} ${INFO} Testing if systemd-resolved DNSStub-Listener is active" + if ( grep -E '#?DNSStubListener=yes' /etc/systemd/resolved.conf &> /dev/null ); then + # Disable the DNSStubListener to unbind it from port 53 + # Note that this breaks dns functionality on host until dnsmasq/ftl are up and running + echo -en "${OVER} ${TICK} Disabling systemd-resolved DNSStubListener" + # Make a backup of the original /etc/systemd/resolved.conf + # (This will need to be restored on uninstallation) + sed -r -i.orig 's/#?DNSStubListener=yes/DNSStubListener=no/g' /etc/systemd/resolved.conf + echo -e " and restarting systemd-resolved" + systemctl reload-or-restart systemd-resolved + else + echo -e "${OVER} ${INFO} Systemd-resolved does not need to be restarted" + fi + else + echo -e "${OVER} ${INFO} Systemd-resolved is not enabled" + fi +} + update_package_cache() { # Running apt-get update/upgrade with minimal output can cause some issues with # requiring user input (e.g password for phpmyadmin see #218) @@ -2290,8 +2314,11 @@ main() { fi fi - echo -e " ${INFO} Restarting services..." - # Start services + # Check for and disable systemd-resolved-DNSStubListener before reloading resolved + # DNSStubListener needs to remain in place for installer to download needed files, + # so this change needs to be made after installation is complete, + # but before starting or resarting the dnsmasq or ftl services + disable_resolved_stublistener # If the Web server was installed, if [[ "${INSTALL_WEB_SERVER}" == true ]]; then @@ -2304,6 +2331,9 @@ main() { fi fi + echo -e " ${INFO} Restarting services..." + # Start services + # Enable FTL start_service pihole-FTL enable_service pihole-FTL diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index b8797dbb..44753a59 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -65,6 +65,7 @@ elif [ -x "$(command -v rpm)" ]; then package_check() { rpm -qa | grep "^$1-" > /dev/null } + else echo -e " ${CROSS} OS distribution not supported" exit 1 diff --git a/pihole b/pihole index cb75861b..b3f532af 100755 --- a/pihole +++ b/pihole @@ -232,7 +232,7 @@ Options: # Handle notices if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then - echo -e " ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} found within block lists" + echo -e " ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} within the block lists" exit 0 elif [[ -z "${results[*]}" ]]; then # Result found in WL/BL/Wildcards From f35ea9a3ca65c5cea9c0d9e64a629fc644d23cbe Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Fri, 1 Jun 2018 11:57:59 +1000 Subject: [PATCH 039/163] replace code i missed during merge Signed-off-by: Rob Gill --- automated install/uninstall.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index b8797dbb..301be4e3 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -88,7 +88,7 @@ removeAndPurge() { done else echo -e " ${INFO} Package ${i} not installed" - fi + fi done # Remove dnsmasq config files @@ -154,6 +154,10 @@ removeNoPurge() { # Remove FTL if command -v pihole-FTL &> /dev/null; then + if [[ -e /etc/systemd/resolved.conf.orig ]]; then + ${SUDO} cp /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf + systemctl reload-or-restart systemd-resolved + fi echo -ne " ${INFO} Removing pihole-FTL..." if [[ -x "$(command -v systemctl)" ]]; then systemctl stop pihole-FTL From cadd0e42447869c73aae3efad0358f510d60046d Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Fri, 1 Jun 2018 14:03:13 +1000 Subject: [PATCH 040/163] move code back to correct location Signed-off-by: Rob Gill --- automated install/uninstall.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 301be4e3..0a51fd4a 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -152,12 +152,14 @@ removeNoPurge() { ${SUDO} rm -f /etc/sudoers.d/pihole &> /dev/null echo -e " ${TICK} Removed config files" + # Restore Resolved + if [[ -e /etc/systemd/resolved.conf.orig ]]; then + ${SUDO} cp /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf + systemctl reload-or-restart systemd-resolved + fi + # Remove FTL if command -v pihole-FTL &> /dev/null; then - if [[ -e /etc/systemd/resolved.conf.orig ]]; then - ${SUDO} cp /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf - systemctl reload-or-restart systemd-resolved - fi echo -ne " ${INFO} Removing pihole-FTL..." if [[ -x "$(command -v systemctl)" ]]; then systemctl stop pihole-FTL From 3be19046538442f977b77f5519b4988d6cf1d422 Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Fri, 1 Jun 2018 10:20:40 +0200 Subject: [PATCH 041/163] basic-install.sh: fix "install: invalid user ''" refs #1767 --- automated install/basic-install.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 3152d9c1..97e8194b 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -57,6 +57,10 @@ IPV6_ADDRESS="" QUERY_LOGGING=true INSTALL_WEB=true +if [ -z "${USER}" ]; then + USER="$(id -un)" +fi + # Find the rows and columns will default to 80x24 if it can not be detected screen_size=$(stty size 2>/dev/null || echo 24 80) From ef17f4913b5444965138179443bbfab2bac30fa2 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Sat, 2 Jun 2018 09:39:51 +1000 Subject: [PATCH 042/163] Create pihole.8 Linux man page for pihole Signed-off-by: Rob Gill --- pihole.8 | 326 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 326 insertions(+) create mode 100644 pihole.8 diff --git a/pihole.8 b/pihole.8 new file mode 100644 index 00000000..d62dbf8a --- /dev/null +++ b/pihole.8 @@ -0,0 +1,326 @@ +.TH "Pi-hole" "8" "Pi-hole" "Pi-hole" "May 2018" +.SH "NAME" + +Pi-hole : A black-hole for internet advertisements +.br +.SH "SYNOPSIS" + +\fBpihole\fR (\fB-w\fR|\fB-b\fR|\fB-wild\fR) [options] domain(s) +.br +\fBpihole -a\fR \fB-p\fR password +.br +\fBpihole -a\fR (-c|-f|-k) +.br +\fBpihole -a\fR [\fB-r\fR hostrecord] +.br +\fBpihole -a -e\fR email +.br +\fBpihole -a -i\fR interface +.br +\fBpihole -a -l\fR privacylevel +.br +\fBpihole -c\fR [-j|-r|-e] +.br +\fBpihole\fR \fB-d\fR [-a] +.br +\fBpihole -f +.br +pihole -r +.br +pihole -t +.br +pihole -g\fR +.br +\fBpihole\fR -\fBq\fR [options] +.br +\fBpihole\fR \fB-l\fR (on|off|off noflush) +.br +\fBpihole -up \fR[--checkonly] +.br +\fBpihole uninstall +.br +pihole status +.br +pihole restartdns\fR +.br +\fBpihole\fR (\fBenable\fR|\fBdisable\fR [time]) +.br +\fBpihole\fR \fBcheckout\fR repo [branch] +.br +\fBpihole\fR \fBhelp\fR +.br +.SH "DESCRIPTION" + +Available commands and options: +.br + +\fB-w, whitelist\fR [options] [ ] +.br + Adds or removes specified domain or domains tho the Whitelist +.br + +\fB-b, blacklist\fR [options] [ ] +.br + Adds or removes specified domain or domains to the blacklist +.br + +\fB-wild, wildcard\fR [options] [ ] +.br + Add or removes specified domain, and all subdomains to the blacklist +.br + +.br + (Whitelist/Blacklist manipulation options): +.br + -d, --delmode Remove domain(s) from the list +.br + -nr, --noreload Update list without refreshing dnsmasq +.br + -q, --quiet Make output less verbose +.br + -l, --list Display all your listed domains +.br + --nuke Removes all entries in a list +.br + +\fB-d, debug\fR [\fB-a\fR] +.br + Start a debugging session Add '-a' to enable automated debugging +.br + +\fB-f, flush\fR +.br + Flush the Pi-hole log +.br + +.br +\fB-r, reconfigure\fR +.br + Reconfigure or Repair Pi-hole subsystems +.br + +\fB-t, tail\fR +.br + View the live output of the Pi-hole log +.br + +\fB-a, admin\fR [options] Console +.br + +.br + (Admin console options): +.br + -p, password Set Admin Console password +.br + -c, celsius Set Celsius as preferred temperature unit +.br + -f, fahrenheit Set Fahrenheit as preferred temperature unit +.br + -k, kelvin Set Kelvin as preferred temperature unit +.br + -r, hostrecord Add a name to the DNS associated to an IPv4/IPv6 address +.br + -e, email Set an administrative contact address for the Block Page +.br + -i, interface Specify dnsmasq's interface listening behavior +.br + -l, privacylevel Set privacy level (0 = lowest, 3 = highest) +.br + +.br +\fB-c, chronometer\fR [options] +.br + Calculates stats and displays to an LCD +.br + + (Chronometer Options): +.br + -j, --json Output stats as JSON formatted string +.br + -r, --refresh Set update frequency (in seconds) +.br + -e, --exit Output stats and exit witout refreshing +.br + +\fB-g, updateGravity\fR +.br + Update the list of ad-serving domains +.br + +\fB-q, query\fR [option] +.br + Query the adlists for a specified domain +.br + + (Query options): +.br + -adlist Print the name of the block list URL +.br + -exact Search the block lists for exact domain matches +.br + -all Return all query matches within a block list +.br + +\fB-h, --help, help\fR +.br + Show a help dialog +.br + +\fB-l, logging\fR [on|off|off noflush] +.br + Specify whether the Pi-hole log should be used +.br + + (Logging options): +.br + on Enable the Pi-hole log at /var/log/pihole.log +.br + off Disable and flush the Pi-hole log at /var/log/pihole.log +.br + off noflush Disable the Pi-hole log at /var/log/pihole.log +.br + +.br +\fB-up, updat\fBe\fR\fR\fBPihole\fR [--check-only] +.br + Update Pi-hole subsystems +.br + Add '--check-only' to exit script before update is performed. +.br + +\fB-v, version\fR [repo] [options] +.br + Show installed versions of Pi-hole, Admin Console & FTL +.br + +.br + (repo options): +.br + -p, --pihole Only retrieve info regarding Pi-hole repository +.br + -a, --admin Only retrieve info regarding AdminLTE repository +.br + -f, --ftl Only retrieve info regarding FTL repository +.br + (version options): +.br + -c, --current Return the current version +.br + -l, --latest Return the latest version +.br + --hash Return the Github hash from your local repositories +.br + +\fBuninstall\fR +.br + Uninstall Pi-hole from your system +.br + +\fBstatus\fR +.br + Display the running status of Pi-hole subsystems +.br + +\fBenable\fR +.br + Enable Pi-hole subsystems +.br + +\fBdisable\fR [time] +.br + Disable Pi-hole subsystems, optionally for a set duration +.br + +.br + (time options): +.br + #s Disable Pi-hole functionality for # second(s) +.br + #m Disable Pi-hole functionality for # minute(s) +.br + +\fBrestartdns\fR +.br + Restart Pi-hole subsystems +.br + +\fBcheckout\fR [repo][branch] +.br + Switch Pi-hole subsystems to a different Github branch +.br + + (repo options): +.br + core Change the branch of Pi-hole's core subsystem +.br + web Change the branch of Admin Console subsystem +.br + ftl Change the branch of Pi-hole's FTL subsystem +.br + (branch options): +.br + master Update subsystems to the latest stable release +.br + dev Update subsystems to the latest development release +.br + +.SH "EXAMPLE" +Some usage examples +.br + + Whitelist/blacklist manipulation +.br + + \fBpihole -w iloveads.example.com\fR Add "iloveads.example.com" to whitelist +.br + \fBpihole -b -d noads.example.com\fR Remove "noads.example.com" from blacklist +.br + \fBpihole -wild ads.example\fR Add "ads.example" as wildcard - would block ads.example.net +.br + + Changing the web ui password +.br + +.br + \fBpihole -a -p ExamplePassword\fR Change the password to "ExamplePassword" +.br + + Updating lists from internet sources +.br + +.br + \fBpihole -g\fR Update the list of ad-serving domains +.br + + Displaying version information +.br + +.br + \fBpihole -v -c\fR Display the current version of AdminLTE +.br + +.br + Temporarily disabling Pi-hole +.br + +.br + \fBpihole disable 5m\fR Disable Pi-hole functionality for five minutes +.br + +.br + Switching Pi-hole subsystem branches +.br + +.br + \fBpihole checkout master\fR Switch to master branch +.br + \fBpihole checkout core dev\fR Switch to core development branch +.br +.SH "SEE ALSO" + +dmasq(8), lighttpd(8) +.br +.SH "COLOPHON" + +Get sucked into the latest news and community activity by entering Pi-hole's orbit. Information about Pi-hole, and the latest version of the software can be found at https://pi-hole.net. +.br From a8103ca22d1c3de39376809e97f07bff6ba23584 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Sat, 2 Jun 2018 09:45:04 +1000 Subject: [PATCH 043/163] Manual page install function Function to install man page. Verifies that man pages are installed, and correct directory for the pihole manpage is present. Copies file, and runs man-db to update man page database. Signed-off-by: Rob Gill --- automated install/basic-install.sh | 34 ++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 43d4b7c2..9b5fde8e 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1193,6 +1193,37 @@ installConfigs() { fi } +install_manpage() { + # Copy Pi-hole man page and call mandb to update man page database + # Default location for man files for /usr/local/bin is /usr/local/share/man + # on lightweight systems may not be present, so check before copying. + echo -en " ${INFO} Testing man page installation" + if ! command -v mandb; then + # if mandb is not present, no manpage support + echo -e "${OVER} ${INFO} man not installed" + return + elif [[ ! -d "/usr/local/share/man" ]]; then + # appropriate directory for Pi-hole's man page is not present + echo -e "${OVER} ${INFO} man page not installed" + return + elif [[ ! -d "/usr/local/share/man/man8"]]; then + # if not present, create man8 directory + mkdir /usr/local/share/man/man8 + fi + # Testing complete, copy the file & update the man db + cp ${PI_HOLE_LOCAL_REPO}/pihole.8 /usr/local/share/man/man8/pihole.8 + if mandb -q &>/dev/null; then + # Updated successfully + echo -e "${OVER} ${TICK} man page installed and database updated" + return + else + # Something is wrong with the system's man installation, clean up + # our file, (leave everything how we found it). + rm /usr/local/share/man/man8/pihole.8 + echo -e "${OVER} ${INFO} man page db not updated, man page not installed" + fi +} + stop_service() { # Stop service passed in as argument. # Can softfail, as process may not be installed when this is called @@ -1695,6 +1726,9 @@ installPihole() { configureFirewall fi + # install a man page entry for pihole + install_manpage + # Update setupvars.conf with any variables that may or may not have been changed during the install finalExports } From 999e47a26c28c627836c4b8dd4ea56f8f29f8f80 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Sat, 2 Jun 2018 09:50:52 +1000 Subject: [PATCH 044/163] Remove manual page Tests for presence of pihole man page. If it is present, deletes it and runs man-db to rebuild manual database. Signed-off-by: Rob Gill --- automated install/uninstall.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 0a51fd4a..4525f42f 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -171,6 +171,13 @@ removeNoPurge() { echo -e "${OVER} ${TICK} Removed pihole-FTL" fi + # If the pihole manpage exists, then delete and rebuild man-db + if [[ -f /usr/local/share/man/man8/pihole.8 ]]; then + ${SUDO} rm -f /usr/local/share/man/man8/pihole.8 + ${SUDO} mandb -q &>/dev/null + echo -e " ${TICK} Removed pihole man page" + fi + # If the pihole user exists, then remove if id "pihole" &> /dev/null; then if ${SUDO} userdel -r pihole 2> /dev/null; then From a8f0283e93ba8d7dd3ea4cfb6d4361c9523d0d4d Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Sat, 2 Jun 2018 10:11:11 +1000 Subject: [PATCH 045/163] Update basic-install.sh This hands checking of lighttpd's status over to the existing check_service_active() function. All other checks of service status within the install script are handled by this function. Use of existing function: Avoids duplication of service detection logic. Uses return code to determine status, thereby avoids parsing text to determine status, and reliance on English language locale to determine activity, (which may also be broken on some systems (# 2204) Signed-off-by: Rob Gill --- automated install/basic-install.sh | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 43d4b7c2..fcc66fe1 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2290,15 +2290,13 @@ main() { if [[ "${INSTALL_WEB_SERVER}" == true ]]; then enable_service lighttpd fi - - if [[ -x "$(command -v systemctl)" ]]; then - # Value will either be 1, if true, or 0 - LIGHTTPD_ENABLED=$(systemctl is-enabled lighttpd | grep -c 'enabled' || true) - else - # Value will either be 1, if true, or 0 - LIGHTTPD_ENABLED=$(service lighttpd status | awk '/Loaded:/ {print $0}' | grep -c 'enabled' || true) + # Determin if lighttpd is correctly enabled + if check_service_active "lighttpd"; then + LIGHTTPD_ENABLED=true; + else + LIGHTTPD_ENABLED=false; fi - + # Install and log everything to a file installPihole | tee -a /proc/$$/fd/3 @@ -2327,7 +2325,7 @@ main() { # If the Web server was installed, if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - if [[ "${LIGHTTPD_ENABLED}" == "1" ]]; then + if [[ "${LIGHTTPD_ENABLED}" == true ]]; then start_service lighttpd enable_service lighttpd else From aa191e92021302dca9bc3e0bcba6612f2880c6ec Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Sat, 2 Jun 2018 10:46:07 +1000 Subject: [PATCH 046/163] Update basic-install.sh Signed-off-by: Rob Gill --- automated install/basic-install.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index fcc66fe1..5e10c9e9 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2290,11 +2290,11 @@ main() { if [[ "${INSTALL_WEB_SERVER}" == true ]]; then enable_service lighttpd fi - # Determin if lighttpd is correctly enabled + # Determine if lighttpd is correctly enabled if check_service_active "lighttpd"; then - LIGHTTPD_ENABLED=true; + LIGHTTPD_ENABLED=true else - LIGHTTPD_ENABLED=false; + LIGHTTPD_ENABLED=false fi # Install and log everything to a file From ce5429aba7e7dcfd3260684bbce85f0fe2dc92f2 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Sat, 2 Jun 2018 11:02:06 +1000 Subject: [PATCH 047/163] stickler Signed-off-by: Rob Gill --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9b5fde8e..83764a0f 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1206,7 +1206,7 @@ install_manpage() { # appropriate directory for Pi-hole's man page is not present echo -e "${OVER} ${INFO} man page not installed" return - elif [[ ! -d "/usr/local/share/man/man8"]]; then + elif [[ ! -d "/usr/local/share/man/man8" ]]; then # if not present, create man8 directory mkdir /usr/local/share/man/man8 fi From c0837c726fb5ee32354d3ec2d9ac5b9b776f57f4 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Sat, 2 Jun 2018 11:18:13 +1000 Subject: [PATCH 048/163] blank space Signed-off-by: Rob Gill --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 5e10c9e9..91589914 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2296,7 +2296,7 @@ main() { else LIGHTTPD_ENABLED=false fi - + # Install and log everything to a file installPihole | tee -a /proc/$$/fd/3 From ef65bac79b5b6627b51892d2634149d42ce9a97c Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Sat, 2 Jun 2018 11:27:35 +1000 Subject: [PATCH 049/163] change status from INFO to CROSS Signed-Off-By: Rob Gill --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 83764a0f..4064ec04 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1220,7 +1220,7 @@ install_manpage() { # Something is wrong with the system's man installation, clean up # our file, (leave everything how we found it). rm /usr/local/share/man/man8/pihole.8 - echo -e "${OVER} ${INFO} man page db not updated, man page not installed" + echo -e "${OVER} ${CROSS} man page db not updated, man page not installed" fi } From d01a568b8bb867beb3304d1af644deaa15e1f269 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Sat, 2 Jun 2018 14:13:30 +1000 Subject: [PATCH 050/163] Requested changes (also tabs -> spaces) Signed-off-by: Rob Gill --- pihole.8 | 186 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 96 insertions(+), 90 deletions(-) diff --git a/pihole.8 b/pihole.8 index d62dbf8a..75e3e565 100644 --- a/pihole.8 +++ b/pihole.8 @@ -9,7 +9,7 @@ Pi-hole : A black-hole for internet advertisements .br \fBpihole -a\fR \fB-p\fR password .br -\fBpihole -a\fR (-c|-f|-k) +\fBpihole -a\fR (\fB-c|-f|-k\fR) .br \fBpihole -a\fR [\fB-r\fR hostrecord] .br @@ -33,10 +33,12 @@ pihole -g\fR .br \fBpihole\fR -\fBq\fR [options] .br -\fBpihole\fR \fB-l\fR (on|off|off noflush) +\fBpihole\fR \fB-l\fR (\fBon|off|off noflush\fR) .br \fBpihole -up \fR[--checkonly] .br +\fBpihole -v\fR [-p|-a|-f] [-c|-l|-hash] +.br \fBpihole uninstall .br pihole status @@ -56,269 +58,273 @@ Available commands and options: \fB-w, whitelist\fR [options] [ ] .br - Adds or removes specified domain or domains tho the Whitelist + Adds or removes specified domain or domains tho the Whitelist .br \fB-b, blacklist\fR [options] [ ] .br - Adds or removes specified domain or domains to the blacklist + Adds or removes specified domain or domains to the blacklist .br \fB-wild, wildcard\fR [options] [ ] .br - Add or removes specified domain, and all subdomains to the blacklist -.br - + Add or removes specified domain, and all subdomains to the blacklist .br - (Whitelist/Blacklist manipulation options): + + (Whitelist/Blacklist manipulation options): .br - -d, --delmode Remove domain(s) from the list + -d, --delmode Remove domain(s) from the list .br - -nr, --noreload Update list without refreshing dnsmasq + -nr, --noreload Update list without refreshing dnsmasq .br - -q, --quiet Make output less verbose + -q, --quiet Make output less verbose .br - -l, --list Display all your listed domains + -l, --list Display all your listed domains .br - --nuke Removes all entries in a list + --nuke Removes all entries in a list .br -\fB-d, debug\fR [\fB-a\fR] +\fB-d, debug\fR [-a] .br - Start a debugging session Add '-a' to enable automated debugging + Start a debugging session +.br + + -a Enable automated debugging .br \fB-f, flush\fR .br - Flush the Pi-hole log + Flush the Pi-hole log .br .br \fB-r, reconfigure\fR .br - Reconfigure or Repair Pi-hole subsystems + Reconfigure or Repair Pi-hole subsystems .br \fB-t, tail\fR .br - View the live output of the Pi-hole log + View the live output of the Pi-hole log .br -\fB-a, admin\fR [options] Console +\fB-a, admin\fR [options] .br - -.br - (Admin console options): + + (Admin options): .br - -p, password Set Admin Console password + -p, password Set Web Interface password .br - -c, celsius Set Celsius as preferred temperature unit + -c, celsius Set Celsius as preferred temperature unit .br - -f, fahrenheit Set Fahrenheit as preferred temperature unit + -f, fahrenheit Set Fahrenheit as preferred temperature unit .br - -k, kelvin Set Kelvin as preferred temperature unit + -k, kelvin Set Kelvin as preferred temperature unit .br - -r, hostrecord Add a name to the DNS associated to an IPv4/IPv6 address + -r, hostrecord Add a name to the DNS associated to an IPv4/IPv6 address .br - -e, email Set an administrative contact address for the Block Page + -e, email Set an administrative contact address for the Block Page .br - -i, interface Specify dnsmasq's interface listening behavior + -i, interface Specify dnsmasq's interface listening behavior .br - -l, privacylevel Set privacy level (0 = lowest, 3 = highest) + -l, privacylevel Set privacy level (0 = lowest, 3 = highest) .br .br \fB-c, chronometer\fR [options] .br - Calculates stats and displays to an LCD + Calculates stats and displays to an LCD .br - (Chronometer Options): + (Chronometer Options): .br - -j, --json Output stats as JSON formatted string + -j, --json Output stats as JSON formatted string .br - -r, --refresh Set update frequency (in seconds) + -r, --refresh Set update frequency (in seconds) .br - -e, --exit Output stats and exit witout refreshing + -e, --exit Output stats and exit witout refreshing .br \fB-g, updateGravity\fR .br - Update the list of ad-serving domains + Update the list of ad-serving domains .br \fB-q, query\fR [option] .br - Query the adlists for a specified domain + Query the adlists for a specified domain .br - (Query options): + (Query options): .br - -adlist Print the name of the block list URL + -adlist Print the name of the block list URL .br - -exact Search the block lists for exact domain matches + -exact Search the block lists for exact domain matches .br - -all Return all query matches within a block list + -all Return all query matches within a block list .br \fB-h, --help, help\fR .br - Show a help dialog + Show a help dialog .br \fB-l, logging\fR [on|off|off noflush] .br - Specify whether the Pi-hole log should be used + Specify whether the Pi-hole log should be used .br (Logging options): .br - on Enable the Pi-hole log at /var/log/pihole.log + on Enable the Pi-hole log at /var/log/pihole.log .br - off Disable and flush the Pi-hole log at /var/log/pihole.log + off Disable and flush the Pi-hole log at /var/log/pihole.log .br - off noflush Disable the Pi-hole log at /var/log/pihole.log + off noflush Disable the Pi-hole log at /var/log/pihole.log .br .br \fB-up, updat\fBe\fR\fR\fBPihole\fR [--check-only] .br - Update Pi-hole subsystems + Update Pi-hole subsystems .br - Add '--check-only' to exit script before update is performed. + + --check-only Exit script before update is performed. .br \fB-v, version\fR [repo] [options] .br - Show installed versions of Pi-hole, Admin Console & FTL + Show installed versions of Pi-hole, Web Interface & FTL .br .br - (repo options): + (repo options): .br - -p, --pihole Only retrieve info regarding Pi-hole repository + -p, --pihole Only retrieve info regarding Pi-hole repository .br - -a, --admin Only retrieve info regarding AdminLTE repository + -a, --admin Only retrieve info regarding AdminLTE repository .br - -f, --ftl Only retrieve info regarding FTL repository + -f, --ftl Only retrieve info regarding FTL repository .br - (version options): + (version options): .br - -c, --current Return the current version + -c, --current Return the current version .br - -l, --latest Return the latest version + -l, --latest Return the latest version .br - --hash Return the Github hash from your local repositories + --hash Return the Github hash from your local repositories .br \fBuninstall\fR .br - Uninstall Pi-hole from your system + Uninstall Pi-hole from your system .br \fBstatus\fR .br - Display the running status of Pi-hole subsystems + Display the running status of Pi-hole subsystems .br \fBenable\fR .br - Enable Pi-hole subsystems + Enable Pi-hole subsystems .br \fBdisable\fR [time] .br - Disable Pi-hole subsystems, optionally for a set duration + Disable Pi-hole subsystems, optionally for a set duration .br .br - (time options): + (time options): .br - #s Disable Pi-hole functionality for # second(s) + #s Disable Pi-hole functionality for # second(s) .br - #m Disable Pi-hole functionality for # minute(s) + #m Disable Pi-hole functionality for # minute(s) .br \fBrestartdns\fR .br - Restart Pi-hole subsystems + Restart Pi-hole subsystems .br \fBcheckout\fR [repo][branch] .br - Switch Pi-hole subsystems to a different Github branch + Switch Pi-hole subsystems to a different Github branch .br - (repo options): + (repo options): .br - core Change the branch of Pi-hole's core subsystem + core Change the branch of Pi-hole's core subsystem .br - web Change the branch of Admin Console subsystem + web Change the branch of Admin Console subsystem .br - ftl Change the branch of Pi-hole's FTL subsystem + ftl Change the branch of Pi-hole's FTL subsystem .br - (branch options): + (branch options): .br - master Update subsystems to the latest stable release + master Update subsystems to the latest stable release .br - dev Update subsystems to the latest development release + dev Update subsystems to the latest development release +.br + branchname Update subsystems to the specified branchname .br - .SH "EXAMPLE" + Some usage examples .br - Whitelist/blacklist manipulation + Whitelist/blacklist manipulation .br - \fBpihole -w iloveads.example.com\fR Add "iloveads.example.com" to whitelist + \fBpihole -w iloveads.example.com\fR Add "iloveads.example.com" to whitelist .br - \fBpihole -b -d noads.example.com\fR Remove "noads.example.com" from blacklist + \fBpihole -b -d noads.example.com\fR Remove "noads.example.com" from blacklist .br - \fBpihole -wild ads.example\fR Add "ads.example" as wildcard - would block ads.example.net + \fBpihole -wild example\fR Add "example" as wildcard - would block ads.example.net, example.com etc. .br - Changing the web ui password + Changing the web ui password .br .br - \fBpihole -a -p ExamplePassword\fR Change the password to "ExamplePassword" + \fBpihole -a -p ExamplePassword\fR Change the password to "ExamplePassword" .br - Updating lists from internet sources + Updating lists from internet sources .br .br - \fBpihole -g\fR Update the list of ad-serving domains + \fBpihole -g\fR Update the list of ad-serving domains .br - Displaying version information + Displaying version information .br .br - \fBpihole -v -c\fR Display the current version of AdminLTE + \fBpihole -v -a -c\fR Display the current version of AdminLTE .br .br - Temporarily disabling Pi-hole + Temporarily disabling Pi-hole .br .br - \fBpihole disable 5m\fR Disable Pi-hole functionality for five minutes + \fBpihole disable 5m\fR Disable Pi-hole functionality for five minutes .br .br - Switching Pi-hole subsystem branches + Switching Pi-hole subsystem branches .br .br - \fBpihole checkout master\fR Switch to master branch + \fBpihole checkout master\fR Switch to master branch .br - \fBpihole checkout core dev\fR Switch to core development branch + \fBpihole checkout core dev\fR Switch to core development branch .br .SH "SEE ALSO" -dmasq(8), lighttpd(8) +dnsmasq(8), lighttpd(8) .br .SH "COLOPHON" From 9970f3786fba774db6ebfb087a4c97d28ea91267 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Sat, 2 Jun 2018 14:14:55 +1000 Subject: [PATCH 051/163] Requested change Signed-off-by: Rob Gill --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4064ec04..e8b1b9bc 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1198,7 +1198,7 @@ install_manpage() { # Default location for man files for /usr/local/bin is /usr/local/share/man # on lightweight systems may not be present, so check before copying. echo -en " ${INFO} Testing man page installation" - if ! command -v mandb; then + if ! command -v mandb &>/dev/null; then # if mandb is not present, no manpage support echo -e "${OVER} ${INFO} man not installed" return From 213f23aaf6c347e7331b03eb0db16a410efbfac1 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Sat, 2 Jun 2018 14:46:45 +1000 Subject: [PATCH 052/163] wildcard & space Signed-off-by: Rob Gill --- pihole.8 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pihole.8 b/pihole.8 index 75e3e565..76d6b72b 100644 --- a/pihole.8 +++ b/pihole.8 @@ -230,7 +230,7 @@ Available commands and options: Enable Pi-hole subsystems .br -\fBdisable\fR [time] +\fBdisable\fR [time] .br Disable Pi-hole subsystems, optionally for a set duration .br @@ -248,7 +248,7 @@ Available commands and options: Restart Pi-hole subsystems .br -\fBcheckout\fR [repo][branch] +\fBcheckout\fR [repo] [branch] .br Switch Pi-hole subsystems to a different Github branch .br @@ -281,7 +281,7 @@ Some usage examples .br \fBpihole -b -d noads.example.com\fR Remove "noads.example.com" from blacklist .br - \fBpihole -wild example\fR Add "example" as wildcard - would block ads.example.net, example.com etc. + \fBpihole -wild example.com\fR Add "example.com" as wildcard - would block ads.example.com, www.example.com etc. .br Changing the web ui password From 83ff0055b672eab6d237b65ea88fe60acc96dbf6 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Sat, 2 Jun 2018 15:30:51 +1000 Subject: [PATCH 053/163] Test and indicate status This added function tests if pihole-FTL is running when the sysV. Signed-off-by: Rob Gill --- advanced/pihole-FTL.service | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/advanced/pihole-FTL.service b/advanced/pihole-FTL.service index ef8ee9c2..ecc7a52a 100644 --- a/advanced/pihole-FTL.service +++ b/advanced/pihole-FTL.service @@ -69,13 +69,25 @@ stop() { echo } +# Indicate the service status +status() { + if is_running; then + echo "[ ok ] pihole-FTL is running" + exit 0 + else + echo "[ ] pihole-FTL is not running" + exit 1 + fi +} + + ### main logic ### case "$1" in stop) stop ;; status) - status pihole-FTL + status ;; start|restart|reload|condrestart) stop From f5541860bc16b96bf9f8121657d4c054d43c42d5 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 2 Jun 2018 09:56:08 +0100 Subject: [PATCH 054/163] Don't run the installer when doing ftl checkout, instead just run the ftl install functions Signed-off-by: Adam Warner --- advanced/Scripts/piholeCheckout.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index 65bb5b7a..5125a2b4 100644 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -166,6 +166,9 @@ checkout() { if check_download_exists "$path"; then echo " ${TICK} Branch ${2} exists" echo "${2}" > /etc/pihole/ftlbranch + FTLinstall "${binary}" + start_service pihole-FTL + enable_service pihole-FTL else echo " ${CROSS} Requested branch \"${2}\" is not available" ftlbranches=( $(git ls-remote https://github.com/pi-hole/ftl | grep 'heads' | sed 's/refs\/heads\///;s/ //g' | awk '{print $2}') ) @@ -180,7 +183,7 @@ checkout() { fi # Force updating everything - if [[ ! "${1}" == "web" ]]; then + if [[ ! "${1}" == "web" && ! "${1}" == "ftl" ]]; then echo -e " ${INFO} Running installer to upgrade your installation" if "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh" --unattended; then exit 0 From ed7ebfd58df4192876e7d6fc4ecc96d2897c9470 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Sun, 3 Jun 2018 22:15:26 +1000 Subject: [PATCH 055/163] man page for pihole-FTL Signed-off-by: Rob Gill --- automated install/basic-install.sh | 25 +++++--- automated install/uninstall.sh | 2 +- manpages/pihole-FTL.8 | 94 ++++++++++++++++++++++++++++++ manpages/pihole-FTL.conf.5 | 74 +++++++++++++++++++++++ pihole.8 => manpages/pihole.8 | 65 +++++++++------------ 5 files changed, 212 insertions(+), 48 deletions(-) create mode 100644 manpages/pihole-FTL.8 create mode 100644 manpages/pihole-FTL.conf.5 rename pihole.8 => manpages/pihole.8 (90%) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 325688cd..8181bcb7 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1194,7 +1194,7 @@ installConfigs() { } install_manpage() { - # Copy Pi-hole man page and call mandb to update man page database + # Copy Pi-hole man pages and call mandb to update man page database # Default location for man files for /usr/local/bin is /usr/local/share/man # on lightweight systems may not be present, so check before copying. echo -en " ${INFO} Testing man page installation" @@ -1204,23 +1204,30 @@ install_manpage() { return elif [[ ! -d "/usr/local/share/man" ]]; then # appropriate directory for Pi-hole's man page is not present - echo -e "${OVER} ${INFO} man page not installed" + echo -e "${OVER} ${INFO} man pages not installed" return - elif [[ ! -d "/usr/local/share/man/man8" ]]; then + fi + if [[ ! -d "/usr/local/share/man/man8" ]]; then # if not present, create man8 directory mkdir /usr/local/share/man/man8 fi - # Testing complete, copy the file & update the man db - cp ${PI_HOLE_LOCAL_REPO}/pihole.8 /usr/local/share/man/man8/pihole.8 + if [[ ! -d "/usr/local/share/man/man5" ]]; then + # if not present, create man8 directory + mkdir /usr/local/share/man/man5 + fi + # Testing complete, copy the files & update the man db + cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole.8 /usr/local/share/man/man8/pihole.8 + cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.8 /usr/local/share/man/man8/pihole-FTL.8 + cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.conf.5 /usr/local/share/man/man5/pihole-FTL.conf.5 if mandb -q &>/dev/null; then # Updated successfully - echo -e "${OVER} ${TICK} man page installed and database updated" + echo -e "${OVER} ${TICK} man pages installed and database updated" return else # Something is wrong with the system's man installation, clean up - # our file, (leave everything how we found it). - rm /usr/local/share/man/man8/pihole.8 - echo -e "${OVER} ${CROSS} man page db not updated, man page not installed" + # our files, (leave everything how we found it). + rm /usr/local/share/man/man8/pihole.8 /usr/local/share/man/man8/pihole-FTL.8 /usr/local/share/man/man5/pihole-FTL.conf.5 + echo -e "${OVER} ${CROSS} man page db not updated, man pages not installed" fi } diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 4525f42f..9322de92 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -173,7 +173,7 @@ removeNoPurge() { # If the pihole manpage exists, then delete and rebuild man-db if [[ -f /usr/local/share/man/man8/pihole.8 ]]; then - ${SUDO} rm -f /usr/local/share/man/man8/pihole.8 + ${SUDO} rm -f /usr/local/share/man/man8/pihole.8 /usr/local/share/man/man8/pihole-FTL.8 /usr/local/share/man/man5/pihole-FTL.conf.5 ${SUDO} mandb -q &>/dev/null echo -e " ${TICK} Removed pihole man page" fi diff --git a/manpages/pihole-FTL.8 b/manpages/pihole-FTL.8 new file mode 100644 index 00000000..b11f00de --- /dev/null +++ b/manpages/pihole-FTL.8 @@ -0,0 +1,94 @@ +.TH "Pihole-FTL" "8" "pihole-FTL" "Pi-hole" "June 2018" +.SH "NAME" +pihole-FTL - Pi-hole : The Faster-Than-Light (FTL) Engine +.br +.SH "SYNOPSIS" +\fBservice pihole-FTL \fR(\fBstart\fR|\fBstop\fR|\fBrestart\fR) +.br + +\fBpihole-FTL debug\fR +.br +\fBpihole-FTL test\fR +.br +\fBpihole-FTL -v\fR +.br +\fBpihole-FTL -t\fR +.br +\fBpihole-FTL -b\fR +.br +\fBpihole-FTL -f\fR +.br +\fBpihole-FTL dnsmasq-test\fR +.br + +.SH "DESCRIPTION" +Pi-hole : The Faster-Than-Light (FTL) Engine is a lightweight, purpose-built daemon used to provide statistics needed for the Pi-hole Web Interface, and its API can be easily integrated into your own projects. Although it is an optional component of the Pi-hole ecosystem, it will be installed by default to provide statistics. As the name implies, FTL does its work \fIvery\fR \fIquickly\fR! +.br + +Usage +.br + +\fB\fBservi\fRce pihole-FTL start\fR +.br + Start the pihole-FTL daemon +.br + +\fBservice pihole-FTL stop\fR +.br + Stop the pihole-FTL daemon +.br + +\fBservice pihole-FTL restart\fR +.br + If the pihole-FTP daemon is running, stop and then start, otherwise start. +.br + +Command line arguments +.br + +\fBdebug\fR +.br + Don't go into daemon mode (stay in foreground) + more verbose logging +.br + +\fBtest\fR +.br + Start FTL and process everything, but shut down immediately afterwards +.br + +\fB-v, version\fR +.br + Don't start FTL, show only version +.br + +\fB-t, tag\fR +.br + Don't start FTL, show only git tag +.br + +\fB-b, branch\fR +.br + Don't start FTL, show only git branch FTL was compiled from +.br + +\fB-f, no-daemon\fR +.br + Don't go into background (daemon mode) +.br + +\fB-h, help\fR +.br + Don't start FTL, show help +.br +.SH "EXAMPLE" +Command line arguments can be arbitrarily combined, e.g: +.br + +\fBpihole-FTL debug test\fR +.br + +Start ftl in foreground with more verbose logging, process everything and shutdown immediately +.br +.SH "SEE ALSO" +\fBpihole\fR(8), \fBpihole-FTL.conf\fR(5) +.br diff --git a/manpages/pihole-FTL.conf.5 b/manpages/pihole-FTL.conf.5 new file mode 100644 index 00000000..1db71ace --- /dev/null +++ b/manpages/pihole-FTL.conf.5 @@ -0,0 +1,74 @@ +.TH "pihole-FTL.conf" "5" "pihole-FTL.conf" "pihole-FTL.conf" "June 2018" +.SH "NAME" + +pihole-FTL.conf - FTL's config file +.br +.SH "DESCRIPTION" + +/etc/pihole/pihole-FTL.conf will be read by \fBpihole-FTL(8)\fR on startup. +.br + +SOCKET_LISTENING=localonly|all +.br + Listen only for local socket connections or permit all connections +.br + +QUERY_DISPLAY=yes|no +.br + Display all queries? Set to no to hide query display +.br + +AAAA_QUERY_ANALYSIS=yes|no +.br + Allow FTL to analyze AAAA queries from pihole.log? +.br + +MAXDBDAYS=365 +.br + How long should queries be stored in the database? +.br + Setting this to 0 disables the database altogether +.br + +RESOLVE_IPV6=yes|no +.br + Should FTL try to resolve IPv6 addresses to host names? +.br + +RESOLVE_IPV4=yes|no +.br + Should FTL try to resolve IPv4 addresses to host names? +.br + +DBINTERVAL=1.0 +.br + How often do we store queries in FTL's database [minutes]? +.br + +DBFILE=/etc/pihole/pihole-FTL.db +.br + Specify path and filename of FTL's SQLite long-term database. +.br + Setting this to DBFILE= disables the database altogether +.br + +MAXLOGAGE=24.0 +.br + Up to how many hours of queries should be imported from the database and logs? +.br + Maximum is 744 (31 days) +.br + +For each setting, the option shown first is the default. +.br +.SH "SEE ALSO" + +\fBpihole\fR(8), \fBpihole-FTL\fR(8) +.br +.SH "COLOPHON" + +Pi-hole : The Faster-Than-Light (FTL) Engine is a lightweight, purpose-built daemon used to provide statistics needed for the Pi-hole Web Interface, and its API can be easily integrated into your own projects. Although it is an optional component of the Pi-hole ecosystem, it will be installed by default to provide statistics. As the name implies, FTL does its work \fIvery quickly\fR! +.br + +Get sucked into the latest news and community activity by entering Pi-hole's orbit. Information about Pi-hole, and the latest version of the software can be found at https://pi-hole.net +.br diff --git a/pihole.8 b/manpages/pihole.8 similarity index 90% rename from pihole.8 rename to manpages/pihole.8 index 76d6b72b..fc719ffb 100644 --- a/pihole.8 +++ b/manpages/pihole.8 @@ -21,13 +21,13 @@ Pi-hole : A black-hole for internet advertisements .br \fBpihole -c\fR [-j|-r|-e] .br -\fBpihole\fR \fB-d\fR [-a] +\fBpihole\fR \fB-d\fR [-a] .br \fBpihole -f .br pihole -r .br -pihole -t +pihole -t .br pihole -g\fR .br @@ -84,9 +84,9 @@ Available commands and options: --nuke Removes all entries in a list .br -\fB-d, debug\fR [-a] +\fB-d, debug\fR [-a] .br - Start a debugging session + Start a debugging session .br -a Enable automated debugging @@ -96,8 +96,7 @@ Available commands and options: .br Flush the Pi-hole log .br - -.br + \fB-r, reconfigure\fR .br Reconfigure or Repair Pi-hole subsystems @@ -129,11 +128,10 @@ Available commands and options: .br -l, privacylevel Set privacy level (0 = lowest, 3 = highest) .br - -.br + \fB-c, chronometer\fR [options] .br - Calculates stats and displays to an LCD + Calculates stats and displays to an LCD .br (Chronometer Options): @@ -145,12 +143,12 @@ Available commands and options: -e, --exit Output stats and exit witout refreshing .br -\fB-g, updateGravity\fR +\fB-g, updateGravity\fR .br Update the list of ad-serving domains .br -\fB-q, query\fR [option] +\fB-q, query\fR [option] .br Query the adlists for a specified domain .br @@ -164,7 +162,7 @@ Available commands and options: -all Return all query matches within a block list .br -\fB-h, --help, help\fR +\fB-h, --help, help\fR .br Show a help dialog .br @@ -178,21 +176,20 @@ Available commands and options: .br on Enable the Pi-hole log at /var/log/pihole.log .br - off Disable and flush the Pi-hole log at /var/log/pihole.log + off Disable and flush the Pi-hole log at /var/log/pihole.log .br off noflush Disable the Pi-hole log at /var/log/pihole.log .br - -.br -\fB-up, updat\fBe\fR\fR\fBPihole\fR [--check-only] + +\fB-up, updat\fBe\fR\fR\fBPihole\fR [--check-only] .br - Update Pi-hole subsystems + Update Pi-hole subsystems .br --check-only Exit script before update is performed. .br -\fB-v, version\fR [repo] [options] +\fB-v, version\fR [repo] [options] .br Show installed versions of Pi-hole, Web Interface & FTL .br @@ -234,8 +231,7 @@ Available commands and options: .br Disable Pi-hole subsystems, optionally for a set duration .br - -.br + (time options): .br #s Disable Pi-hole functionality for # second(s) @@ -248,7 +244,7 @@ Available commands and options: Restart Pi-hole subsystems .br -\fBcheckout\fR [repo] [branch] +\fBcheckout\fR [repo] [branch] .br Switch Pi-hole subsystems to a different Github branch .br @@ -281,50 +277,43 @@ Some usage examples .br \fBpihole -b -d noads.example.com\fR Remove "noads.example.com" from blacklist .br - \fBpihole -wild example.com\fR Add "example.com" as wildcard - would block ads.example.com, www.example.com etc. + \fB\fBpihole -wild example\fR.com\fR Add "example" as wildcard - would block ads.example.com, www.example.com etc. .br - Changing the web ui password -.br - + Changing the Web Interface password .br + \fBpihole -a -p ExamplePassword\fR Change the password to "ExamplePassword" .br Updating lists from internet sources .br - -.br + \fBpihole -g\fR Update the list of ad-serving domains .br Displaying version information .br - -.br + \fBpihole -v -a -c\fR Display the current version of AdminLTE .br - -.br + Temporarily disabling Pi-hole .br - -.br + \fBpihole disable 5m\fR Disable Pi-hole functionality for five minutes .br - -.br + Switching Pi-hole subsystem branches .br - -.br + \fBpihole checkout master\fR Switch to master branch .br \fBpihole checkout core dev\fR Switch to core development branch .br .SH "SEE ALSO" -dnsmasq(8), lighttpd(8) +\fBlighttpd\fR(8), \fBpihole-FTL\fR(8) .br .SH "COLOPHON" From 3149a95d6ac98154cedde481bafcf92f05a8a6e1 Mon Sep 17 00:00:00 2001 From: Mark Drobnak Date: Sun, 3 Jun 2018 13:33:33 -0400 Subject: [PATCH 056/163] Fix gravity_ParseFileIntoDomains Awk comment Signed-off-by: Mark Drobnak --- gravity.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index 58419029..dfcb753a 100755 --- a/gravity.sh +++ b/gravity.sh @@ -277,9 +277,9 @@ gravity_ParseFileIntoDomains() { # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious # This helps with that and makes it easier to read # It also helps with debugging so each stage of the script can be researched more in depth - #Awk -F splits on given IFS, we grab the right hand side (chops trailing #coments and /'s to grab the domain only. - #Last awk command takes non-commented lines and if they have 2 fields, take the left field (the domain) and leave - #+ the right (IP address), otherwise grab the single field. + # Awk -F splits on given IFS, we grab the right hand side (chops trailing #coments and /'s to grab the domain only. + # Last awk command takes non-commented lines and if they have 2 fields, take the right field (the domain) and leave + # the left (IP address), otherwise grab the single field. < ${source} awk -F '#' '{print $1}' | \ awk -F '/' '{print $1}' | \ From 49fb4421dd10d491e1bae7998e86e52cc2c49bce Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Mon, 4 Jun 2018 09:54:53 +1000 Subject: [PATCH 057/163] Requested changes to manpages - now align with docs.pi-hole.net/ftldns. weird bolding issues fixed Signed-off-by: Rob Gill --- manpages/pihole-FTL.8 | 20 +++++++++++++++- manpages/pihole-FTL.conf.5 | 48 +++++++++++++++++++++++++++----------- manpages/pihole.8 | 4 ++-- 3 files changed, 55 insertions(+), 17 deletions(-) diff --git a/manpages/pihole-FTL.8 b/manpages/pihole-FTL.8 index b11f00de..812801a4 100644 --- a/manpages/pihole-FTL.8 +++ b/manpages/pihole-FTL.8 @@ -18,8 +18,12 @@ pihole-FTL - Pi-hole : The Faster-Than-Light (FTL) Engine .br \fBpihole-FTL -f\fR .br +\fBpihole-FTL -h\fR +.br \fBpihole-FTL dnsmasq-test\fR .br +\fBpihole-FTL --\fR (\fBoptions\fR) +.br .SH "DESCRIPTION" Pi-hole : The Faster-Than-Light (FTL) Engine is a lightweight, purpose-built daemon used to provide statistics needed for the Pi-hole Web Interface, and its API can be easily integrated into your own projects. Although it is an optional component of the Pi-hole ecosystem, it will be installed by default to provide statistics. As the name implies, FTL does its work \fIvery\fR \fIquickly\fR! @@ -28,7 +32,7 @@ Pi-hole : The Faster-Than-Light (FTL) Engine is a lightweight, purpose-built dae Usage .br -\fB\fBservi\fRce pihole-FTL start\fR +\fBservice pihole-FTL start\fR .br Start the pihole-FTL daemon .br @@ -80,6 +84,16 @@ Command line arguments .br Don't start FTL, show help .br + +\fBdnsmasq-test\fR +.br + Test resolver config file syntax +.br + +\fB--\fR (options) +.br + Pass options to internal resolver +.br .SH "EXAMPLE" Command line arguments can be arbitrarily combined, e.g: .br @@ -92,3 +106,7 @@ Start ftl in foreground with more verbose logging, process everything and shutdo .SH "SEE ALSO" \fBpihole\fR(8), \fBpihole-FTL.conf\fR(5) .br +.SH "COLOPHON" + +Get sucked into the latest news and community activity by entering Pi-hole's orbit. Information about Pi-hole, and the latest version of the software can be found at https://pi-hole.net +.br diff --git a/manpages/pihole-FTL.conf.5 b/manpages/pihole-FTL.conf.5 index 1db71ace..b99a6c1d 100644 --- a/manpages/pihole-FTL.conf.5 +++ b/manpages/pihole-FTL.conf.5 @@ -8,57 +8,77 @@ pihole-FTL.conf - FTL's config file /etc/pihole/pihole-FTL.conf will be read by \fBpihole-FTL(8)\fR on startup. .br -SOCKET_LISTENING=localonly|all +\fBSOCKET_LISTENING=localonly|all\fR .br Listen only for local socket connections or permit all connections .br -QUERY_DISPLAY=yes|no +\fBQUERY_DISPLAY=yes|no\fR .br Display all queries? Set to no to hide query display .br -AAAA_QUERY_ANALYSIS=yes|no +\fBAAAA_QUERY_ANALYSIS=yes|no\fR .br Allow FTL to analyze AAAA queries from pihole.log? .br -MAXDBDAYS=365 +\fBRESOLVE_IPV6=yes|no\fR .br - How long should queries be stored in the database? -.br - Setting this to 0 disables the database altogether + Should FTL try to resolve IPv6 addresses to host names? .br -RESOLVE_IPV6=yes|no +\fBRESOLVE_IPV4=yes|no\fR .br - Should FTL try to resolve IPv6 addresses to host names? + Should FTL try to resolve IPv4 addresses to host names? .br -RESOLVE_IPV4=yes|no +\fBMAXDBDAYS=365\fR .br - Should FTL try to resolve IPv4 addresses to host names? + How long should queries be stored in the database? +.br + Setting this to 0 disables the database .br -DBINTERVAL=1.0 +\fBDBINTERVAL=1.0\fR .br How often do we store queries in FTL's database [minutes]? .br -DBFILE=/etc/pihole/pihole-FTL.db +\fBDBFILE=/etc/pihole/pihole-FTL.db\fR .br Specify path and filename of FTL's SQLite long-term database. .br Setting this to DBFILE= disables the database altogether .br -MAXLOGAGE=24.0 +\fBMAXLOGAGE=24.0\fR .br Up to how many hours of queries should be imported from the database and logs? .br Maximum is 744 (31 days) .br +\fBFTLPORT=4711\fR +.br + On which port should FTL be listening? +.br + +\fBPRIVACYLEVEL=0|1|2|3\fR +.br + Which privacy level is used? +.br + +\fBIGNORE_LOCALHOST=no|yes\fR +.br + Should FTL ignore queries coming from the local machine? +.br + +\fBBLOCKINGMODE=IP|IP-AAAA-NODATA|NXDOMAIN|NULL\fR +.br + How should FTL reply to blocked queries? +.br + For each setting, the option shown first is the default. .br .SH "SEE ALSO" diff --git a/manpages/pihole.8 b/manpages/pihole.8 index fc719ffb..54bf4a31 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -181,7 +181,7 @@ Available commands and options: off noflush Disable the Pi-hole log at /var/log/pihole.log .br -\fB-up, updat\fBe\fR\fR\fBPihole\fR [--check-only] +\fB-up, updatePihole\fR [--check-only] .br Update Pi-hole subsystems .br @@ -277,7 +277,7 @@ Some usage examples .br \fBpihole -b -d noads.example.com\fR Remove "noads.example.com" from blacklist .br - \fB\fBpihole -wild example\fR.com\fR Add "example" as wildcard - would block ads.example.com, www.example.com etc. + \fBpihole -wild example.com\fR Add "example.com" as wildcard - would block ads.example.com, www.example.com etc. .br Changing the Web Interface password From 40bc390c3bd84893cbc5f54cffe020bcceec3d57 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Mon, 4 Jun 2018 13:34:42 +1000 Subject: [PATCH 058/163] Admin Console -> "Web Interface" User facing --help text changes Admin Console --> Web Interface and additional branch option Signed-Off-By: Rob Gill --- pihole | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pihole b/pihole index b3f532af..c125d0d9 100755 --- a/pihole +++ b/pihole @@ -541,12 +541,13 @@ Switch Pi-hole subsystems to a different Github branch Repositories: core [branch] Change the branch of Pi-hole's core subsystem - web [branch] Change the branch of Admin Console subsystem + web [branch] Change the branch of Web Interface subsystem ftl [branch] Change the branch of Pi-hole's FTL subsystem Branches: master Update subsystems to the latest stable release - dev Update subsystems to the latest development release" + dev Update subsystems to the latest development release + branchname Update subsystems to the specified branchname" exit 0 fi @@ -610,8 +611,8 @@ Debugging Options: -t, tail View the live output of the Pi-hole log Options: - -a, admin Admin Console options - Add '-h' for more info on admin console usage + -a, admin Web interface options + Add '-h' for more info on Web Interface usage -c, chronometer Calculates stats and displays to an LCD Add '-h' for more info on chronometer usage -g, updateGravity Update the list of ad-serving domains @@ -622,7 +623,7 @@ Options: Add '-h' for more info on query usage -up, updatePihole Update Pi-hole subsystems Add '--check-only' to exit script before update is performed. - -v, version Show installed versions of Pi-hole, Admin Console & FTL + -v, version Show installed versions of Pi-hole, Web Interface & FTL Add '-h' for more info on version usage uninstall Uninstall Pi-hole from your system status Display the running status of Pi-hole subsystems From a11e5e2debac1efde56b8e00ca7a439d73329a56 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Tue, 5 Jun 2018 10:23:23 +1000 Subject: [PATCH 059/163] dnsmasq Signed-off-by: Rob Gill --- manpages/pihole-FTL.8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manpages/pihole-FTL.8 b/manpages/pihole-FTL.8 index 812801a4..2928f2d8 100644 --- a/manpages/pihole-FTL.8 +++ b/manpages/pihole-FTL.8 @@ -92,7 +92,7 @@ Command line arguments \fB--\fR (options) .br - Pass options to internal resolver + Pass options to internal dnsmasq resolver .br .SH "EXAMPLE" Command line arguments can be arbitrarily combined, e.g: From 4188fb536e1fd78914ffb22de923a8e2cf2c9393 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Tue, 5 Jun 2018 10:25:51 +1000 Subject: [PATCH 060/163] Privacy levels Signed-off-by: Rob Gill --- manpages/pihole-FTL.conf.5 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/manpages/pihole-FTL.conf.5 b/manpages/pihole-FTL.conf.5 index b99a6c1d..50536279 100644 --- a/manpages/pihole-FTL.conf.5 +++ b/manpages/pihole-FTL.conf.5 @@ -68,6 +68,14 @@ pihole-FTL.conf - FTL's config file .br Which privacy level is used? .br + 0 - show everything +.br + 1 - hide domains +.br + 2 - hide domains and clients +.br + 3 - paranoia mode (hide everything) +.br \fBIGNORE_LOCALHOST=no|yes\fR .br From 7398a9ebf937cadeb3f47537684ed60facce5ffe Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Mon, 4 Jun 2018 19:59:23 -0500 Subject: [PATCH 061/163] remove projects that are not compliant with our trademarks Signed-off-by: Jacob Salmela --- README.md | 36 +++++++++++++----------------------- 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 60c54f87..40a9254f 100644 --- a/README.md +++ b/README.md @@ -178,32 +178,13 @@ Pi-hole being a **advertising-aware DNS/Web server**, makes use of the following * [AdminLTE Dashboard](https://github.com/almasaeed2010/AdminLTE) - premium admin control panel based on Bootstrap 3.x While quite outdated at this point, [this original blog post about Pi-hole](https://jacobsalmela.com/2015/06/16/block-millions-ads-network-wide-with-a-raspberry-pi-hole-2-0/) goes into **great detail** about how Pi-hole was originally setup and how it works. Syntactically, it's no longer accurate, but the same basic principles and logic still apply to Pi-hole's current state. - ------ - -## Pi-hole Projects -- [The Big Blocklist Collection](https://wally3k.github.io) -- [Docker Pi-hole container (x86 and ARM)](https://hub.docker.com/r/diginc/pi-hole/) -- [Pi-Hole in the cloud](http://blog.codybunch.com/2015/07/28/Pi-Hole-in-the-cloud/) -- [Pie in the Sky-Hole [A Pi-Hole in the cloud for ad-blocking via DNS]](https://dlaa.me/blog/post/skyhole) -- [Pi-hole Enable/Disable Button](http://thetimmy.silvernight.org/pages/endisbutton/) -- [Minibian Pi-hole](https://munkjensen.net/wiki/index.php/See_my_Pi-Hole#Minibian_Pi-hole) -- [CHiP-hole: Network-wide Ad-blocker](https://www.hackster.io/jacobsalmela/chip-hole-network-wide-ad-blocker-98e037) -- [Chrome Extension: Pi-Hole List Editor](https://chrome.google.com/webstore/detail/pi-hole-list-editor/hlnoeoejkllgkjbnnnhfolapllcnaglh) ([Source Code](https://github.com/packtloss/pihole-extension)) -- [Splunk: Pi-hole Visualiser](https://splunkbase.splunk.com/app/3023/) -- [Adblocking with Pi-hole and Ubuntu 14.04 on VirtualBox](https://hbalagtas.blogspot.com.au/2016/02/adblocking-with-pi-hole-and-ubuntu-1404.html) -- [Pi-hole stats in your Mac's menu bar](https://getbitbar.com/plugins/Network/pi-hole.1m.py) -- [Pi-hole unRAID Template](https://forums.lime-technology.com/topic/36810-support-spants-nodered-mqtt-dashing-couchdb/) -- [Copernicus: Windows Tray Application](https://github.com/goldbattle/copernicus) -- [Let your blink1 device blink when Pi-hole filters ads](https://gist.github.com/elpatron68/ec0b4c582e5abf604885ac1e068d233f) -- [Pi-hole metrics](https://github.com/nlamirault/pihole_exporter) exporter for [Prometheus](https://prometheus.io/) -- [Magic Mirror with DNS Filtering](https://zonksec.com/blog/magic-mirror-dns-filtering/#dnssoftware) -- [Pi-hole Droid: Android client](https://github.com/friimaind/pi-hole-droid) -- [Windows DNS Swapper](https://github.com/roots84/DNS-Swapper), see [#1400](https://github.com/pi-hole/pi-hole/issues/1400) -- [Pi-hole Visualizer](https://www.reddit.com/r/pihole/comments/82ikgb/pihole_visualizer_update/) ----- ## Coverage +- [Software Engineering Daily: Interview with the creator of Pi-hole](https://softwareengineeringdaily.com/2018/05/29/pi-hole-ad-blocker-hardware-with-jacob-salmela/) +- [Bloomberg Business Week: Brotherhood of the Ad blockers](https://www.bloomberg.com/news/features/2018-05-10/inside-the-brotherhood-of-pi-hole-ad-blockers) +- [Securing DNS across all of my devices with Pi-Hole + DNS-over-HTTPS + 1.1.1.1](https://scotthelme.co.uk/securing-dns-across-all-of-my-devices-with-pihole-dns-over-https-1-1-1-1/) +- [Adafruit: installing Pi-hole on a Pi Zero W](https://learn.adafruit.com/pi-hole-ad-blocker-with-pi-zero-w/install-pi-hole) - [Lifehacker: Turn A Raspberry Pi Into An Ad Blocker With A Single Command](https://www.lifehacker.com.au/2015/02/turn-a-raspberry-pi-into-an-ad-blocker-with-a-single-command/) - [MakeUseOf: Adblock Everywhere: The Raspberry Pi-Hole Way](http://www.makeuseof.com/tag/adblock-everywhere-raspberry-pi-hole-way/) - [Catchpoint: Ad-Blocking on Apple iOS9: Valuing the End User Experience](http://blog.catchpoint.com/2015/09/14/ad-blocking-apple/) @@ -222,3 +203,12 @@ While quite outdated at this point, [this original blog post about Pi-hole](http - [CryptoAUSTRALIA: How We Tried 5 Privacy Focused Raspberry Pi Projects](https://blog.cryptoaustralia.org.au/2017/10/05/5-privacy-focused-raspberry-pi-projects/) - [CryptoAUSTRALIA: Pi-hole Workshop](https://blog.cryptoaustralia.org.au/2017/11/02/pi-hole-network-wide-ad-blocker/) - [Know How 355: Killing ads with a Raspberry Pi-Hole!](https://www.twit.tv/shows/know-how/episodes/355) + +----- + +## Pi-hole Projects +- [The Big Blocklist Collection](https://wally3k.github.io) +- [Pie in the Sky-Hole](https://dlaa.me/blog/post/skyhole) +- [Copernicus: Windows Tray Application](https://github.com/goldbattle/copernicus) +- [Magic Mirror with DNS Filtering](https://zonksec.com/blog/magic-mirror-dns-filtering/#dnssoftware) +- [Windows DNS Swapper](https://github.com/roots84/DNS-Swapper) From 35ca54d98f7760e329f11b133431d3bef96623a7 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Mon, 4 Jun 2018 20:00:26 -0500 Subject: [PATCH 062/163] add patreon and adjust affiliate links Signed-off-by: Jacob Salmela --- README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/README.md b/README.md index 40a9254f..77f259e6 100644 --- a/README.md +++ b/README.md @@ -68,12 +68,11 @@ Sending a donation using our links below is **extremely helpful** in offsetting ### Alternative support If you'd rather not [donate](https://pi-hole.net/donate/) (_which is okay!_), there are other ways you can help support us: - +- [Patreon](https://patreon.com/pihole) _Become a patron for rewards_ - [Digital Ocean](http://www.digitalocean.com/?refcode=344d234950e1) _affiliate link_ - [UNIXstickers.com](http://unixstickers.refr.cc/jacobs) _save $5 when you spend $9 using our affiliate link_ - [Pi-hole Swag Store](https://pi-hole.net/shop/) _affiliate link_ - [Amazon](http://www.amazon.com/exec/obidos/redirect-home/pihole09-20) _affiliate link_ -- [Ho-ost](https://clients.ho-ost.com/aff.php?aff=19) _save 50% with our affiliate link_ - [DNS Made Easy](https://cp.dnsmadeeasy.com/u/133706) _affiliate link_ - [Vultr](http://www.vultr.com/?ref=7190426) _affiliate link_ - Spreading the word about our software, and how you have benefited from it @@ -99,9 +98,6 @@ While we are primarily reachable on our Frequently Asked Questions
  • Pi-hole Wiki
  • Feature Requests
    • - -
    -
    • Discourse User Forum
    • Reddit
    • Gitter (Real-time chat)
      • From 9422d57283e0d9f0c5413c1d10cc644f7a22d5ab Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Tue, 5 Jun 2018 12:08:27 +1000 Subject: [PATCH 063/163] Check version of installed php Signed-off-by: Rob Gill --- automated install/basic-install.sh | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 395a6203..4cd1e9aa 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -160,13 +160,27 @@ if command -v apt-get &> /dev/null; then # use iproute iproute_pkg="iproute" fi - # We prefer the php metapackage if it's there - if ${PKG_MANAGER} install --dry-run php > /dev/null 2>&1; then - phpVer="php" - # If not, - else + # Check for and determine version number, major and minor of current php install + if command -v php &> /dev/null; then + phpInsVersion="$(php -v | head -n1 | grep -Po '(? /dev/null 2>&1; then + phpVer="php" # fall back on the php5 packages - phpVer="php5" + else + phpVer="php5" + fi + else + # Supported php is installed, its common, cgi & sqlite counterparts are deps + phpInsMajor="$(echo $phpInsVer | cut -d\. -f1)" + phpInsMinor="$(echo $phpInsVer | cut -d\. -f2)" + phpVer="php$phpInsMajor.phpInsMinor" + fi + fi # We also need the correct version for `php-sqlite` (which differs across distros) if ${PKG_MANAGER} install --dry-run ${phpVer}-sqlite3 > /dev/null 2>&1; then From 7181d7ae6a87dda776ca3579fc6dcbb00ca5948d Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Tue, 5 Jun 2018 14:30:18 +1000 Subject: [PATCH 064/163] Left that fi Signed-off-by: Rob Gill --- automated install/basic-install.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4cd1e9aa..f3f76bd3 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -180,8 +180,6 @@ if command -v apt-get &> /dev/null; then phpInsMinor="$(echo $phpInsVer | cut -d\. -f2)" phpVer="php$phpInsMajor.phpInsMinor" fi - - fi # We also need the correct version for `php-sqlite` (which differs across distros) if ${PKG_MANAGER} install --dry-run ${phpVer}-sqlite3 > /dev/null 2>&1; then phpSqlite="sqlite3" From 714b6c98ef6da478eb46adab9ff34c1b50a6ba09 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Tue, 5 Jun 2018 14:45:01 +1000 Subject: [PATCH 065/163] codacy Signed-off-by: Rob Gill --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f3f76bd3..4c26f1a9 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -176,8 +176,8 @@ if command -v apt-get &> /dev/null; then fi else # Supported php is installed, its common, cgi & sqlite counterparts are deps - phpInsMajor="$(echo $phpInsVer | cut -d\. -f1)" - phpInsMinor="$(echo $phpInsVer | cut -d\. -f2)" + phpInsMajor="$(echo "$phpInsVersion" | cut -d\. -f1)" + phpInsMinor="$(echo "$phpInsVersion" | cut -d\. -f2)" phpVer="php$phpInsMajor.phpInsMinor" fi # We also need the correct version for `php-sqlite` (which differs across distros) From 36dbdf955d7ef06cefcdb4eed22c3bf617216172 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Tue, 5 Jun 2018 14:48:24 +1000 Subject: [PATCH 066/163] Update basic-install.sh Signed-off-by: Rob Gill --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4c26f1a9..481e26db 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -178,7 +178,7 @@ if command -v apt-get &> /dev/null; then # Supported php is installed, its common, cgi & sqlite counterparts are deps phpInsMajor="$(echo "$phpInsVersion" | cut -d\. -f1)" phpInsMinor="$(echo "$phpInsVersion" | cut -d\. -f2)" - phpVer="php$phpInsMajor.phpInsMinor" + phpVer="php$phpInsMajor.$phpInsMinor" fi # We also need the correct version for `php-sqlite` (which differs across distros) if ${PKG_MANAGER} install --dry-run ${phpVer}-sqlite3 > /dev/null 2>&1; then From 13b8e1a2aea2e1d0b57c32ce3d620b407c870d84 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Tue, 5 Jun 2018 15:09:07 +1000 Subject: [PATCH 067/163] quotes. Signed-off-by: Rob Gill --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 481e26db..a4869d24 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -166,7 +166,7 @@ if command -v apt-get &> /dev/null; then echo -e " ${INFO} Existing PHP installation detected : PHP version $phpInsVersion" fi # Check if installed php is supported version (5.4 is EOL) - if [[ $phpInsVersion < "5.5" ]]; then + if [[ "$phpInsVersion" < "5.5" ]]; then # Prefer the php metapackage if it's there if ${PKG_MANAGER} install --dry-run php > /dev/null 2>&1; then phpVer="php" From 7f81cfd45a50476e9654b76f9c4a0393841b75c6 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 5 Jun 2018 09:09:35 +0100 Subject: [PATCH 068/163] Stats can still be generated without logging, no need for this sentence Signed-off-by: Adam Warner --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 395a6203..3d9a902a 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -977,7 +977,7 @@ setLogging() { local LogChoices # Ask if the user wants to log queries - LogToggleCommand=(whiptail --separate-output --radiolist "Do you want to log queries?\\n (Disabling will render graphs on the Admin page useless):" ${r} ${c} 6) + LogToggleCommand=(whiptail --separate-output --radiolist "Do you want to log queries?" ${r} ${c} 6) # The default selection is on LogChooseOptions=("On (Recommended)" "" on Off "" off) From f2f6b6ede9fedf26a0a7b56f365746b7fdaaf14a Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 5 Jun 2018 09:29:03 +0100 Subject: [PATCH 069/163] stickler complaints Signed-off-by: Adam Warner --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 3d9a902a..3d2e5784 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -977,7 +977,7 @@ setLogging() { local LogChoices # Ask if the user wants to log queries - LogToggleCommand=(whiptail --separate-output --radiolist "Do you want to log queries?" ${r} ${c} 6) + LogToggleCommand=(whiptail --separate-output --radiolist "Do you want to log queries?" "${r}" "${c}" 6) # The default selection is on LogChooseOptions=("On (Recommended)" "" on Off "" off) From a3569d88c95ed2dae32753ffa25fab3a217e5f5b Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Tue, 5 Jun 2018 21:42:42 +1000 Subject: [PATCH 070/163] Version strings & bash. Fixed up the version checking. Thanks for your help @dschaper. No longer uses the version string as returned, but uses the major and minor version numbers extracted from it, against the minimum of 5.5. Tested against real install of php 7.0, (and the version check logic separately tested against a variety of artificial version numbers, of multiple digits for both major and minor version. - Lesson learned, I'm never trusting bash again) Signed-off-by: Rob Gill --- automated install/basic-install.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a4869d24..67cd3a4c 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -160,13 +160,19 @@ if command -v apt-get &> /dev/null; then # use iproute iproute_pkg="iproute" fi - # Check for and determine version number, major and minor of current php install + # Check for and determine version number (major and minor) of current php install if command -v php &> /dev/null; then - phpInsVersion="$(php -v | head -n1 | grep -Po '(? /dev/null 2>&1; then phpVer="php" @@ -176,8 +182,6 @@ if command -v apt-get &> /dev/null; then fi else # Supported php is installed, its common, cgi & sqlite counterparts are deps - phpInsMajor="$(echo "$phpInsVersion" | cut -d\. -f1)" - phpInsMinor="$(echo "$phpInsVersion" | cut -d\. -f2)" phpVer="php$phpInsMajor.$phpInsMinor" fi # We also need the correct version for `php-sqlite` (which differs across distros) From 80040806430417625fe624c58e9dd95f9c597656 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Wed, 6 Jun 2018 22:37:43 +1000 Subject: [PATCH 071/163] Additional command-line completion Signed-off-by: Rob Gill --- advanced/bash-completion/pihole | 74 +++++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 3 deletions(-) diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index fc8f2162..5a893f19 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -1,11 +1,79 @@ _pihole() { - local cur prev opts + local cur prev opts opts_admin opts_checkout opts_chronometer opts_debug opts_interface opts_logging opts_privacy opts_query opts_update opts_version COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" - opts="admin blacklist chronometer debug disable enable flush help logging query reconfigure restartdns setupLCD status tail uninstall updateGravity updatePihole version whitelist checkout" + prev2="${COMP_WORDS[COMP_CWORD-2]}" - COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + case "${prev}" in + "pihole") + opts="admin blacklist checkout chronometer debug disable enable flush help logging query reconfigure restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + ;; + "whitelist"|"blacklist"|"wildcard") + opts_lists="\--delmode \--noreload \--quiet \--list \--nuke" + COMPREPLY=( $(compgen -W "${opts_lists}" -- ${cur}) ) + ;; + "admin") + opts_admin="celsius email fahrenheit hostrecord interface kelvin password privacylevel" + COMPREPLY=( $(compgen -W "${opts_admin}" -- ${cur}) ) + ;; + "checkout") + opts_checkout="core ftl web master dev" + COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) + ;; + "chronometer") + opts_chronometer="\--exit \--jason \--refresh" + COMPREPLY=( $(compgen -W "${opts_chronometer}" -- ${cur}) ) + ;; + "debug") + opts_debug="-a" + COMPREPLY=( $(compgen -W "${opts_debug}" -- ${cur}) ) + ;; + "logging") + opts_logging="on off 'off noflush'" + COMPREPLY=( $(compgen -W "${opts_logging}" -- ${cur}) ) + ;; + "query") + opts_query="-adlist -all -exact" + COMPREPLY=( $(compgen -W "${opts_query}" -- ${cur}) ) + ;; + "updatePihole"|"-up") + opts_update="--check-only" + COMPREPLY=( $(compgen -W "${opts_update}" -- ${cur}) ) + ;; + "version") + opts_version="\--admin \--current \--ftl \--hash \--latest \--pihole" + COMPREPLY=( $(compgen -W "${opts_version}" -- ${cur}) ) + ;; + "interface") + if ( [[ "$prev2" == "admin" ]] || [[ "$prev2" == "-a" ]] ); then + opts_interface="$(cat /proc/net/dev | cut -d: -s -f1)" + COMPREPLY=( $(compgen -W "${opts_interface}" -- ${cur}) ) + else + return 1 + fi + ;; + "privacylevel") + if ( [[ "$prev2" == "admin" ]] || [[ "$prev2" == "-a" ]] ); then + opts_privacy="0 1 2 3" + COMPREPLY=( $(compgen -W "${opts_privacy}" -- ${cur}) ) + else + return 1 + fi + ;; + "core"|"admin"|"ftl") + if [[ "$prev2" == "checkout" ]]; then + opts_checkout="master dev" + COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) + else + return 1 + fi + ;; + *) + return 1 + ;; + esac return 0 } complete -F _pihole pihole From d254d6075afa7b2c71003c4fa3e103894d0be01d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 6 Jun 2018 19:26:56 +0200 Subject: [PATCH 072/163] First step from wildcards to regex lists for blocking Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 46 ++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 1d96ea3c..a54d7dc8 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -13,7 +13,7 @@ basename=pihole piholeDir=/etc/"${basename}" whitelist="${piholeDir}"/whitelist.txt blacklist="${piholeDir}"/blacklist.txt -readonly wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf" +readonly regexlist="/etc/pihole/regex.list" reload=false addmode=true verbose=true @@ -31,7 +31,7 @@ helpFunc() { if [[ "${listMain}" == "${whitelist}" ]]; then param="w" type="white" - elif [[ "${listMain}" == "${wildcardlist}" ]]; then + elif [[ "${listMain}" == "${regexlist}" ]]; then param="wild" type="wildcard black" else @@ -57,7 +57,8 @@ Options: EscapeRegexp() { # This way we may safely insert an arbitrary # string in our regular expressions - # Also remove leading "." if present + # This sed is intentionally executed in three steps to ease maintainability + # The first sed removes any amount of leading dots echo $* | sed 's/^\.*//' | sed "s/[]\.|$(){}?+*^]/\\\\&/g" | sed "s/\\//\\\\\//g" } @@ -94,9 +95,6 @@ PoplistFile() { if ${addmode}; then AddDomain "${dom}" "${listMain}" RemoveDomain "${dom}" "${listAlt}" - if [[ "${listMain}" == "${whitelist}" || "${listMain}" == "${blacklist}" ]]; then - RemoveDomain "${dom}" "${wildcardlist}" - fi else RemoveDomain "${dom}" "${listMain}" fi @@ -109,7 +107,6 @@ AddDomain() { [[ "${list}" == "${whitelist}" ]] && listname="whitelist" [[ "${list}" == "${blacklist}" ]] && listname="blacklist" - [[ "${list}" == "${wildcardlist}" ]] && listname="wildcard blacklist" if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only" @@ -121,7 +118,7 @@ AddDomain() { if [[ "${bool}" == false ]]; then # Domain not found in the whitelist file, add it! if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding $1 to $listname..." + echo -e " ${INFO} Adding ${1} to ${listname}..." fi reload=true # Add it to the list we want to add it to @@ -131,28 +128,22 @@ AddDomain() { echo -e " ${INFO} ${1} already exists in ${listname}, no need to add!" fi fi - elif [[ "${list}" == "${wildcardlist}" ]]; then - source "${piholeDir}/setupVars.conf" - # Remove the /* from the end of the IP addresses - IPV4_ADDRESS=${IPV4_ADDRESS%/*} - IPV6_ADDRESS=${IPV6_ADDRESS%/*} + elif [[ "${list}" == "${regexlist}" ]]; then [[ -z "${type}" ]] && type="--wildcard-only" bool=true # Is the domain in the list? - grep -e "address=\/${domain}\/" "${wildcardlist}" > /dev/null 2>&1 || bool=false + # Search only for exactly matching lines + grep -E "^${domain}$" "${regexlist}" > /dev/null 2>&1 || bool=false if [[ "${bool}" == false ]]; then if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding $1 to wildcard blacklist..." + echo -e " ${INFO} Adding ${1} to regex list..." fi reload="restart" - echo "address=/$1/${IPV4_ADDRESS}" >> "${wildcardlist}" - if [[ "${#IPV6_ADDRESS}" > 0 ]]; then - echo "address=/$1/${IPV6_ADDRESS}" >> "${wildcardlist}" - fi + echo "$1" >> "${regexlist}" else if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} already exists in wildcard blacklist, no need to add!" + echo -e " ${INFO} ${1} already exists in regex list, no need to add!" fi fi fi @@ -164,7 +155,6 @@ RemoveDomain() { [[ "${list}" == "${whitelist}" ]] && listname="whitelist" [[ "${list}" == "${blacklist}" ]] && listname="blacklist" - [[ "${list}" == "${wildcardlist}" ]] && listname="wildcard blacklist" if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then bool=true @@ -174,7 +164,7 @@ RemoveDomain() { grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false if [[ "${bool}" == true ]]; then # Remove it from the other one - echo -e " ${INFO} Removing $1 from $listname..." + echo -e " ${INFO} Removing $1 from ${listname}..." # /I flag: search case-insensitive sed -i "/${domain}/Id" "${list}" reload=true @@ -183,20 +173,20 @@ RemoveDomain() { echo -e " ${INFO} ${1} does not exist in ${listname}, no need to remove!" fi fi - elif [[ "${list}" == "${wildcardlist}" ]]; then + elif [[ "${list}" == "${regexlist}" ]]; then [[ -z "${type}" ]] && type="--wildcard-only" bool=true # Is it in the list? - grep -e "address=\/${domain}\/" "${wildcardlist}" > /dev/null 2>&1 || bool=false + grep -E "^${domain}$" "${regexlist}" > /dev/null 2>&1 || bool=false if [[ "${bool}" == true ]]; then # Remove it from the other one - echo -e " ${INFO} Removing $1 from $listname..." + echo -e " ${INFO} Removing $1 from regex list..." # /I flag: search case-insensitive - sed -i "/address=\/${domain}/Id" "${list}" + sed -i "/${domain}/Id" "${list}" reload=true else if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} does not exist in ${listname}, no need to remove!" + echo -e " ${INFO} ${1} does not exist in regex list, no need to remove!" fi fi fi @@ -241,7 +231,7 @@ for var in "$@"; do case "${var}" in "-w" | "whitelist" ) listMain="${whitelist}"; listAlt="${blacklist}";; "-b" | "blacklist" ) listMain="${blacklist}"; listAlt="${whitelist}";; - "-wild" | "wildcard" ) listMain="${wildcardlist}";; + "-wild" | "wildcard" ) listMain="${regexlist}";; "-nr"| "--noreload" ) reload=false;; "-d" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; From 52aa52c3b136e7bbeadc0646f75b91e3fc53b05e Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Thu, 7 Jun 2018 08:19:26 +1000 Subject: [PATCH 073/163] remove duplicate declaration Signed-off-by: Rob Gill --- automated install/basic-install.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index e1702187..4d825c60 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2028,7 +2028,6 @@ FTLinstall() { local ftlBranch local url - local ftlBranch if [[ -f "/etc/pihole/ftlbranch" ]];then ftlBranch=$( Date: Sat, 9 Jun 2018 10:30:04 +1000 Subject: [PATCH 074/163] --json not --jason Signed-off-by: Rob Gill --- advanced/bash-completion/pihole | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 5a893f19..ff8fee56 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -23,7 +23,7 @@ _pihole() { COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) ;; "chronometer") - opts_chronometer="\--exit \--jason \--refresh" + opts_chronometer="\--exit \--json \--refresh" COMPREPLY=( $(compgen -W "${opts_chronometer}" -- ${cur}) ) ;; "debug") From 6381bdbf334e6583e96cb3c0e2bb6e2f21507201 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Sat, 9 Jun 2018 11:35:30 +1000 Subject: [PATCH 075/163] don't use major.minor for PHP 5 Signed-off-by: --- automated install/basic-install.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 67cd3a4c..9fcb478b 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -166,13 +166,13 @@ if command -v apt-get &> /dev/null; then echo -e " ${INFO} Existing PHP installation detected : PHP version $phpInsVersion" phpInsMajor="$(echo "$phpInsVersion" | cut -d\. -f1)" phpInsMinor="$(echo "$phpInsVersion" | cut -d\. -f2)" - # Is installed php version supported? (php 5.4 is EOL) - if [ "$(echo "$phpInsMajor.$phpInsMinor < 5.5" | bc )" == 0 ]; then - phpInsSupported=true + # Is installed php version newer than php5? + if [ "$(echo "$phpInsMajor.$phpInsMinor < 7.0" | bc )" == 0 ]; then + phpInsNewer=true fi fi # Check if installed php is unsupported version (5.4 is EOL) - if [[ "$phpInsSupported" != true ]]; then + if [[ "$phpInsNewer" != true ]]; then # Prefer the php metapackage if it's there if ${PKG_MANAGER} install --dry-run php > /dev/null 2>&1; then phpVer="php" @@ -181,7 +181,7 @@ if command -v apt-get &> /dev/null; then phpVer="php5" fi else - # Supported php is installed, its common, cgi & sqlite counterparts are deps + # Newer php is installed, its common, cgi & sqlite counterparts are deps phpVer="php$phpInsMajor.$phpInsMinor" fi # We also need the correct version for `php-sqlite` (which differs across distros) From 11f0ade921aa0945cf9b20aad3f0097c92c4bd8d Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Sat, 9 Jun 2018 13:18:36 +1000 Subject: [PATCH 076/163] comments Signed-off-by: Rob Gill --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9fcb478b..5186b3cf 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -166,12 +166,12 @@ if command -v apt-get &> /dev/null; then echo -e " ${INFO} Existing PHP installation detected : PHP version $phpInsVersion" phpInsMajor="$(echo "$phpInsVersion" | cut -d\. -f1)" phpInsMinor="$(echo "$phpInsVersion" | cut -d\. -f2)" - # Is installed php version newer than php5? + # Is installed php version 7.0 or greater if [ "$(echo "$phpInsMajor.$phpInsMinor < 7.0" | bc )" == 0 ]; then phpInsNewer=true fi fi - # Check if installed php is unsupported version (5.4 is EOL) + # Check if installed php is v 7.0, or newer to determine packages to install if [[ "$phpInsNewer" != true ]]; then # Prefer the php metapackage if it's there if ${PKG_MANAGER} install --dry-run php > /dev/null 2>&1; then From e422f4154f4a05fa8bcd45e018328fb5c2d5d461 Mon Sep 17 00:00:00 2001 From: Alex Villarreal Date: Mon, 11 Jun 2018 12:56:27 -0500 Subject: [PATCH 077/163] Clean error message on successful response Signed-off-by: Alejandro Villarreal --- advanced/index.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/advanced/index.php b/advanced/index.php index 1575bafc..cad59ec7 100644 --- a/advanced/index.php +++ b/advanced/index.php @@ -329,6 +329,7 @@ setHeader(); setTimeout(function(){window.location.reload(1);}, 10000); $("#bpOutput").removeClass("add"); $("#bpOutput").addClass("success"); + $("#bpOutput").html(""); } else { $("#bpOutput").removeClass("add"); $("#bpOutput").addClass("error"); @@ -338,6 +339,7 @@ setHeader(); error: function(jqXHR, exception) { $("#bpOutput").removeClass("add"); $("#bpOutput").addClass("exception"); + $("#bpOutput").html(""); } }); } From d61fd01d61d95d86f79d1d041319adfc46c59c61 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Wed, 13 Jun 2018 15:47:08 +1000 Subject: [PATCH 078/163] Split queryFunc() into query.sh Signed-off-by: Rob Gill --- advanced/query.sh | 220 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 advanced/query.sh diff --git a/advanced/query.sh b/advanced/query.sh new file mode 100644 index 00000000..3bae7422 --- /dev/null +++ b/advanced/query.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +# Pi-hole: A black hole for Internet advertisements +# (c) 2018 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# Query Domain Lists +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + +# Globals +piholeDir="/etc/pihole" +adListsList="$piholeDir/adlists.list" +options="$*" +adlist="" +all="" +exact="" +blockpage="" +matchType="match" + +colfile="/opt/pihole/COL_TABLE" +source ${colfile} + +# Scan an array of files for matching strings +scanList(){ + # Escape full stops + local domain="${1//./\\.}" lists="${2}" type="${3:-}" + + # Prevent grep from printing file path + cd "$piholeDir" || exit 1 + + # Prevent grep -i matching slowly: http://bit.ly/2xFXtUX + export LC_CTYPE=C + + # /dev/null forces filename to be printed when only one list has been generated + # shellcheck disable=SC2086 + case "${type}" in + "exact" ) grep -i -E -l "(^|\\s)${domain}($|\\s|#)" ${lists} /dev/null;; + "wc" ) grep -i -o -m 1 "/${domain}/" ${lists};; + * ) grep -i "${domain}" ${lists} /dev/null;; + esac +} + +if [[ "${options}" == "-h" ]] || [[ "${options}" == "--help" ]]; then + echo "Usage: pihole -q [option] +Example: 'pihole -q -exact domain.com' +Query the adlists for a specified domain + +Options: + -adlist Print the name of the block list URL + -exact Search the block lists for exact domain matches + -all Return all query matches within a block list + -h, --help Show this help dialog" + exit 0 +fi + +if [[ ! -e "$adListsList" ]]; then + echo -e "${COL_LIGHT_RED}The file '/etc/pihole/adlists.list' was not found${COL_NC}" + exit 1 +fi + +# Handle valid options +if [[ "${options}" == *"-bp"* ]]; then + exact="exact"; blockpage=true +else + [[ "${options}" == *"-adlist"* ]] && adlist=true + [[ "${options}" == *"-all"* ]] && all=true + if [[ "${options}" == *"-exact"* ]]; then + exact="exact"; matchType="exact ${matchType}" + fi +fi + +# Strip valid options, leaving only the domain and invalid options +# This allows users to place the options before or after the domain +options=$(sed -E 's/ ?-(bp|adlists?|all|exact) ?//g' <<< "${options}") + +# Handle remaining options +# If $options contain non ASCII characters, convert to punycode +case "${options}" in + "" ) str="No domain specified";; + *" "* ) str="Unknown query option specified";; + *[![:ascii:]]* ) domainQuery=$(idn2 "${options}");; + * ) domainQuery="${options}";; +esac + +if [[ -n "${str:-}" ]]; then + echo -e "${str}${COL_NC}\\nTry 'pihole -q --help' for more information." + exit 1 +fi + +# Scan Whitelist and Blacklist +lists="whitelist.txt blacklist.txt" +mapfile -t results <<< "$(scanList "${domainQuery}" "${lists}" "${exact}")" + if [[ -n "${results[*]}" ]]; then + wbMatch=true + # Loop through each result in order to print unique file title once + for result in "${results[@]}"; do + fileName="${result%%.*}" + if [[ -n "${blockpage}" ]]; then + echo "Ï€ ${result}" + exit 0 + elif [[ -n "${exact}" ]]; then + echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" + else + # Only print filename title once per file + if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then + echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" + fileName_prev="${fileName}" + fi + echo " ${result#*:}" + fi + done +fi + +# Scan Wildcards +if [[ -e "${wildcardlist}" ]]; then + # Determine all subdomains, domain and TLDs + mapfile -t wildcards <<< "$(processWildcards "${domainQuery}")" + for match in "${wildcards[@]}"; do + # Search wildcard list for matches + mapfile -t results <<< "$(scanList "${match}" "${wildcardlist}" "wc")" + if [[ -n "${results[*]}" ]]; then + if [[ -z "${wcMatch:-}" ]] && [[ -z "${blockpage}" ]]; then + wcMatch=true + echo " ${matchType^} found in ${COL_BOLD}Wildcards${COL_NC}:" + fi + case "${blockpage}" in + true ) echo "Ï€ ${wildcardlist##*/}"; exit 0;; + * ) echo " *.${match}";; + esac + fi + done +fi + +# Get version sorted *.domains filenames (without dir path) +lists=("$(cd "$piholeDir" || exit 0; printf "%s\\n" -- *.domains | sort -V)") + +# Query blocklists for occurences of domain +mapfile -t results <<< "$(scanList "${domainQuery}" "${lists[*]}" "${exact}")" + +# Handle notices +if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then + echo -e " ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} within the block lists" + exit 0 +elif [[ -z "${results[*]}" ]]; then + # Result found in WL/BL/Wildcards + exit 0 +elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 100 ]]; then + echo -e " ${INFO} Over 100 ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} + This can be overridden using the -all option" + exit 0 +fi + +# Remove unwanted content from non-exact $results +if [[ -z "${exact}" ]]; then + # Delete lines starting with # + # Remove comments after domain + # Remove hosts format IP address + mapfile -t results <<< "$(IFS=$'\n'; sed \ + -e "/:#/d" \ + -e "s/[ \\t]#.*//g" \ + -e "s/:.*[ \\t]/:/g" \ + <<< "${results[*]}")" + # Exit if result was in a comment + [[ -z "${results[*]}" ]] && exit 0 +fi + +# Get adlist file content as array +if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then + for adlistUrl in $(< "adListsList"); do + if [[ "${adlistUrl:0:4}" =~ (http|www.) ]]; then + adlists+=("${adlistUrl}") + fi + done +fi + +# Print "Exact matches for" title +if [[ -n "${exact}" ]] && [[ -z "${blockpage}" ]]; then + plural=""; [[ "${#results[*]}" -gt 1 ]] && plural="es" + echo " ${matchType^}${plural} for ${COL_BOLD}${domainQuery}${COL_NC} found in:" +fi + +for result in "${results[@]}"; do + fileName="${result/:*/}" + + # Determine *.domains URL using filename's number + if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then + fileNum="${fileName/list./}"; fileNum="${fileNum%%.*}" + fileName="${adlists[$fileNum]}" + + # Discrepency occurs when adlists has been modified, but Gravity has not been run + if [[ -z "${fileName}" ]]; then + fileName="${COL_LIGHT_RED}(no associated adlists URL found)${COL_NC}" + fi + fi + + if [[ -n "${blockpage}" ]]; then + echo "${fileNum} ${fileName}" + elif [[ -n "${exact}" ]]; then + echo " ${fileName}" + else + if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then + count="" + echo " ${matchType^} found in ${COL_BOLD}${fileName}${COL_NC}:" + fileName_prev="${fileName}" + fi + : $((count++)) + + # Print matching domain if $max_count has not been reached + [[ -z "${all}" ]] && max_count="50" + if [[ -z "${all}" ]] && [[ "${count}" -ge "${max_count}" ]]; then + [[ "${count}" -gt "${max_count}" ]] && continue + echo " ${COL_GRAY}Over ${count} results found, skipping rest of file${COL_NC}" + else + echo " ${result#*:}" + fi + fi +done + +exit 0 From b1207949ac00ea984f57b2b4b9b233a2b33e1db8 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Wed, 13 Jun 2018 15:49:52 +1000 Subject: [PATCH 079/163] Call query.sh to replace queryFunc() Signed-off-by: Rob Gill --- pihole | 186 +-------------------------------------------------------- 1 file changed, 1 insertion(+), 185 deletions(-) diff --git a/pihole b/pihole index c125d0d9..b0cab88e 100755 --- a/pihole +++ b/pihole @@ -125,191 +125,7 @@ processWildcards() { queryFunc() { shift - local options="$*" adlist="" all="" exact="" blockpage="" matchType="match" - - if [[ "${options}" == "-h" ]] || [[ "${options}" == "--help" ]]; then - echo "Usage: pihole -q [option] -Example: 'pihole -q -exact domain.com' -Query the adlists for a specified domain - -Options: - -adlist Print the name of the block list URL - -exact Search the block lists for exact domain matches - -all Return all query matches within a block list - -h, --help Show this help dialog" - exit 0 - fi - - if [[ ! -e "/etc/pihole/adlists.list" ]]; then - echo -e "${COL_LIGHT_RED}The file '/etc/pihole/adlists.list' was not found${COL_NC}" - exit 1 - fi - - # Handle valid options - if [[ "${options}" == *"-bp"* ]]; then - exact="exact"; blockpage=true - else - [[ "${options}" == *"-adlist"* ]] && adlist=true - [[ "${options}" == *"-all"* ]] && all=true - if [[ "${options}" == *"-exact"* ]]; then - exact="exact"; matchType="exact ${matchType}" - fi - fi - - # Strip valid options, leaving only the domain and invalid options - # This allows users to place the options before or after the domain - options=$(sed -E 's/ ?-(bp|adlists?|all|exact) ?//g' <<< "${options}") - - # Handle remaining options - # If $options contain non ASCII characters, convert to punycode - case "${options}" in - "" ) str="No domain specified";; - *" "* ) str="Unknown query option specified";; - *[![:ascii:]]* ) domainQuery=$(idn2 "${options}");; - * ) domainQuery="${options}";; - esac - - if [[ -n "${str:-}" ]]; then - echo -e "${str}${COL_NC}\\nTry 'pihole -q --help' for more information." - exit 1 - fi - - # Scan Whitelist and Blacklist - lists="whitelist.txt blacklist.txt" - mapfile -t results <<< "$(scanList "${domainQuery}" "${lists}" "${exact}")" - - if [[ -n "${results[*]}" ]]; then - wbMatch=true - - # Loop through each result in order to print unique file title once - for result in "${results[@]}"; do - fileName="${result%%.*}" - - if [[ -n "${blockpage}" ]]; then - echo "Ï€ ${result}" - exit 0 - elif [[ -n "${exact}" ]]; then - echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" - else - # Only print filename title once per file - if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then - echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" - fileName_prev="${fileName}" - fi - echo " ${result#*:}" - fi - done - fi - - # Scan Wildcards - if [[ -e "${wildcardlist}" ]]; then - # Determine all subdomains, domain and TLDs - mapfile -t wildcards <<< "$(processWildcards "${domainQuery}")" - - for match in "${wildcards[@]}"; do - # Search wildcard list for matches - mapfile -t results <<< "$(scanList "${match}" "${wildcardlist}" "wc")" - - if [[ -n "${results[*]}" ]]; then - if [[ -z "${wcMatch:-}" ]] && [[ -z "${blockpage}" ]]; then - wcMatch=true - echo " ${matchType^} found in ${COL_BOLD}Wildcards${COL_NC}:" - fi - - case "${blockpage}" in - true ) echo "Ï€ ${wildcardlist##*/}"; exit 0;; - * ) echo " *.${match}";; - esac - fi - done - fi - - # Get version sorted *.domains filenames (without dir path) - lists=("$(cd "/etc/pihole" || exit 0; printf "%s\\n" -- *.domains | sort -V)") - - # Query blocklists for occurences of domain - mapfile -t results <<< "$(scanList "${domainQuery}" "${lists[*]}" "${exact}")" - - # Handle notices - if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then - echo -e " ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} within the block lists" - exit 0 - elif [[ -z "${results[*]}" ]]; then - # Result found in WL/BL/Wildcards - exit 0 - elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 100 ]]; then - echo -e " ${INFO} Over 100 ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} - This can be overridden using the -all option" - exit 0 - fi - - # Remove unwanted content from non-exact $results - if [[ -z "${exact}" ]]; then - # Delete lines starting with # - # Remove comments after domain - # Remove hosts format IP address - mapfile -t results <<< "$(IFS=$'\n'; sed \ - -e "/:#/d" \ - -e "s/[ \\t]#.*//g" \ - -e "s/:.*[ \\t]/:/g" \ - <<< "${results[*]}")" - - # Exit if result was in a comment - [[ -z "${results[*]}" ]] && exit 0 - fi - - # Get adlist file content as array - if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then - for adlistUrl in $(< "/etc/pihole/adlists.list"); do - if [[ "${adlistUrl:0:4}" =~ (http|www.) ]]; then - adlists+=("${adlistUrl}") - fi - done - fi - - # Print "Exact matches for" title - if [[ -n "${exact}" ]] && [[ -z "${blockpage}" ]]; then - plural=""; [[ "${#results[*]}" -gt 1 ]] && plural="es" - echo " ${matchType^}${plural} for ${COL_BOLD}${domainQuery}${COL_NC} found in:" - fi - - for result in "${results[@]}"; do - fileName="${result/:*/}" - - # Determine *.domains URL using filename's number - if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then - fileNum="${fileName/list./}"; fileNum="${fileNum%%.*}" - fileName="${adlists[$fileNum]}" - - # Discrepency occurs when adlists has been modified, but Gravity has not been run - if [[ -z "${fileName}" ]]; then - fileName="${COL_LIGHT_RED}(no associated adlists URL found)${COL_NC}" - fi - fi - - if [[ -n "${blockpage}" ]]; then - echo "${fileNum} ${fileName}" - elif [[ -n "${exact}" ]]; then - echo " ${fileName}" - else - if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then - count="" - echo " ${matchType^} found in ${COL_BOLD}${fileName}${COL_NC}:" - fileName_prev="${fileName}" - fi - : $((count++)) - - # Print matching domain if $max_count has not been reached - [[ -z "${all}" ]] && max_count="50" - if [[ -z "${all}" ]] && [[ "${count}" -ge "${max_count}" ]]; then - [[ "${count}" -gt "${max_count}" ]] && continue - echo " ${COL_GRAY}Over ${count} results found, skipping rest of file${COL_NC}" - else - echo " ${result#*:}" - fi - fi - done - + "${PI_HOLE_SCRIPT_DIR}"/query.sh "$@" exit 0 } From 45a8eda49b7d632ca4d6652558aa8d349fe2bbd3 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Wed, 13 Jun 2018 16:08:21 +1000 Subject: [PATCH 080/163] Stop grep leak in query Fix grep error leak from #1805 Signed-off-by: Rob Gill --- advanced/{ => Scripts}/query.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename advanced/{ => Scripts}/query.sh (97%) diff --git a/advanced/query.sh b/advanced/Scripts/query.sh similarity index 97% rename from advanced/query.sh rename to advanced/Scripts/query.sh index 3bae7422..88118ee1 100644 --- a/advanced/query.sh +++ b/advanced/Scripts/query.sh @@ -35,9 +35,9 @@ scanList(){ # /dev/null forces filename to be printed when only one list has been generated # shellcheck disable=SC2086 case "${type}" in - "exact" ) grep -i -E -l "(^|\\s)${domain}($|\\s|#)" ${lists} /dev/null;; - "wc" ) grep -i -o -m 1 "/${domain}/" ${lists};; - * ) grep -i "${domain}" ${lists} /dev/null;; + "exact" ) grep -i -E -l "(^|\\s)${domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;; + "wc" ) grep -i -o -m 1 "/${domain}/" ${lists} 2>/dev/null;; + * ) grep -i "${domain}" ${lists} /dev/null 2>/dev/null;; esac } From 2255d05664d07c581ca3a81eb37e20cff9c95a19 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Wed, 13 Jun 2018 16:09:49 +1000 Subject: [PATCH 081/163] Remove scanlist(), now in query.sh Signed-off-by: Rob Gill --- pihole | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/pihole b/pihole index b0cab88e..59ab760f 100755 --- a/pihole +++ b/pihole @@ -86,26 +86,6 @@ updateGravityFunc() { exit 0 } -# Scan an array of files for matching strings -scanList(){ - # Escape full stops - local domain="${1//./\\.}" lists="${2}" type="${3:-}" - - # Prevent grep from printing file path - cd "/etc/pihole" || exit 1 - - # Prevent grep -i matching slowly: http://bit.ly/2xFXtUX - export LC_CTYPE=C - - # /dev/null forces filename to be printed when only one list has been generated - # shellcheck disable=SC2086 - case "${type}" in - "exact" ) grep -i -E -l "(^|\\s)${domain}($|\\s|#)" ${lists} /dev/null;; - "wc" ) grep -i -o -m 1 "/${domain}/" ${lists};; - * ) grep -i "${domain}" ${lists} /dev/null;; - esac -} - # Print each subdomain # e.g: foo.bar.baz.com = "foo.bar.baz.com bar.baz.com baz.com com" processWildcards() { From b8e1849cec356d2ccfa5a9cd519636dd8858daa6 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Wed, 13 Jun 2018 16:19:07 +1000 Subject: [PATCH 082/163] wildcardlist Signed-off-by: Rob Gill --- advanced/Scripts/query.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 88118ee1..8b570ffe 100644 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -11,6 +11,7 @@ # Globals piholeDir="/etc/pihole" adListsList="$piholeDir/adlists.list" +wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf" options="$*" adlist="" all="" @@ -19,7 +20,7 @@ blockpage="" matchType="match" colfile="/opt/pihole/COL_TABLE" -source ${colfile} +source "$colfile" # Scan an array of files for matching strings scanList(){ From 8ab0b0e46073c6e06e03745113484d422bdf6afc Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Wed, 13 Jun 2018 16:25:43 +1000 Subject: [PATCH 083/163] colfile Signed-off-by: Rob Gill --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 8b570ffe..42f9c895 100644 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -20,7 +20,7 @@ blockpage="" matchType="match" colfile="/opt/pihole/COL_TABLE" -source "$colfile" +source "${colfile}" # Scan an array of files for matching strings scanList(){ From bf5566649251273f5be5a25593c40f73dfaa7c1d Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Wed, 13 Jun 2018 16:29:07 +1000 Subject: [PATCH 084/163] Appease stickler. Signed-off-by: Rob Gill --- advanced/Scripts/query.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 42f9c895..e1cd6457 100644 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck disable=SC1090 # Pi-hole: A black hole for Internet advertisements # (c) 2018 Pi-hole, LLC (https://pi-hole.net) # Network-wide ad blocking via your own hardware. From a7347238e6b22a9b78addef55809d59be7ca9e65 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Fri, 15 Jun 2018 14:42:30 +1000 Subject: [PATCH 085/163] $adListsList replace filename in text Signed-off-by: Rob Gill --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index e1cd6457..f9d25511 100644 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -57,7 +57,7 @@ Options: fi if [[ ! -e "$adListsList" ]]; then - echo -e "${COL_LIGHT_RED}The file '/etc/pihole/adlists.list' was not found${COL_NC}" + echo -e "${COL_LIGHT_RED}The file "$adListsList" was not found${COL_NC}" exit 1 fi From 23adbf9540dbe1034a87d81adc15d7b03f609a51 Mon Sep 17 00:00:00 2001 From: Rob Gill Date: Fri, 15 Jun 2018 14:45:27 +1000 Subject: [PATCH 086/163] remove quotes Signed-off-by: Rob Gill --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index f9d25511..5dc80f6b 100644 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -57,7 +57,7 @@ Options: fi if [[ ! -e "$adListsList" ]]; then - echo -e "${COL_LIGHT_RED}The file "$adListsList" was not found${COL_NC}" + echo -e "${COL_LIGHT_RED}The file $adListsList was not found${COL_NC}" exit 1 fi From a7c73036f0b7801feea2b643221d420caec84282 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 17 Jun 2018 13:37:41 +0200 Subject: [PATCH 087/163] pihole -t: Warn user if Pi-hole's logging is disabled Signed-off-by: DL6ER --- pihole | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pihole b/pihole index 59ab760f..21329849 100755 --- a/pihole +++ b/pihole @@ -312,6 +312,13 @@ statusFunc() { } tailFunc() { + # Warn user if Pi-hole's logging is disabled + local logging_enabled=$(grep -c "^log-queries" /etc/dnsmasq.d/01-pihole.conf) + if [[ "${logging_enabled}" == "0" ]]; then + # No "log-queries" lines are found. + # Commented out lines (such as "#log-queries") are ignored + echo " ${CROSS} Warning: Query logging is disabled" + fi echo -e " ${INFO} Press Ctrl-C to exit" # Retrieve IPv4/6 addresses From bc705aac039ba7c3fbfcd0aa73b0d991db2b8a14 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 17 Jun 2018 14:26:57 +0200 Subject: [PATCH 088/163] Add automated wildcard list -> regex filter conversion Signed-off-by: DL6ER --- advanced/Scripts/wildcard_regex_converter.sh | 18 ++++++++++++++++++ gravity.sh | 17 ++++++++--------- 2 files changed, 26 insertions(+), 9 deletions(-) create mode 100644 advanced/Scripts/wildcard_regex_converter.sh diff --git a/advanced/Scripts/wildcard_regex_converter.sh b/advanced/Scripts/wildcard_regex_converter.sh new file mode 100644 index 00000000..e862f391 --- /dev/null +++ b/advanced/Scripts/wildcard_regex_converter.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +wildcardFile="/etc/dnsmasq.d/03-pihole-wildcard.conf" + +convert_wildcard_to_regex() { + if [ ! -f "${wildcardFile}" ]; then + return + fi + local addrlines domains uniquedomains + # Obtain wildcard domains from old file + addrlines="$(grep -oE "/.*/" ${wildcardFile})" + # Strip "/" from domain names + domains="$(sed 's/\///g;' <<< "${addrlines}")" + # Remove repeated domains (may have been inserted two times due to A and AAAA blocking) + uniquedomains="$(uniq <<< "${domains}")" + # Automatically generate regex filters and remove old wildcards file + awk '{print "(^)|(\\.)"$0"$"}' <<< "${uniquedomains}" >> "${regexFile}" && rm "${wildcardFile}" +} diff --git a/gravity.sh b/gravity.sh index 58419029..0f64be68 100755 --- a/gravity.sh +++ b/gravity.sh @@ -15,6 +15,8 @@ export LC_ALL=C coltable="/opt/pihole/COL_TABLE" source "${coltable}" +regexconverter="/opt/pihole/wildcard_regex_converter.sh" +source "${regexconverter}" basename="pihole" PIHOLE_COMMAND="/usr/local/bin/${basename}" @@ -26,7 +28,7 @@ adListDefault="${piholeDir}/adlists.default" whitelistFile="${piholeDir}/whitelist.txt" blacklistFile="${piholeDir}/blacklist.txt" -wildcardFile="/etc/dnsmasq.d/03-pihole-wildcard.conf" +regexFile="${piholeDir}/regex.list" adList="${piholeDir}/gravity.list" blackList="${piholeDir}/black.list" @@ -453,7 +455,7 @@ gravity_Whitelist() { echo -e "${OVER} ${INFO} ${str}" } -# Output count of blacklisted domains and wildcards +# Output count of blacklisted domains and regex filters gravity_ShowBlockCount() { local num @@ -462,13 +464,9 @@ gravity_ShowBlockCount() { echo -e " ${INFO} Number of blacklisted domains: ${num}" fi - if [[ -f "${wildcardFile}" ]]; then - num=$(grep -c "^" "${wildcardFile}") - # If IPv4 and IPv6 is used, divide total wildcard count by 2 - if [[ -n "${IPV4_ADDRESS}" ]] && [[ -n "${IPV6_ADDRESS}" ]];then - num=$(( num/2 )) - fi - echo -e " ${INFO} Number of wildcard blocked domains: ${num}" + if [[ -f "${regexFile}" ]]; then + num=$(grep -c "^" "${regexFile}") + echo -e " ${INFO} Number of regex filters: ${num}" fi } @@ -646,6 +644,7 @@ if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "whitelist" ]]; then gravity_Whitelist fi +convert_wildcard_to_regex gravity_ShowBlockCount # Perform when downloading blocklists, or modifying the white/blacklist (not wildcards) From cd026cd865603bac7ca0e1b88b7e1bce5556c556 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 17 Jun 2018 14:43:33 +0200 Subject: [PATCH 089/163] Add comment where regexList is defined Signed-off-by: DL6ER --- advanced/Scripts/wildcard_regex_converter.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/advanced/Scripts/wildcard_regex_converter.sh b/advanced/Scripts/wildcard_regex_converter.sh index e862f391..7c80d07e 100644 --- a/advanced/Scripts/wildcard_regex_converter.sh +++ b/advanced/Scripts/wildcard_regex_converter.sh @@ -1,4 +1,14 @@ #!/bin/bash +# Pi-hole: A black hole for Internet advertisements +# (c) 2017 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# Provides an automated migration subroutine to convert Pi-hole v3.x wildcard domains to Pi-hole v4.x regex filters +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + +# regexFile set in gravity.sh wildcardFile="/etc/dnsmasq.d/03-pihole-wildcard.conf" From c7afe3e9a4a5728faf3c61722609b66d885be8b8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 17 Jun 2018 14:46:26 +0200 Subject: [PATCH 090/163] Please stickler by making clear that the variable regexFile is declared outside of this routine Signed-off-by: DL6ER --- advanced/Scripts/wildcard_regex_converter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/wildcard_regex_converter.sh b/advanced/Scripts/wildcard_regex_converter.sh index 7c80d07e..46c5e952 100644 --- a/advanced/Scripts/wildcard_regex_converter.sh +++ b/advanced/Scripts/wildcard_regex_converter.sh @@ -24,5 +24,5 @@ convert_wildcard_to_regex() { # Remove repeated domains (may have been inserted two times due to A and AAAA blocking) uniquedomains="$(uniq <<< "${domains}")" # Automatically generate regex filters and remove old wildcards file - awk '{print "(^)|(\\.)"$0"$"}' <<< "${uniquedomains}" >> "${regexFile}" && rm "${wildcardFile}" + awk '{print "(^)|(\\.)"$0"$"}' <<< "${uniquedomains}" >> "${regexFile:?}" && rm "${wildcardFile}" } From 2809579dd736dfe5f5a5273c55777baf59e3eb8e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 17 Jun 2018 14:59:21 +0200 Subject: [PATCH 091/163] Refine regex filter generation Signed-off-by: DL6ER --- advanced/Scripts/wildcard_regex_converter.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/wildcard_regex_converter.sh b/advanced/Scripts/wildcard_regex_converter.sh index 46c5e952..d8ebeeb8 100644 --- a/advanced/Scripts/wildcard_regex_converter.sh +++ b/advanced/Scripts/wildcard_regex_converter.sh @@ -19,10 +19,10 @@ convert_wildcard_to_regex() { local addrlines domains uniquedomains # Obtain wildcard domains from old file addrlines="$(grep -oE "/.*/" ${wildcardFile})" - # Strip "/" from domain names - domains="$(sed 's/\///g;' <<< "${addrlines}")" + # Strip "/" from domain names and convert "." to regex-compatible "\." + domains="$(sed 's/\///g;s/\./\\./g' <<< "${addrlines}")" # Remove repeated domains (may have been inserted two times due to A and AAAA blocking) uniquedomains="$(uniq <<< "${domains}")" # Automatically generate regex filters and remove old wildcards file - awk '{print "(^)|(\\.)"$0"$"}' <<< "${uniquedomains}" >> "${regexFile:?}" && rm "${wildcardFile}" + awk '{print "((^)|(\\.))"$0"$"}' <<< "${uniquedomains}" >> "${regexFile:?}" && rm "${wildcardFile}" } From e71492a2b3995cb123fe0266438be5d571da54f4 Mon Sep 17 00:00:00 2001 From: Mark Drobnak Date: Wed, 20 Jun 2018 20:19:38 -0400 Subject: [PATCH 092/163] Update debug script for simple list format Gravity is now just a list of domains, not IP addresses and domains separated by a space. Signed-off-by: Mcat12 --- advanced/Scripts/piholeDebug.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index a7fa4c2a..cffbfe82 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -780,7 +780,7 @@ dig_at() { # Find a random blocked url that has not been whitelisted. # This helps emulate queries to different domains that a user might query # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains - local random_url=$(shuf -n 1 "${PIHOLE_BLOCKLIST_FILE}" | awk -F ' ' '{ print $2 }') + local random_url=$(shuf -n 1 "${PIHOLE_BLOCKLIST_FILE}") # First, do a dig on localhost to see if Pi-hole can use itself to block a domain if local_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @${local_address} +short "${record_type}"); then From 868948509a4a7b70feff67537f709eb933cfc33c Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Wed, 20 Jun 2018 20:30:42 -0400 Subject: [PATCH 093/163] Split declaration and assignment of random_url Signed-off-by: Mcat12 --- advanced/Scripts/piholeDebug.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index cffbfe82..2f2b3745 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -780,7 +780,8 @@ dig_at() { # Find a random blocked url that has not been whitelisted. # This helps emulate queries to different domains that a user might query # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains - local random_url=$(shuf -n 1 "${PIHOLE_BLOCKLIST_FILE}") + local random_url + random_url=$(shuf -n 1 "${PIHOLE_BLOCKLIST_FILE}") # First, do a dig on localhost to see if Pi-hole can use itself to block a domain if local_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @${local_address} +short "${record_type}"); then From d5547f5c7c6eb450a85fde721579ac3835bad50c Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Sun, 24 Jun 2018 16:06:55 -0700 Subject: [PATCH 094/163] Revert "Fix error: /opt/pihole/gravity.sh: 385: Warning: command substitution: ignored null byte in input" --- gravity.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index ade27b8a..dfcb753a 100755 --- a/gravity.sh +++ b/gravity.sh @@ -387,8 +387,9 @@ gravity_ConsolidateDownloadedBlocklists() { if [[ -r "${i}" ]]; then # Remove windows CRs from file, convert list to lower case, and append into $matterAndLight tr -d '\r' < "${i}" | tr '[:upper:]' '[:lower:]' >> "${piholeDir}/${matterAndLight}" + # Ensure that the first line of a new list is on a new line - IFS= read -r -d '' lastLine <"${piholeDir}/${matterAndLight}" || [[ $lastLine ]] + lastLine=$(tail -1 "${piholeDir}/${matterAndLight}") if [[ "${#lastLine}" -gt 0 ]]; then echo "" >> "${piholeDir}/${matterAndLight}" fi From 9d3d33b6a276a7c94159b4f20429956615362093 Mon Sep 17 00:00:00 2001 From: bcambl Date: Tue, 26 Jun 2018 00:09:30 -0600 Subject: [PATCH 095/163] add tests for selinux checking Signed-off-by: bcambl --- .gitignore | 1 + test/test_automated_install.py | 50 ++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/.gitignore b/.gitignore index 73f14ae3..91bb6aff 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *.swp __pycache__ .cache +.pytest_cache # Created by https://www.gitignore.io/api/jetbrains+iml diff --git a/test/test_automated_install.py b/test/test_automated_install.py index 2c65c660..684b7004 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -173,6 +173,56 @@ def test_configureFirewall_IPTables_enabled_not_exist_no_errors(Pihole): assert 'iptables -I INPUT 1 -p tcp -m tcp --dport 53 -j ACCEPT' in firewall_calls assert 'iptables -I INPUT 1 -p udp -m udp --dport 53 -j ACCEPT' in firewall_calls +def test_selinux_enforcing_default_exit(Pihole): + ''' confirms installer prompts to exit when SELinux is Enforcing by default ''' + # getenforce returns the running state of SELinux + mock_command('getenforce', {'*':('Enforcing', '0')}, Pihole) + # Whiptail dialog returns Cancel for user prompt + mock_command('whiptail', {'*':('', '1')}, Pihole) + check_selinux = Pihole.run(''' + source /opt/pihole/basic-install.sh + checkSelinux + ''') + assert info_box + ' SELinux mode detected: Enforcing' in check_selinux.stdout + assert 'SELinux Enforcing detected, exiting installer' in check_selinux.stdout + assert check_selinux.rc == 1 + +def test_selinux_enforcing_continue(Pihole): + ''' confirms installer prompts to continue with custom policy warning ''' + # getenforce returns the running state of SELinux + mock_command('getenforce', {'*':('Enforcing', '0')}, Pihole) + # Whiptail dialog returns Cancel for user prompt + mock_command('whiptail', {'*':('', '0')}, Pihole) + check_selinux = Pihole.run(''' + source /opt/pihole/basic-install.sh + checkSelinux + ''') + assert info_box + ' SELinux mode detected: Enforcing' in check_selinux.stdout + assert info_box + ' Continuing installation with SELinux Enforcing' in check_selinux.stdout + assert info_box + ' Please refer to official SELinux documentation to create a custom policy' in check_selinux.stdout + assert check_selinux.rc == 0 + +def test_selinux_permissive(Pihole): + ''' confirms installer continues when SELinux is Permissive ''' + # getenforce returns the running state of SELinux + mock_command('getenforce', {'*':('Permissive', '0')}, Pihole) + check_selinux = Pihole.run(''' + source /opt/pihole/basic-install.sh + checkSelinux + ''') + assert info_box + ' SELinux mode detected: Permissive' in check_selinux.stdout + assert check_selinux.rc == 0 + +def test_selinux_disabled(Pihole): + ''' confirms installer continues when SELinux is Disabled ''' + mock_command('getenforce', {'*':('Disabled', '0')}, Pihole) + check_selinux = Pihole.run(''' + source /opt/pihole/basic-install.sh + checkSelinux + ''') + assert info_box + ' SELinux mode detected: Disabled' in check_selinux.stdout + assert check_selinux.rc == 0 + def test_installPiholeWeb_fresh_install_no_errors(Pihole): ''' confirms all web page assets from Core repo are installed on a fresh build ''' installWeb = Pihole.run(''' From b74fb3f179f6cc29f71d7284a86e9b4e1ea64134 Mon Sep 17 00:00:00 2001 From: Andrew Janke Date: Fri, 29 Jun 2018 00:28:43 +1000 Subject: [PATCH 096/163] typo fix --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 3152d9c1..2be62865 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -850,7 +850,7 @@ setDNS() { fi # Dialog for the user to enter custom upstream servers - piholeDNS=$(whiptail --backtitle "Specify Upstream DNS Provider(s)" --inputbox "Enter your desired upstream DNS provider(s), seperated by a comma.\\n\\nFor example '8.8.8.8, 8.8.4.4'" ${r} ${c} "${prePopulate}" 3>&1 1>&2 2>&3) || \ + piholeDNS=$(whiptail --backtitle "Specify Upstream DNS Provider(s)" --inputbox "Enter your desired upstream DNS provider(s), separated by a comma.\\n\\nFor example '8.8.8.8, 8.8.4.4'" ${r} ${c} "${prePopulate}" 3>&1 1>&2 2>&3) || \ { echo -e " ${COL_LIGHT_RED}Cancel was selected, exiting installer${COL_NC}"; exit 1; } # PIHOLE_DNS_1=$(echo "${piholeDNS}" | sed 's/[, \t]\+/,/g' | awk -F, '{print$1}') From 55175087c4738063b2c778befbc6517bd74c6c14 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Thu, 28 Jun 2018 22:52:44 -0400 Subject: [PATCH 097/163] Set regex file permissions for web access Signed-off-by: Mcat12 --- gravity.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gravity.sh b/gravity.sh index 0f64be68..aaab6237 100755 --- a/gravity.sh +++ b/gravity.sh @@ -644,6 +644,11 @@ if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "whitelist" ]]; then gravity_Whitelist fi +# Set proper permissions on the regex file +touch "${regexFile}" +chown pihole:www-data "${regexFile}" +chmod 664 "${regexFile}" + convert_wildcard_to_regex gravity_ShowBlockCount From cffb4de83b99361a46ced3ba33253fe69c987075 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Thu, 28 Jun 2018 22:59:15 -0400 Subject: [PATCH 098/163] Use strict grep instead of regex for duplicates Signed-off-by: Mcat12 --- advanced/Scripts/list.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index a54d7dc8..3a9c8ece 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -133,7 +133,7 @@ AddDomain() { bool=true # Is the domain in the list? # Search only for exactly matching lines - grep -E "^${domain}$" "${regexlist}" > /dev/null 2>&1 || bool=false + grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false if [[ "${bool}" == false ]]; then if [[ "${verbose}" == true ]]; then @@ -177,7 +177,7 @@ RemoveDomain() { [[ -z "${type}" ]] && type="--wildcard-only" bool=true # Is it in the list? - grep -E "^${domain}$" "${regexlist}" > /dev/null 2>&1 || bool=false + grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false if [[ "${bool}" == true ]]; then # Remove it from the other one echo -e " ${INFO} Removing $1 from regex list..." From 8435eeed4dc5b7bf62dd56a8db3c0760e78afd90 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Thu, 28 Jun 2018 23:21:01 -0400 Subject: [PATCH 099/163] Don't validate regex domains Signed-off-by: Mcat12 --- advanced/Scripts/list.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 3a9c8ece..04eed7db 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -66,10 +66,14 @@ HandleOther() { # Convert to lowercase domain="${1,,}" - # Check validity of domain + # Check validity of domain (don't check for regex entries) if [[ "${#domain}" -le 253 ]]; then - validDomain=$(grep -P "^((-|_)*[a-z\d]((-|_)*[a-z\d])*(-|_)*)(\.(-|_)*([a-z\d]((-|_)*[a-z\d])*))*$" <<< "${domain}") # Valid chars check - validDomain=$(grep -P "^[^\.]{1,63}(\.[^\.]{1,63})*$" <<< "${validDomain}") # Length of each label + if [[ "${listMain}" == "${regexlist}" ]]; then + validDomain="" + else + validDomain=$(grep -P "^((-|_)*[a-z\d]((-|_)*[a-z\d])*(-|_)*)(\.(-|_)*([a-z\d]((-|_)*[a-z\d])*))*$" <<< "${domain}") # Valid chars check + validDomain=$(grep -P "^[^\.]{1,63}(\.[^\.]{1,63})*$" <<< "${validDomain}") # Length of each label + fi fi if [[ -n "${validDomain}" ]]; then From 4a953b66e0e989307dcfd1114cd06b8f1f242ffa Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 29 Jun 2018 22:51:37 -0400 Subject: [PATCH 100/163] Make removing regex domains work correctly Signed-off-by: Mcat12 --- advanced/Scripts/list.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 04eed7db..38eea69f 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -69,7 +69,7 @@ HandleOther() { # Check validity of domain (don't check for regex entries) if [[ "${#domain}" -le 253 ]]; then if [[ "${listMain}" == "${regexlist}" ]]; then - validDomain="" + validDomain="${domain}" else validDomain=$(grep -P "^((-|_)*[a-z\d]((-|_)*[a-z\d])*(-|_)*)(\.(-|_)*([a-z\d]((-|_)*[a-z\d])*))*$" <<< "${domain}") # Valid chars check validDomain=$(grep -P "^[^\.]{1,63}(\.[^\.]{1,63})*$" <<< "${validDomain}") # Length of each label @@ -185,8 +185,9 @@ RemoveDomain() { if [[ "${bool}" == true ]]; then # Remove it from the other one echo -e " ${INFO} Removing $1 from regex list..." - # /I flag: search case-insensitive - sed -i "/${domain}/Id" "${list}" + local lineNumber + lineNumber=$(grep -Fnx "$1" "${list}" | cut -f1 -d:) + sed -i "${lineNumber}d" "${list}" reload=true else if [[ "${verbose}" == true ]]; then From 0decc37b5ad7371f5faa0c4cd1012e8ddab3b9f9 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 29 Jun 2018 22:55:00 -0400 Subject: [PATCH 101/163] Appease stickler Signed-off-by: Mcat12 --- advanced/Scripts/list.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 38eea69f..b2a15434 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -71,8 +71,8 @@ HandleOther() { if [[ "${listMain}" == "${regexlist}" ]]; then validDomain="${domain}" else - validDomain=$(grep -P "^((-|_)*[a-z\d]((-|_)*[a-z\d])*(-|_)*)(\.(-|_)*([a-z\d]((-|_)*[a-z\d])*))*$" <<< "${domain}") # Valid chars check - validDomain=$(grep -P "^[^\.]{1,63}(\.[^\.]{1,63})*$" <<< "${validDomain}") # Length of each label + validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check + validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label fi fi From 3077c22e4fa9cd7ebf69f759df3f70e03d53e67e Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 29 Jun 2018 23:10:49 -0400 Subject: [PATCH 102/163] Change help strings from wildcard to regex blacklist Signed-off-by: Mcat12 --- advanced/Scripts/list.sh | 2 +- pihole | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index b2a15434..bd80c3d8 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -33,7 +33,7 @@ helpFunc() { type="white" elif [[ "${listMain}" == "${regexlist}" ]]; then param="wild" - type="wildcard black" + type="regex black" else param="b" type="black" diff --git a/pihole b/pihole index b3f532af..8cda6938 100755 --- a/pihole +++ b/pihole @@ -599,7 +599,7 @@ Add '-h' after specific commands for more information on usage Whitelist/Blacklist Options: -w, whitelist Whitelist domain(s) -b, blacklist Blacklist domain(s) - -wild, wildcard Blacklist domain(s), and all its subdomains + -wild, wildcard Regex blacklist domain(s) Add '-h' for more info on whitelist/blacklist usage Debugging Options: From 61f0cbe10a1bd9f61b4f9307a2d4d41c765bc278 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 29 Jun 2018 23:34:15 -0400 Subject: [PATCH 103/163] Update pihole man page for regex Signed-off-by: Mcat12 --- manpages/pihole.8 | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 54bf4a31..76ca0c7c 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -68,7 +68,7 @@ Available commands and options: \fB-wild, wildcard\fR [options] [ ] .br - Add or removes specified domain, and all subdomains to the blacklist + Add or removes specified regex pattern to the regex blacklist .br (Whitelist/Blacklist manipulation options): @@ -167,9 +167,9 @@ Available commands and options: Show a help dialog .br -\fB-l, logging\fR [on|off|off noflush] +\fB-l, logging\fR [on|off|off noflush] .br - Specify whether the Pi-hole log should be used + Specify whether the Pi-hole log should be used .br (Logging options): @@ -193,7 +193,7 @@ Available commands and options: .br Show installed versions of Pi-hole, Web Interface & FTL .br - + .br (repo options): .br @@ -232,7 +232,7 @@ Available commands and options: Disable Pi-hole subsystems, optionally for a set duration .br - (time options): + (time options): .br #s Disable Pi-hole functionality for # second(s) .br @@ -275,9 +275,9 @@ Some usage examples \fBpihole -w iloveads.example.com\fR Add "iloveads.example.com" to whitelist .br - \fBpihole -b -d noads.example.com\fR Remove "noads.example.com" from blacklist + \fBpihole -b -d noads.example.com\fR Remove "noads.example.com" from blacklist .br - \fBpihole -wild example.com\fR Add "example.com" as wildcard - would block ads.example.com, www.example.com etc. + \fBpihole -wild ^example.*$\fR Add "^example.*$" as a regex pattern - would block all domains starting with "example" .br Changing the Web Interface password From f8680520629692a35096e60f8cf8da2d219d8653 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sat, 30 Jun 2018 15:50:43 -0400 Subject: [PATCH 104/163] Fix incorrect variable string substitution Signed-off-by: Mcat12 --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 5dc80f6b..e36a9d2b 100644 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -169,7 +169,7 @@ fi # Get adlist file content as array if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then - for adlistUrl in $(< "adListsList"); do + for adlistUrl in $(< "${adListsList}"); do if [[ "${adlistUrl:0:4}" =~ (http|www.) ]]; then adlists+=("${adlistUrl}") fi From 4763969c8f4a2651995cb4dc55b87e6159ea08ea Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sat, 30 Jun 2018 22:03:53 -0400 Subject: [PATCH 105/163] Move processWildcards to query.sh Closes #2255 Signed-off-by: Mcat12 --- advanced/Scripts/query.sh | 17 +++++++++++++++++ pihole | 17 ----------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index e36a9d2b..6bd37079 100644 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -23,6 +23,23 @@ matchType="match" colfile="/opt/pihole/COL_TABLE" source "${colfile}" +# Print each subdomain +# e.g: foo.bar.baz.com = "foo.bar.baz.com bar.baz.com baz.com com" +processWildcards() { + IFS="." read -r -a array <<< "${1}" + for (( i=${#array[@]}-1; i>=0; i-- )); do + ar="" + for (( j=${#array[@]}-1; j>${#array[@]}-i-2; j-- )); do + if [[ $j == $((${#array[@]}-1)) ]]; then + ar="${array[$j]}" + else + ar="${array[$j]}.${ar}" + fi + done + echo "${ar}" + done +} + # Scan an array of files for matching strings scanList(){ # Escape full stops diff --git a/pihole b/pihole index 21329849..0ee0c22f 100755 --- a/pihole +++ b/pihole @@ -86,23 +86,6 @@ updateGravityFunc() { exit 0 } -# Print each subdomain -# e.g: foo.bar.baz.com = "foo.bar.baz.com bar.baz.com baz.com com" -processWildcards() { - IFS="." read -r -a array <<< "${1}" - for (( i=${#array[@]}-1; i>=0; i-- )); do - ar="" - for (( j=${#array[@]}-1; j>${#array[@]}-i-2; j-- )); do - if [[ $j == $((${#array[@]}-1)) ]]; then - ar="${array[$j]}" - else - ar="${array[$j]}.${ar}" - fi - done - echo "${ar}" - done -} - queryFunc() { shift "${PI_HOLE_SCRIPT_DIR}"/query.sh "$@" From 7ddae8f2eb94fa751b346c24d8c256706f81ba0f Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Mon, 2 Jul 2018 19:59:22 +1000 Subject: [PATCH 106/163] Treat URLs without a protocol as HTTP Signed off by WaLLy3K --- gravity.sh | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/gravity.sh b/gravity.sh index dfcb753a..b7089ed1 100755 --- a/gravity.sh +++ b/gravity.sh @@ -218,8 +218,15 @@ gravity_DownloadBlocklistFromUrl() { httpCode=$(curl -s -L ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${patternBuffer}" 2> /dev/null) case $url in + # Did we "download" a local file? + "file"*) + if [[ -s "${patternBuffer}" ]]; then + echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true + else + echo -e "${OVER} ${CROSS} ${str} Not found / empty list" + fi;; # Did we "download" a remote file? - "http"*) + *) # Determine "Status:" output based on HTTP response case "${httpCode}" in "200") echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true;; @@ -233,16 +240,8 @@ gravity_DownloadBlocklistFromUrl() { "504") echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Gateway)";; "521") echo -e "${OVER} ${CROSS} ${str} Web Server Is Down (Cloudflare)";; "522") echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Cloudflare)";; - * ) echo -e "${OVER} ${CROSS} ${str} ${httpCode}";; + * ) echo -e "${OVER} ${CROSS} ${str} ${url} (${httpCode})";; esac;; - # Did we "download" a local file? - "file"*) - if [[ -s "${patternBuffer}" ]]; then - echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true - else - echo -e "${OVER} ${CROSS} ${str} Not found / empty list" - fi;; - *) echo -e "${OVER} ${CROSS} ${str} ${url} ${httpCode}";; esac # Determine if the blocklist was downloaded and saved correctly From 4468d81472cf43cb4d83909d3ca642499b6660a2 Mon Sep 17 00:00:00 2001 From: bcambl Date: Mon, 2 Jul 2018 14:54:19 -0600 Subject: [PATCH 107/163] python linting: 2 lines prior to defs (E302) Signed-off-by: bcambl --- test/conftest.py | 6 ++++++ test/test_000_build_containers.py | 1 + test/test_automated_install.py | 33 +++++++++++++++++++++++++++++++ test/test_shellcheck.py | 1 + 4 files changed, 41 insertions(+) diff --git a/test/conftest.py b/test/conftest.py index 5960cc24..44e0a4a4 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -5,6 +5,7 @@ check_output = testinfra.get_backend( "local://" ).get_module("Command").check_output + @pytest.fixture def Pihole(Docker): ''' used to contain some script stubbing, now pretty much an alias. @@ -25,6 +26,7 @@ def Pihole(Docker): Docker.run = funcType(run_bash, Docker, testinfra.backend.docker.DockerBackend) return Docker + @pytest.fixture def Docker(request, args, image, cmd): ''' combine our fixtures into a docker run command and setup finalizer to cleanup ''' @@ -40,21 +42,25 @@ def Docker(request, args, image, cmd): docker_container.id = docker_id return docker_container + @pytest.fixture def args(request): ''' -t became required when tput began being used ''' return '-t -d' + @pytest.fixture(params=['debian', 'centos']) def tag(request): ''' consumed by image to make the test matrix ''' return request.param + @pytest.fixture() def image(request, tag): ''' built by test_000_build_containers.py ''' return 'pytest_pihole:{}'.format(tag) + @pytest.fixture() def cmd(request): ''' default to doing nothing by tailing null, but don't exit ''' diff --git a/test/test_000_build_containers.py b/test/test_000_build_containers.py index c617f3ae..725136d8 100644 --- a/test/test_000_build_containers.py +++ b/test/test_000_build_containers.py @@ -6,6 +6,7 @@ run_local = testinfra.get_backend( "local://" ).get_module("Command").run + @pytest.mark.parametrize("image,tag", [ ( 'test/debian.Dockerfile', 'pytest_pihole:debian' ), ( 'test/centos.Dockerfile', 'pytest_pihole:centos' ), diff --git a/test/test_automated_install.py b/test/test_automated_install.py index 684b7004..f7b8702d 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -13,6 +13,7 @@ tick_box="[\x1b[1;32m\xe2\x9c\x93\x1b[0m]".decode("utf-8") cross_box="[\x1b[1;31m\xe2\x9c\x97\x1b[0m]".decode("utf-8") info_box="[i]".decode("utf-8") + def test_setupVars_are_sourced_to_global_scope(Pihole): ''' currently update_dialogs sources setupVars with a dot, then various other functions use the variables. @@ -46,6 +47,7 @@ def test_setupVars_are_sourced_to_global_scope(Pihole): for k,v in SETUPVARS.iteritems(): assert "{}={}".format(k, v) in output + def test_setupVars_saved_to_file(Pihole): ''' confirm saved settings are written to a file for future updates to re-use ''' set_setup_vars = '\n' # dedent works better with this and padding matching script below @@ -70,6 +72,7 @@ def test_setupVars_saved_to_file(Pihole): for k,v in SETUPVARS.iteritems(): assert "{}={}".format(k, v) in output + def test_configureFirewall_firewalld_running_no_errors(Pihole): ''' confirms firewalld rules are applied when firewallD is running ''' # firewallD returns 'running' as status @@ -87,6 +90,7 @@ def test_configureFirewall_firewalld_running_no_errors(Pihole): assert 'firewall-cmd --permanent --add-service=http --add-service=dns' in firewall_calls assert 'firewall-cmd --reload' in firewall_calls + def test_configureFirewall_firewalld_disabled_no_errors(Pihole): ''' confirms firewalld rules are not applied when firewallD is not running ''' # firewallD returns non-running status @@ -98,6 +102,7 @@ def test_configureFirewall_firewalld_disabled_no_errors(Pihole): expected_stdout = 'No active firewall detected.. skipping firewall configuration' assert expected_stdout in configureFirewall.stdout + def test_configureFirewall_firewalld_enabled_declined_no_errors(Pihole): ''' confirms firewalld rules are not applied when firewallD is running, user declines ruleset ''' # firewallD returns running status @@ -111,6 +116,7 @@ def test_configureFirewall_firewalld_enabled_declined_no_errors(Pihole): expected_stdout = 'Not installing firewall rulesets.' assert expected_stdout in configureFirewall.stdout + def test_configureFirewall_no_firewall(Pihole): ''' confirms firewall skipped no daemon is running ''' configureFirewall = Pihole.run(''' @@ -120,6 +126,7 @@ def test_configureFirewall_no_firewall(Pihole): expected_stdout = 'No active firewall detected' assert expected_stdout in configureFirewall.stdout + def test_configureFirewall_IPTables_enabled_declined_no_errors(Pihole): ''' confirms IPTables rules are not applied when IPTables is running, user declines ruleset ''' # iptables command exists @@ -135,6 +142,7 @@ def test_configureFirewall_IPTables_enabled_declined_no_errors(Pihole): expected_stdout = 'Not installing firewall rulesets.' assert expected_stdout in configureFirewall.stdout + def test_configureFirewall_IPTables_enabled_rules_exist_no_errors(Pihole): ''' confirms IPTables rules are not applied when IPTables is running and rules exist ''' # iptables command exists and returns 0 on calls (should return 0 on iptables -C) @@ -154,6 +162,7 @@ def test_configureFirewall_IPTables_enabled_rules_exist_no_errors(Pihole): assert 'iptables -I INPUT 1 -p tcp -m tcp --dport 53 -j ACCEPT' not in firewall_calls assert 'iptables -I INPUT 1 -p udp -m udp --dport 53 -j ACCEPT' not in firewall_calls + def test_configureFirewall_IPTables_enabled_not_exist_no_errors(Pihole): ''' confirms IPTables rules are applied when IPTables is running and rules do not exist ''' # iptables command and returns 0 on calls (should return 1 on iptables -C) @@ -173,6 +182,7 @@ def test_configureFirewall_IPTables_enabled_not_exist_no_errors(Pihole): assert 'iptables -I INPUT 1 -p tcp -m tcp --dport 53 -j ACCEPT' in firewall_calls assert 'iptables -I INPUT 1 -p udp -m udp --dport 53 -j ACCEPT' in firewall_calls + def test_selinux_enforcing_default_exit(Pihole): ''' confirms installer prompts to exit when SELinux is Enforcing by default ''' # getenforce returns the running state of SELinux @@ -187,6 +197,7 @@ def test_selinux_enforcing_default_exit(Pihole): assert 'SELinux Enforcing detected, exiting installer' in check_selinux.stdout assert check_selinux.rc == 1 + def test_selinux_enforcing_continue(Pihole): ''' confirms installer prompts to continue with custom policy warning ''' # getenforce returns the running state of SELinux @@ -202,6 +213,7 @@ def test_selinux_enforcing_continue(Pihole): assert info_box + ' Please refer to official SELinux documentation to create a custom policy' in check_selinux.stdout assert check_selinux.rc == 0 + def test_selinux_permissive(Pihole): ''' confirms installer continues when SELinux is Permissive ''' # getenforce returns the running state of SELinux @@ -213,6 +225,7 @@ def test_selinux_permissive(Pihole): assert info_box + ' SELinux mode detected: Permissive' in check_selinux.stdout assert check_selinux.rc == 0 + def test_selinux_disabled(Pihole): ''' confirms installer continues when SELinux is Disabled ''' mock_command('getenforce', {'*':('Disabled', '0')}, Pihole) @@ -223,6 +236,7 @@ def test_selinux_disabled(Pihole): assert info_box + ' SELinux mode detected: Disabled' in check_selinux.stdout assert check_selinux.rc == 0 + def test_installPiholeWeb_fresh_install_no_errors(Pihole): ''' confirms all web page assets from Core repo are installed on a fresh build ''' installWeb = Pihole.run(''' @@ -238,6 +252,7 @@ def test_installPiholeWeb_fresh_install_no_errors(Pihole): assert 'index.php' in web_directory assert 'blockingpage.css' in web_directory + def test_update_package_cache_success_no_errors(Pihole): ''' confirms package cache was updated without any errors''' updateCache = Pihole.run(''' @@ -248,6 +263,7 @@ def test_update_package_cache_success_no_errors(Pihole): assert tick_box + ' Update local cache of available packages' in updateCache.stdout assert 'Error: Unable to update package cache.' not in updateCache.stdout + def test_update_package_cache_failure_no_errors(Pihole): ''' confirms package cache was not updated''' mock_command('apt-get', {'update':('', '1')}, Pihole) @@ -259,6 +275,7 @@ def test_update_package_cache_failure_no_errors(Pihole): assert cross_box + ' Update local cache of available packages' in updateCache.stdout assert 'Error: Unable to update package cache.' in updateCache.stdout + def test_FTL_detect_aarch64_no_errors(Pihole): ''' confirms only aarch64 package is downloaded for FTL engine ''' # mock uname to return aarch64 platform @@ -276,6 +293,7 @@ def test_FTL_detect_aarch64_no_errors(Pihole): expected_stdout = tick_box + ' Downloading and Installing FTL' assert expected_stdout in detectPlatform.stdout + def test_FTL_detect_armv6l_no_errors(Pihole): ''' confirms only armv6l package is downloaded for FTL engine ''' # mock uname to return armv6l platform @@ -293,6 +311,7 @@ def test_FTL_detect_armv6l_no_errors(Pihole): expected_stdout = tick_box + ' Downloading and Installing FTL' assert expected_stdout in detectPlatform.stdout + def test_FTL_detect_armv7l_no_errors(Pihole): ''' confirms only armv7l package is downloaded for FTL engine ''' # mock uname to return armv7l platform @@ -310,6 +329,7 @@ def test_FTL_detect_armv7l_no_errors(Pihole): expected_stdout = tick_box + ' Downloading and Installing FTL' assert expected_stdout in detectPlatform.stdout + def test_FTL_detect_x86_64_no_errors(Pihole): ''' confirms only x86_64 package is downloaded for FTL engine ''' detectPlatform = Pihole.run(''' @@ -323,6 +343,7 @@ def test_FTL_detect_x86_64_no_errors(Pihole): expected_stdout = tick_box + ' Downloading and Installing FTL' assert expected_stdout in detectPlatform.stdout + def test_FTL_detect_unknown_no_errors(Pihole): ''' confirms only generic package is downloaded for FTL engine ''' # mock uname to return generic platform @@ -334,6 +355,7 @@ def test_FTL_detect_unknown_no_errors(Pihole): expected_stdout = 'Not able to detect architecture (unknown: mips)' assert expected_stdout in detectPlatform.stdout + def test_FTL_download_aarch64_no_errors(Pihole): ''' confirms only aarch64 package is downloaded for FTL engine ''' # mock uname to return generic platform @@ -348,6 +370,7 @@ def test_FTL_download_aarch64_no_errors(Pihole): error = 'Error: URL not found' assert error not in download_binary.stdout + def test_FTL_download_unknown_fails_no_errors(Pihole): ''' confirms unknown binary is not downloaded for FTL engine ''' # mock uname to return generic platform @@ -360,6 +383,7 @@ def test_FTL_download_unknown_fails_no_errors(Pihole): error = 'Error: URL not found' assert error in download_binary.stdout + def test_FTL_binary_installed_and_responsive_no_errors(Pihole): ''' confirms FTL binary is copied and functional in installed location ''' installed_binary = Pihole.run(''' @@ -370,6 +394,7 @@ def test_FTL_binary_installed_and_responsive_no_errors(Pihole): expected_stdout = 'v' assert expected_stdout in installed_binary.stdout + # def test_FTL_support_files_installed(Pihole): # ''' confirms FTL support files are installed ''' # support_files = Pihole.run(''' @@ -384,6 +409,7 @@ def test_FTL_binary_installed_and_responsive_no_errors(Pihole): # assert '644 /run/pihole-FTL.pid' in support_files.stdout # assert '644 /var/log/pihole-FTL.log' in support_files.stdout + def test_IPv6_only_link_local(Pihole): ''' confirms IPv6 blocking is disabled for Link-local address ''' # mock ip -6 address to return Link-local address @@ -395,6 +421,7 @@ def test_IPv6_only_link_local(Pihole): expected_stdout = 'Unable to find IPv6 ULA/GUA address, IPv6 adblocking will not be enabled' assert expected_stdout in detectPlatform.stdout + def test_IPv6_only_ULA(Pihole): ''' confirms IPv6 blocking is enabled for ULA addresses ''' # mock ip -6 address to return ULA address @@ -406,6 +433,7 @@ def test_IPv6_only_ULA(Pihole): expected_stdout = 'Found IPv6 ULA address, using it for blocking IPv6 ads' assert expected_stdout in detectPlatform.stdout + def test_IPv6_only_GUA(Pihole): ''' confirms IPv6 blocking is enabled for GUA addresses ''' # mock ip -6 address to return GUA address @@ -417,6 +445,7 @@ def test_IPv6_only_GUA(Pihole): expected_stdout = 'Found IPv6 GUA address, using it for blocking IPv6 ads' assert expected_stdout in detectPlatform.stdout + def test_IPv6_GUA_ULA_test(Pihole): ''' confirms IPv6 blocking is enabled for GUA and ULA addresses ''' # mock ip -6 address to return GUA and ULA addresses @@ -428,6 +457,7 @@ def test_IPv6_GUA_ULA_test(Pihole): expected_stdout = 'Found IPv6 ULA address, using it for blocking IPv6 ads' assert expected_stdout in detectPlatform.stdout + def test_IPv6_ULA_GUA_test(Pihole): ''' confirms IPv6 blocking is enabled for GUA and ULA addresses ''' # mock ip -6 address to return ULA and GUA addresses @@ -439,6 +469,7 @@ def test_IPv6_ULA_GUA_test(Pihole): expected_stdout = 'Found IPv6 ULA address, using it for blocking IPv6 ads' assert expected_stdout in detectPlatform.stdout + # Helper functions def mock_command(script, args, container): ''' Allows for setup of commands we don't really want to have to run for real in unit tests ''' @@ -461,6 +492,7 @@ def mock_command(script, args, container): chmod +x {script} rm -f /var/log/{scriptlog}'''.format(script=full_script_path, content=mock_script, scriptlog=script)) + def mock_command_2(script, args, container): ''' Allows for setup of commands we don't really want to have to run for real in unit tests ''' full_script_path = '/usr/local/bin/{}'.format(script) @@ -482,6 +514,7 @@ def mock_command_2(script, args, container): chmod +x {script} rm -f /var/log/{scriptlog}'''.format(script=full_script_path, content=mock_script, scriptlog=script)) + def run_script(Pihole, script): result = Pihole.run(script) assert result.rc == 0 diff --git a/test/test_shellcheck.py b/test/test_shellcheck.py index 5b1a8961..0bd03f36 100644 --- a/test/test_shellcheck.py +++ b/test/test_shellcheck.py @@ -5,6 +5,7 @@ run_local = testinfra.get_backend( "local://" ).get_module("Command").run + def test_scripts_pass_shellcheck(): ''' Make sure shellcheck does not find anything wrong with our shell scripts ''' shellcheck = "find . -type f -name 'update.sh' | while read file; do shellcheck -x \"$file\" -e SC1090,SC1091; done;" From da3dfd0998c1cd86afdf3ecf6cec7bc95b77ac57 Mon Sep 17 00:00:00 2001 From: bcambl Date: Mon, 2 Jul 2018 15:25:51 -0600 Subject: [PATCH 108/163] python linting: missing whitespace after ':' (E231) Signed-off-by: bcambl --- test/test_automated_install.py | 66 +++++++++++++++++----------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/test/test_automated_install.py b/test/test_automated_install.py index f7b8702d..55709af9 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -76,9 +76,9 @@ def test_setupVars_saved_to_file(Pihole): def test_configureFirewall_firewalld_running_no_errors(Pihole): ''' confirms firewalld rules are applied when firewallD is running ''' # firewallD returns 'running' as status - mock_command('firewall-cmd', {'*':('running', 0)}, Pihole) + mock_command('firewall-cmd', {'*': ('running', 0)}, Pihole) # Whiptail dialog returns Ok for user prompt - mock_command('whiptail', {'*':('', 0)}, Pihole) + mock_command('whiptail', {'*': ('', 0)}, Pihole) configureFirewall = Pihole.run(''' source /opt/pihole/basic-install.sh configureFirewall @@ -94,7 +94,7 @@ def test_configureFirewall_firewalld_running_no_errors(Pihole): def test_configureFirewall_firewalld_disabled_no_errors(Pihole): ''' confirms firewalld rules are not applied when firewallD is not running ''' # firewallD returns non-running status - mock_command('firewall-cmd', {'*':('not running', '1')}, Pihole) + mock_command('firewall-cmd', {'*': ('not running', '1')}, Pihole) configureFirewall = Pihole.run(''' source /opt/pihole/basic-install.sh configureFirewall @@ -106,9 +106,9 @@ def test_configureFirewall_firewalld_disabled_no_errors(Pihole): def test_configureFirewall_firewalld_enabled_declined_no_errors(Pihole): ''' confirms firewalld rules are not applied when firewallD is running, user declines ruleset ''' # firewallD returns running status - mock_command('firewall-cmd', {'*':('running', 0)}, Pihole) + mock_command('firewall-cmd', {'*': ('running', 0)}, Pihole) # Whiptail dialog returns Cancel for user prompt - mock_command('whiptail', {'*':('', 1)}, Pihole) + mock_command('whiptail', {'*': ('', 1)}, Pihole) configureFirewall = Pihole.run(''' source /opt/pihole/basic-install.sh configureFirewall @@ -130,11 +130,11 @@ def test_configureFirewall_no_firewall(Pihole): def test_configureFirewall_IPTables_enabled_declined_no_errors(Pihole): ''' confirms IPTables rules are not applied when IPTables is running, user declines ruleset ''' # iptables command exists - mock_command('iptables', {'*':('', '0')}, Pihole) + mock_command('iptables', {'*': ('', '0')}, Pihole) # modinfo returns always true (ip_tables module check) - mock_command('modinfo', {'*':('', '0')}, Pihole) + mock_command('modinfo', {'*': ('', '0')}, Pihole) # Whiptail dialog returns Cancel for user prompt - mock_command('whiptail', {'*':('', '1')}, Pihole) + mock_command('whiptail', {'*': ('', '1')}, Pihole) configureFirewall = Pihole.run(''' source /opt/pihole/basic-install.sh configureFirewall @@ -146,11 +146,11 @@ def test_configureFirewall_IPTables_enabled_declined_no_errors(Pihole): def test_configureFirewall_IPTables_enabled_rules_exist_no_errors(Pihole): ''' confirms IPTables rules are not applied when IPTables is running and rules exist ''' # iptables command exists and returns 0 on calls (should return 0 on iptables -C) - mock_command('iptables', {'-S':('-P INPUT DENY', '0')}, Pihole) + mock_command('iptables', {'-S': ('-P INPUT DENY', '0')}, Pihole) # modinfo returns always true (ip_tables module check) - mock_command('modinfo', {'*':('', '0')}, Pihole) + mock_command('modinfo', {'*': ('', '0')}, Pihole) # Whiptail dialog returns Cancel for user prompt - mock_command('whiptail', {'*':('', '0')}, Pihole) + mock_command('whiptail', {'*': ('', '0')}, Pihole) configureFirewall = Pihole.run(''' source /opt/pihole/basic-install.sh configureFirewall @@ -166,11 +166,11 @@ def test_configureFirewall_IPTables_enabled_rules_exist_no_errors(Pihole): def test_configureFirewall_IPTables_enabled_not_exist_no_errors(Pihole): ''' confirms IPTables rules are applied when IPTables is running and rules do not exist ''' # iptables command and returns 0 on calls (should return 1 on iptables -C) - mock_command('iptables', {'-S':('-P INPUT DENY', '0'), '-C':('', 1), '-I':('', 0)}, Pihole) + mock_command('iptables', {'-S': ('-P INPUT DENY', '0'), '-C': ('', 1), '-I': ('', 0)}, Pihole) # modinfo returns always true (ip_tables module check) - mock_command('modinfo', {'*':('', '0')}, Pihole) + mock_command('modinfo', {'*': ('', '0')}, Pihole) # Whiptail dialog returns Cancel for user prompt - mock_command('whiptail', {'*':('', '0')}, Pihole) + mock_command('whiptail', {'*': ('', '0')}, Pihole) configureFirewall = Pihole.run(''' source /opt/pihole/basic-install.sh configureFirewall @@ -186,9 +186,9 @@ def test_configureFirewall_IPTables_enabled_not_exist_no_errors(Pihole): def test_selinux_enforcing_default_exit(Pihole): ''' confirms installer prompts to exit when SELinux is Enforcing by default ''' # getenforce returns the running state of SELinux - mock_command('getenforce', {'*':('Enforcing', '0')}, Pihole) + mock_command('getenforce', {'*': ('Enforcing', '0')}, Pihole) # Whiptail dialog returns Cancel for user prompt - mock_command('whiptail', {'*':('', '1')}, Pihole) + mock_command('whiptail', {'*': ('', '1')}, Pihole) check_selinux = Pihole.run(''' source /opt/pihole/basic-install.sh checkSelinux @@ -201,9 +201,9 @@ def test_selinux_enforcing_default_exit(Pihole): def test_selinux_enforcing_continue(Pihole): ''' confirms installer prompts to continue with custom policy warning ''' # getenforce returns the running state of SELinux - mock_command('getenforce', {'*':('Enforcing', '0')}, Pihole) + mock_command('getenforce', {'*': ('Enforcing', '0')}, Pihole) # Whiptail dialog returns Cancel for user prompt - mock_command('whiptail', {'*':('', '0')}, Pihole) + mock_command('whiptail', {'*': ('', '0')}, Pihole) check_selinux = Pihole.run(''' source /opt/pihole/basic-install.sh checkSelinux @@ -217,7 +217,7 @@ def test_selinux_enforcing_continue(Pihole): def test_selinux_permissive(Pihole): ''' confirms installer continues when SELinux is Permissive ''' # getenforce returns the running state of SELinux - mock_command('getenforce', {'*':('Permissive', '0')}, Pihole) + mock_command('getenforce', {'*': ('Permissive', '0')}, Pihole) check_selinux = Pihole.run(''' source /opt/pihole/basic-install.sh checkSelinux @@ -228,7 +228,7 @@ def test_selinux_permissive(Pihole): def test_selinux_disabled(Pihole): ''' confirms installer continues when SELinux is Disabled ''' - mock_command('getenforce', {'*':('Disabled', '0')}, Pihole) + mock_command('getenforce', {'*': ('Disabled', '0')}, Pihole) check_selinux = Pihole.run(''' source /opt/pihole/basic-install.sh checkSelinux @@ -266,7 +266,7 @@ def test_update_package_cache_success_no_errors(Pihole): def test_update_package_cache_failure_no_errors(Pihole): ''' confirms package cache was not updated''' - mock_command('apt-get', {'update':('', '1')}, Pihole) + mock_command('apt-get', {'update': ('', '1')}, Pihole) updateCache = Pihole.run(''' source /opt/pihole/basic-install.sh distro_check @@ -279,9 +279,9 @@ def test_update_package_cache_failure_no_errors(Pihole): def test_FTL_detect_aarch64_no_errors(Pihole): ''' confirms only aarch64 package is downloaded for FTL engine ''' # mock uname to return aarch64 platform - mock_command('uname', {'-m':('aarch64', '0')}, Pihole) + mock_command('uname', {'-m': ('aarch64', '0')}, Pihole) # mock ldd to respond with aarch64 shared library - mock_command('ldd', {'/bin/ls':('/lib/ld-linux-aarch64.so.1', '0')}, Pihole) + mock_command('ldd', {'/bin/ls': ('/lib/ld-linux-aarch64.so.1', '0')}, Pihole) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh FTLdetect @@ -297,9 +297,9 @@ def test_FTL_detect_aarch64_no_errors(Pihole): def test_FTL_detect_armv6l_no_errors(Pihole): ''' confirms only armv6l package is downloaded for FTL engine ''' # mock uname to return armv6l platform - mock_command('uname', {'-m':('armv6l', '0')}, Pihole) + mock_command('uname', {'-m': ('armv6l', '0')}, Pihole) # mock ldd to respond with aarch64 shared library - mock_command('ldd', {'/bin/ls':('/lib/ld-linux-armhf.so.3', '0')}, Pihole) + mock_command('ldd', {'/bin/ls': ('/lib/ld-linux-armhf.so.3', '0')}, Pihole) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh FTLdetect @@ -315,9 +315,9 @@ def test_FTL_detect_armv6l_no_errors(Pihole): def test_FTL_detect_armv7l_no_errors(Pihole): ''' confirms only armv7l package is downloaded for FTL engine ''' # mock uname to return armv7l platform - mock_command('uname', {'-m':('armv7l', '0')}, Pihole) + mock_command('uname', {'-m': ('armv7l', '0')}, Pihole) # mock ldd to respond with aarch64 shared library - mock_command('ldd', {'/bin/ls':('/lib/ld-linux-armhf.so.3', '0')}, Pihole) + mock_command('ldd', {'/bin/ls': ('/lib/ld-linux-armhf.so.3', '0')}, Pihole) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh FTLdetect @@ -347,7 +347,7 @@ def test_FTL_detect_x86_64_no_errors(Pihole): def test_FTL_detect_unknown_no_errors(Pihole): ''' confirms only generic package is downloaded for FTL engine ''' # mock uname to return generic platform - mock_command('uname', {'-m':('mips', '0')}, Pihole) + mock_command('uname', {'-m': ('mips', '0')}, Pihole) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh FTLdetect @@ -413,7 +413,7 @@ def test_FTL_binary_installed_and_responsive_no_errors(Pihole): def test_IPv6_only_link_local(Pihole): ''' confirms IPv6 blocking is disabled for Link-local address ''' # mock ip -6 address to return Link-local address - mock_command_2('ip', {'-6 address':('inet6 fe80::d210:52fa:fe00:7ad7/64 scope link', '0')}, Pihole) + mock_command_2('ip', {'-6 address': ('inet6 fe80::d210:52fa:fe00:7ad7/64 scope link', '0')}, Pihole) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh useIPv6dialog @@ -425,7 +425,7 @@ def test_IPv6_only_link_local(Pihole): def test_IPv6_only_ULA(Pihole): ''' confirms IPv6 blocking is enabled for ULA addresses ''' # mock ip -6 address to return ULA address - mock_command_2('ip', {'-6 address':('inet6 fda2:2001:5555:0:d210:52fa:fe00:7ad7/64 scope global', '0')}, Pihole) + mock_command_2('ip', {'-6 address': ('inet6 fda2:2001:5555:0:d210:52fa:fe00:7ad7/64 scope global', '0')}, Pihole) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh useIPv6dialog @@ -437,7 +437,7 @@ def test_IPv6_only_ULA(Pihole): def test_IPv6_only_GUA(Pihole): ''' confirms IPv6 blocking is enabled for GUA addresses ''' # mock ip -6 address to return GUA address - mock_command_2('ip', {'-6 address':('inet6 2003:12:1e43:301:d210:52fa:fe00:7ad7/64 scope global', '0')}, Pihole) + mock_command_2('ip', {'-6 address': ('inet6 2003:12:1e43:301:d210:52fa:fe00:7ad7/64 scope global', '0')}, Pihole) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh useIPv6dialog @@ -449,7 +449,7 @@ def test_IPv6_only_GUA(Pihole): def test_IPv6_GUA_ULA_test(Pihole): ''' confirms IPv6 blocking is enabled for GUA and ULA addresses ''' # mock ip -6 address to return GUA and ULA addresses - mock_command_2('ip', {'-6 address':('inet6 2003:12:1e43:301:d210:52fa:fe00:7ad7/64 scope global\ninet6 fda2:2001:5555:0:d210:52fa:fe00:7ad7/64 scope global', '0')}, Pihole) + mock_command_2('ip', {'-6 address': ('inet6 2003:12:1e43:301:d210:52fa:fe00:7ad7/64 scope global\ninet6 fda2:2001:5555:0:d210:52fa:fe00:7ad7/64 scope global', '0')}, Pihole) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh useIPv6dialog @@ -461,7 +461,7 @@ def test_IPv6_GUA_ULA_test(Pihole): def test_IPv6_ULA_GUA_test(Pihole): ''' confirms IPv6 blocking is enabled for GUA and ULA addresses ''' # mock ip -6 address to return ULA and GUA addresses - mock_command_2('ip', {'-6 address':('inet6 fda2:2001:5555:0:d210:52fa:fe00:7ad7/64 scope global\ninet6 2003:12:1e43:301:d210:52fa:fe00:7ad7/64 scope global', '0')}, Pihole) + mock_command_2('ip', {'-6 address': ('inet6 fda2:2001:5555:0:d210:52fa:fe00:7ad7/64 scope global\ninet6 2003:12:1e43:301:d210:52fa:fe00:7ad7/64 scope global', '0')}, Pihole) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh useIPv6dialog From c3d443aaffcf75bd2993358eb2e25a1bb33040d5 Mon Sep 17 00:00:00 2001 From: bcambl Date: Tue, 3 Jul 2018 00:05:24 -0600 Subject: [PATCH 109/163] python linting: lines > 79 characters (E501) Signed-off-by: bcambl --- test/conftest.py | 31 +++- test/test_automated_install.py | 307 ++++++++++++++++++++++++++------- test/test_shellcheck.py | 9 +- 3 files changed, 271 insertions(+), 76 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 44e0a4a4..80fea82b 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -8,8 +8,10 @@ check_output = testinfra.get_backend( @pytest.fixture def Pihole(Docker): - ''' used to contain some script stubbing, now pretty much an alias. - Also provides bash as the default run function shell ''' + ''' + used to contain some script stubbing, now pretty much an alias. + Also provides bash as the default run function shell + ''' def run_bash(self, command, *args, **kwargs): cmd = self.get_command(command, *args) if self.user is not None: @@ -23,13 +25,18 @@ def Pihole(Docker): return out funcType = type(Docker.run) - Docker.run = funcType(run_bash, Docker, testinfra.backend.docker.DockerBackend) + Docker.run = funcType(run_bash, + Docker, + testinfra.backend.docker.DockerBackend) return Docker @pytest.fixture def Docker(request, args, image, cmd): - ''' combine our fixtures into a docker run command and setup finalizer to cleanup ''' + ''' + combine our fixtures into a docker run command and setup finalizer to + cleanup + ''' assert 'docker' in check_output('id'), "Are you in the docker group?" docker_run = "docker run {} {} {}".format(args, image, cmd) docker_id = check_output(docker_run) @@ -45,23 +52,31 @@ def Docker(request, args, image, cmd): @pytest.fixture def args(request): - ''' -t became required when tput began being used ''' + ''' + -t became required when tput began being used + ''' return '-t -d' @pytest.fixture(params=['debian', 'centos']) def tag(request): - ''' consumed by image to make the test matrix ''' + ''' + consumed by image to make the test matrix + ''' return request.param @pytest.fixture() def image(request, tag): - ''' built by test_000_build_containers.py ''' + ''' + built by test_000_build_containers.py + ''' return 'pytest_pihole:{}'.format(tag) @pytest.fixture() def cmd(request): - ''' default to doing nothing by tailing null, but don't exit ''' + ''' + default to doing nothing by tailing null, but don't exit + ''' return 'tail -f /dev/null' diff --git a/test/test_automated_install.py b/test/test_automated_install.py index 55709af9..a604e197 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -15,9 +15,11 @@ info_box="[i]".decode("utf-8") def test_setupVars_are_sourced_to_global_scope(Pihole): - ''' currently update_dialogs sources setupVars with a dot, + ''' + currently update_dialogs sources setupVars with a dot, then various other functions use the variables. - This confirms the sourced variables are in scope between functions ''' + This confirms the sourced variables are in scope between functions + ''' setup_var_file = 'cat < /etc/pihole/setupVars.conf\n' for k,v in SETUPVARS.iteritems(): setup_var_file += "{}={}\n".format(k, v) @@ -49,8 +51,11 @@ def test_setupVars_are_sourced_to_global_scope(Pihole): def test_setupVars_saved_to_file(Pihole): - ''' confirm saved settings are written to a file for future updates to re-use ''' - set_setup_vars = '\n' # dedent works better with this and padding matching script below + ''' + confirm saved settings are written to a file for future updates to re-use + ''' + # dedent works better with this and padding matching script below + set_setup_vars = '\n' for k,v in SETUPVARS.iteritems(): set_setup_vars += " {}={}\n".format(k, v) Pihole.run(set_setup_vars).stdout @@ -74,7 +79,9 @@ def test_setupVars_saved_to_file(Pihole): def test_configureFirewall_firewalld_running_no_errors(Pihole): - ''' confirms firewalld rules are applied when firewallD is running ''' + ''' + confirms firewalld rules are applied when firewallD is running + ''' # firewallD returns 'running' as status mock_command('firewall-cmd', {'*': ('running', 0)}, Pihole) # Whiptail dialog returns Ok for user prompt @@ -87,24 +94,33 @@ def test_configureFirewall_firewalld_running_no_errors(Pihole): assert expected_stdout in configureFirewall.stdout firewall_calls = Pihole.run('cat /var/log/firewall-cmd').stdout assert 'firewall-cmd --state' in firewall_calls - assert 'firewall-cmd --permanent --add-service=http --add-service=dns' in firewall_calls + assert ('firewall-cmd ' + '--permanent ' + '--add-service=http ' + '--add-service=dns') in firewall_calls assert 'firewall-cmd --reload' in firewall_calls def test_configureFirewall_firewalld_disabled_no_errors(Pihole): - ''' confirms firewalld rules are not applied when firewallD is not running ''' + ''' + confirms firewalld rules are not applied when firewallD is not running + ''' # firewallD returns non-running status mock_command('firewall-cmd', {'*': ('not running', '1')}, Pihole) configureFirewall = Pihole.run(''' source /opt/pihole/basic-install.sh configureFirewall ''') - expected_stdout = 'No active firewall detected.. skipping firewall configuration' + expected_stdout = ('No active firewall detected.. ' + 'skipping firewall configuration') assert expected_stdout in configureFirewall.stdout def test_configureFirewall_firewalld_enabled_declined_no_errors(Pihole): - ''' confirms firewalld rules are not applied when firewallD is running, user declines ruleset ''' + ''' + confirms firewalld rules are not applied when firewallD is running, user + declines ruleset + ''' # firewallD returns running status mock_command('firewall-cmd', {'*': ('running', 0)}, Pihole) # Whiptail dialog returns Cancel for user prompt @@ -128,7 +144,10 @@ def test_configureFirewall_no_firewall(Pihole): def test_configureFirewall_IPTables_enabled_declined_no_errors(Pihole): - ''' confirms IPTables rules are not applied when IPTables is running, user declines ruleset ''' + ''' + confirms IPTables rules are not applied when IPTables is running, user + declines ruleset + ''' # iptables command exists mock_command('iptables', {'*': ('', '0')}, Pihole) # modinfo returns always true (ip_tables module check) @@ -144,8 +163,12 @@ def test_configureFirewall_IPTables_enabled_declined_no_errors(Pihole): def test_configureFirewall_IPTables_enabled_rules_exist_no_errors(Pihole): - ''' confirms IPTables rules are not applied when IPTables is running and rules exist ''' - # iptables command exists and returns 0 on calls (should return 0 on iptables -C) + ''' + confirms IPTables rules are not applied when IPTables is running and rules + exist + ''' + # iptables command exists and returns 0 on calls + # (should return 0 on iptables -C) mock_command('iptables', {'-S': ('-P INPUT DENY', '0')}, Pihole) # modinfo returns always true (ip_tables module check) mock_command('modinfo', {'*': ('', '0')}, Pihole) @@ -158,15 +181,38 @@ def test_configureFirewall_IPTables_enabled_rules_exist_no_errors(Pihole): expected_stdout = 'Installing new IPTables firewall rulesets' assert expected_stdout in configureFirewall.stdout firewall_calls = Pihole.run('cat /var/log/iptables').stdout - assert 'iptables -I INPUT 1 -p tcp -m tcp --dport 80 -j ACCEPT' not in firewall_calls - assert 'iptables -I INPUT 1 -p tcp -m tcp --dport 53 -j ACCEPT' not in firewall_calls - assert 'iptables -I INPUT 1 -p udp -m udp --dport 53 -j ACCEPT' not in firewall_calls + iptables_line = 'iptables -I INPUT 1 -p tcp -m tcp --dport 80 -j ACCEPT' + assert iptables_line not in firewall_calls + iptables_line = 'iptables -I INPUT 1 -p tcp -m tcp --dport 53 -j ACCEPT' + assert iptables_line not in firewall_calls + iptables_line = 'iptables -I INPUT 1 -p udp -m udp --dport 53 -j ACCEPT' + assert iptables_line not in firewall_calls def test_configureFirewall_IPTables_enabled_not_exist_no_errors(Pihole): - ''' confirms IPTables rules are applied when IPTables is running and rules do not exist ''' + ''' + confirms IPTables rules are applied when IPTables is running and rules do + not exist + ''' # iptables command and returns 0 on calls (should return 1 on iptables -C) - mock_command('iptables', {'-S': ('-P INPUT DENY', '0'), '-C': ('', 1), '-I': ('', 0)}, Pihole) + mock_command( + 'iptables', + { + '-S': ( + '-P INPUT DENY', + '0' + ), + '-C': ( + '', + 1 + ), + '-I': ( + '', + 0 + ) + }, + Pihole + ) # modinfo returns always true (ip_tables module check) mock_command('modinfo', {'*': ('', '0')}, Pihole) # Whiptail dialog returns Cancel for user prompt @@ -178,13 +224,18 @@ def test_configureFirewall_IPTables_enabled_not_exist_no_errors(Pihole): expected_stdout = 'Installing new IPTables firewall rulesets' assert expected_stdout in configureFirewall.stdout firewall_calls = Pihole.run('cat /var/log/iptables').stdout - assert 'iptables -I INPUT 1 -p tcp -m tcp --dport 80 -j ACCEPT' in firewall_calls - assert 'iptables -I INPUT 1 -p tcp -m tcp --dport 53 -j ACCEPT' in firewall_calls - assert 'iptables -I INPUT 1 -p udp -m udp --dport 53 -j ACCEPT' in firewall_calls + iptables_line = 'iptables -I INPUT 1 -p tcp -m tcp --dport 80 -j ACCEPT' + assert iptables_line in firewall_calls + iptables_line = 'iptables -I INPUT 1 -p tcp -m tcp --dport 53 -j ACCEPT' + assert iptables_line in firewall_calls + iptables_line = 'iptables -I INPUT 1 -p udp -m udp --dport 53 -j ACCEPT' + assert iptables_line in firewall_calls def test_selinux_enforcing_default_exit(Pihole): - ''' confirms installer prompts to exit when SELinux is Enforcing by default ''' + ''' + confirms installer prompts to exit when SELinux is Enforcing by default + ''' # getenforce returns the running state of SELinux mock_command('getenforce', {'*': ('Enforcing', '0')}, Pihole) # Whiptail dialog returns Cancel for user prompt @@ -193,13 +244,17 @@ def test_selinux_enforcing_default_exit(Pihole): source /opt/pihole/basic-install.sh checkSelinux ''') - assert info_box + ' SELinux mode detected: Enforcing' in check_selinux.stdout - assert 'SELinux Enforcing detected, exiting installer' in check_selinux.stdout + expected_stdout = info_box + ' SELinux mode detected: Enforcing' + assert expected_stdout in check_selinux.stdout + expected_stdout = 'SELinux Enforcing detected, exiting installer' + assert expected_stdout in check_selinux.stdout assert check_selinux.rc == 1 def test_selinux_enforcing_continue(Pihole): - ''' confirms installer prompts to continue with custom policy warning ''' + ''' + confirms installer prompts to continue with custom policy warning + ''' # getenforce returns the running state of SELinux mock_command('getenforce', {'*': ('Enforcing', '0')}, Pihole) # Whiptail dialog returns Cancel for user prompt @@ -208,80 +263,117 @@ def test_selinux_enforcing_continue(Pihole): source /opt/pihole/basic-install.sh checkSelinux ''') - assert info_box + ' SELinux mode detected: Enforcing' in check_selinux.stdout - assert info_box + ' Continuing installation with SELinux Enforcing' in check_selinux.stdout - assert info_box + ' Please refer to official SELinux documentation to create a custom policy' in check_selinux.stdout + expected_stdout = info_box + ' SELinux mode detected: Enforcing' + assert expected_stdout in check_selinux.stdout + expected_stdout = info_box + (' Continuing installation with SELinux ' + 'Enforcing') + assert expected_stdout in check_selinux.stdout + expected_stdout = info_box + (' Please refer to official SELinux ' + 'documentation to create a custom policy') + assert expected_stdout in check_selinux.stdout assert check_selinux.rc == 0 def test_selinux_permissive(Pihole): - ''' confirms installer continues when SELinux is Permissive ''' + ''' + confirms installer continues when SELinux is Permissive + ''' # getenforce returns the running state of SELinux mock_command('getenforce', {'*': ('Permissive', '0')}, Pihole) check_selinux = Pihole.run(''' source /opt/pihole/basic-install.sh checkSelinux ''') - assert info_box + ' SELinux mode detected: Permissive' in check_selinux.stdout + expected_stdout = info_box + ' SELinux mode detected: Permissive' + assert expected_stdout in check_selinux.stdout assert check_selinux.rc == 0 def test_selinux_disabled(Pihole): - ''' confirms installer continues when SELinux is Disabled ''' + ''' + confirms installer continues when SELinux is Disabled + ''' mock_command('getenforce', {'*': ('Disabled', '0')}, Pihole) check_selinux = Pihole.run(''' source /opt/pihole/basic-install.sh checkSelinux ''') - assert info_box + ' SELinux mode detected: Disabled' in check_selinux.stdout + expected_stdout = info_box + ' SELinux mode detected: Disabled' + assert expected_stdout in check_selinux.stdout assert check_selinux.rc == 0 def test_installPiholeWeb_fresh_install_no_errors(Pihole): - ''' confirms all web page assets from Core repo are installed on a fresh build ''' + ''' + confirms all web page assets from Core repo are installed on a fresh build + ''' installWeb = Pihole.run(''' source /opt/pihole/basic-install.sh installPiholeWeb ''') - assert info_box + ' Installing blocking page...' in installWeb.stdout - assert tick_box + ' Creating directory for blocking page, and copying files' in installWeb.stdout - assert cross_box + ' Backing up index.lighttpd.html' in installWeb.stdout - assert 'No default index.lighttpd.html file found... not backing up' in installWeb.stdout - assert tick_box + ' Installing sudoer file' in installWeb.stdout + expected_stdout = info_box + ' Installing blocking page...' + assert expected_stdout in installWeb.stdout + expected_stdout = tick_box + (' Creating directory for blocking page, ' + 'and copying files') + assert expected_stdout in installWeb.stdout + expected_stdout = cross_box + ' Backing up index.lighttpd.html' + assert expected_stdout in installWeb.stdout + expected_stdout = ('No default index.lighttpd.html file found... ' + 'not backing up') + assert expected_stdout in installWeb.stdout + expected_stdout = tick_box + ' Installing sudoer file' + assert expected_stdout in installWeb.stdout web_directory = Pihole.run('ls -r /var/www/html/pihole').stdout assert 'index.php' in web_directory assert 'blockingpage.css' in web_directory def test_update_package_cache_success_no_errors(Pihole): - ''' confirms package cache was updated without any errors''' + ''' + confirms package cache was updated without any errors + ''' updateCache = Pihole.run(''' source /opt/pihole/basic-install.sh distro_check update_package_cache ''') - assert tick_box + ' Update local cache of available packages' in updateCache.stdout + expected_stdout = tick_box + ' Update local cache of available packages' + assert expected_stdout in updateCache.stdout assert 'Error: Unable to update package cache.' not in updateCache.stdout def test_update_package_cache_failure_no_errors(Pihole): - ''' confirms package cache was not updated''' + ''' + confirms package cache was not updated + ''' mock_command('apt-get', {'update': ('', '1')}, Pihole) updateCache = Pihole.run(''' source /opt/pihole/basic-install.sh distro_check update_package_cache ''') - assert cross_box + ' Update local cache of available packages' in updateCache.stdout + expected_stdout = cross_box + ' Update local cache of available packages' + assert expected_stdout in updateCache.stdout assert 'Error: Unable to update package cache.' in updateCache.stdout def test_FTL_detect_aarch64_no_errors(Pihole): - ''' confirms only aarch64 package is downloaded for FTL engine ''' + ''' + confirms only aarch64 package is downloaded for FTL engine + ''' # mock uname to return aarch64 platform mock_command('uname', {'-m': ('aarch64', '0')}, Pihole) # mock ldd to respond with aarch64 shared library - mock_command('ldd', {'/bin/ls': ('/lib/ld-linux-aarch64.so.1', '0')}, Pihole) + mock_command( + 'ldd', + { + '/bin/ls': ( + '/lib/ld-linux-aarch64.so.1', + '0' + ) + }, + Pihole + ) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh FTLdetect @@ -295,7 +387,9 @@ def test_FTL_detect_aarch64_no_errors(Pihole): def test_FTL_detect_armv6l_no_errors(Pihole): - ''' confirms only armv6l package is downloaded for FTL engine ''' + ''' + confirms only armv6l package is downloaded for FTL engine + ''' # mock uname to return armv6l platform mock_command('uname', {'-m': ('armv6l', '0')}, Pihole) # mock ldd to respond with aarch64 shared library @@ -306,14 +400,17 @@ def test_FTL_detect_armv6l_no_errors(Pihole): ''') expected_stdout = info_box + ' FTL Checks...' assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + ' Detected ARM-hf architecture (armv6 or lower)' + expected_stdout = tick_box + (' Detected ARM-hf architecture ' + '(armv6 or lower)') assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + ' Downloading and Installing FTL' assert expected_stdout in detectPlatform.stdout def test_FTL_detect_armv7l_no_errors(Pihole): - ''' confirms only armv7l package is downloaded for FTL engine ''' + ''' + confirms only armv7l package is downloaded for FTL engine + ''' # mock uname to return armv7l platform mock_command('uname', {'-m': ('armv7l', '0')}, Pihole) # mock ldd to respond with aarch64 shared library @@ -331,7 +428,9 @@ def test_FTL_detect_armv7l_no_errors(Pihole): def test_FTL_detect_x86_64_no_errors(Pihole): - ''' confirms only x86_64 package is downloaded for FTL engine ''' + ''' + confirms only x86_64 package is downloaded for FTL engine + ''' detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh FTLdetect @@ -357,7 +456,9 @@ def test_FTL_detect_unknown_no_errors(Pihole): def test_FTL_download_aarch64_no_errors(Pihole): - ''' confirms only aarch64 package is downloaded for FTL engine ''' + ''' + confirms only aarch64 package is downloaded for FTL engine + ''' # mock uname to return generic platform download_binary = Pihole.run(''' source /opt/pihole/basic-install.sh @@ -372,7 +473,9 @@ def test_FTL_download_aarch64_no_errors(Pihole): def test_FTL_download_unknown_fails_no_errors(Pihole): - ''' confirms unknown binary is not downloaded for FTL engine ''' + ''' + confirms unknown binary is not downloaded for FTL engine + ''' # mock uname to return generic platform download_binary = Pihole.run(''' source /opt/pihole/basic-install.sh @@ -385,7 +488,9 @@ def test_FTL_download_unknown_fails_no_errors(Pihole): def test_FTL_binary_installed_and_responsive_no_errors(Pihole): - ''' confirms FTL binary is copied and functional in installed location ''' + ''' + confirms FTL binary is copied and functional in installed location + ''' installed_binary = Pihole.run(''' source /opt/pihole/basic-install.sh FTLdetect @@ -396,7 +501,9 @@ def test_FTL_binary_installed_and_responsive_no_errors(Pihole): # def test_FTL_support_files_installed(Pihole): -# ''' confirms FTL support files are installed ''' +# ''' +# confirms FTL support files are installed +# ''' # support_files = Pihole.run(''' # source /opt/pihole/basic-install.sh # FTLdetect @@ -411,21 +518,44 @@ def test_FTL_binary_installed_and_responsive_no_errors(Pihole): def test_IPv6_only_link_local(Pihole): - ''' confirms IPv6 blocking is disabled for Link-local address ''' + ''' + confirms IPv6 blocking is disabled for Link-local address + ''' # mock ip -6 address to return Link-local address - mock_command_2('ip', {'-6 address': ('inet6 fe80::d210:52fa:fe00:7ad7/64 scope link', '0')}, Pihole) + mock_command_2( + 'ip', + { + '-6 address': ( + 'inet6 fe80::d210:52fa:fe00:7ad7/64 scope link', + '0' + ) + }, + Pihole + ) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh useIPv6dialog ''') - expected_stdout = 'Unable to find IPv6 ULA/GUA address, IPv6 adblocking will not be enabled' + expected_stdout = ('Unable to find IPv6 ULA/GUA address, ' + 'IPv6 adblocking will not be enabled') assert expected_stdout in detectPlatform.stdout def test_IPv6_only_ULA(Pihole): - ''' confirms IPv6 blocking is enabled for ULA addresses ''' + ''' + confirms IPv6 blocking is enabled for ULA addresses + ''' # mock ip -6 address to return ULA address - mock_command_2('ip', {'-6 address': ('inet6 fda2:2001:5555:0:d210:52fa:fe00:7ad7/64 scope global', '0')}, Pihole) + mock_command_2( + 'ip', + { + '-6 address': ( + 'inet6 fda2:2001:5555:0:d210:52fa:fe00:7ad7/64 scope global', + '0' + ) + }, + Pihole + ) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh useIPv6dialog @@ -435,9 +565,20 @@ def test_IPv6_only_ULA(Pihole): def test_IPv6_only_GUA(Pihole): - ''' confirms IPv6 blocking is enabled for GUA addresses ''' + ''' + confirms IPv6 blocking is enabled for GUA addresses + ''' # mock ip -6 address to return GUA address - mock_command_2('ip', {'-6 address': ('inet6 2003:12:1e43:301:d210:52fa:fe00:7ad7/64 scope global', '0')}, Pihole) + mock_command_2( + 'ip', + { + '-6 address': ( + 'inet6 2003:12:1e43:301:d210:52fa:fe00:7ad7/64 scope global', + '0' + ) + }, + Pihole + ) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh useIPv6dialog @@ -447,9 +588,21 @@ def test_IPv6_only_GUA(Pihole): def test_IPv6_GUA_ULA_test(Pihole): - ''' confirms IPv6 blocking is enabled for GUA and ULA addresses ''' + ''' + confirms IPv6 blocking is enabled for GUA and ULA addresses + ''' # mock ip -6 address to return GUA and ULA addresses - mock_command_2('ip', {'-6 address': ('inet6 2003:12:1e43:301:d210:52fa:fe00:7ad7/64 scope global\ninet6 fda2:2001:5555:0:d210:52fa:fe00:7ad7/64 scope global', '0')}, Pihole) + mock_command_2( + 'ip', + { + '-6 address': ( + 'inet6 2003:12:1e43:301:d210:52fa:fe00:7ad7/64 scope global\n' + 'inet6 fda2:2001:5555:0:d210:52fa:fe00:7ad7/64 scope global', + '0' + ) + }, + Pihole + ) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh useIPv6dialog @@ -459,9 +612,21 @@ def test_IPv6_GUA_ULA_test(Pihole): def test_IPv6_ULA_GUA_test(Pihole): - ''' confirms IPv6 blocking is enabled for GUA and ULA addresses ''' + ''' + confirms IPv6 blocking is enabled for GUA and ULA addresses + ''' # mock ip -6 address to return ULA and GUA addresses - mock_command_2('ip', {'-6 address': ('inet6 fda2:2001:5555:0:d210:52fa:fe00:7ad7/64 scope global\ninet6 2003:12:1e43:301:d210:52fa:fe00:7ad7/64 scope global', '0')}, Pihole) + mock_command_2( + 'ip', + { + '-6 address': ( + 'inet6 fda2:2001:5555:0:d210:52fa:fe00:7ad7/64 scope global\n' + 'inet6 2003:12:1e43:301:d210:52fa:fe00:7ad7/64 scope global', + '0' + ) + }, + Pihole + ) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh useIPv6dialog @@ -472,7 +637,10 @@ def test_IPv6_ULA_GUA_test(Pihole): # Helper functions def mock_command(script, args, container): - ''' Allows for setup of commands we don't really want to have to run for real in unit tests ''' + ''' + Allows for setup of commands we don't really want to have to run for real + in unit tests + ''' full_script_path = '/usr/local/bin/{}'.format(script) mock_script = dedent('''\ #!/bin/bash -e @@ -490,11 +658,16 @@ def mock_command(script, args, container): container.run(''' cat < {script}\n{content}\nEOF chmod +x {script} - rm -f /var/log/{scriptlog}'''.format(script=full_script_path, content=mock_script, scriptlog=script)) + rm -f /var/log/{scriptlog}'''.format(script=full_script_path, + content=mock_script, + scriptlog=script)) def mock_command_2(script, args, container): - ''' Allows for setup of commands we don't really want to have to run for real in unit tests ''' + ''' + Allows for setup of commands we don't really want to have to run for real + in unit tests + ''' full_script_path = '/usr/local/bin/{}'.format(script) mock_script = dedent('''\ #!/bin/bash -e @@ -512,7 +685,9 @@ def mock_command_2(script, args, container): container.run(''' cat < {script}\n{content}\nEOF chmod +x {script} - rm -f /var/log/{scriptlog}'''.format(script=full_script_path, content=mock_script, scriptlog=script)) + rm -f /var/log/{scriptlog}'''.format(script=full_script_path, + content=mock_script, + scriptlog=script)) def run_script(Pihole, script): diff --git a/test/test_shellcheck.py b/test/test_shellcheck.py index 0bd03f36..b9b1cea5 100644 --- a/test/test_shellcheck.py +++ b/test/test_shellcheck.py @@ -7,8 +7,13 @@ run_local = testinfra.get_backend( def test_scripts_pass_shellcheck(): - ''' Make sure shellcheck does not find anything wrong with our shell scripts ''' - shellcheck = "find . -type f -name 'update.sh' | while read file; do shellcheck -x \"$file\" -e SC1090,SC1091; done;" + ''' + Make sure shellcheck does not find anything wrong with our shell scripts + ''' + shellcheck = ("find . -type f -name 'update.sh' " + "| while read file; do " + "shellcheck -x \"$file\" -e SC1090,SC1091; " + "done;") results = run_local(shellcheck) print results.stdout assert '' == results.stdout From 9bd4986781fbfc610bc8cb174f3b53152491ac9d Mon Sep 17 00:00:00 2001 From: bcambl Date: Tue, 3 Jul 2018 00:21:57 -0600 Subject: [PATCH 110/163] python linting: whitespace before ':' (E203) Signed-off-by: bcambl --- test/test_automated_install.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test_automated_install.py b/test/test_automated_install.py index a604e197..1bbdefa1 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -2,11 +2,11 @@ import pytest from textwrap import dedent SETUPVARS = { - 'PIHOLE_INTERFACE' : 'eth99', - 'IPV4_ADDRESS' : '1.1.1.1', - 'IPV6_ADDRESS' : 'FE80::240:D0FF:FE48:4672', - 'PIHOLE_DNS_1' : '4.2.2.1', - 'PIHOLE_DNS_2' : '4.2.2.2' + 'PIHOLE_INTERFACE': 'eth99', + 'IPV4_ADDRESS': '1.1.1.1', + 'IPV6_ADDRESS': 'FE80::240:D0FF:FE48:4672', + 'PIHOLE_DNS_1': '4.2.2.1', + 'PIHOLE_DNS_2': '4.2.2.2' } tick_box="[\x1b[1;32m\xe2\x9c\x93\x1b[0m]".decode("utf-8") From 79232d02c935e88b9df211e66cddcda85144b109 Mon Sep 17 00:00:00 2001 From: bcambl Date: Tue, 3 Jul 2018 00:30:00 -0600 Subject: [PATCH 111/163] python linting: 'pytest' imported but unused (F401) Signed-off-by: bcambl --- test/test_automated_install.py | 1 - test/test_shellcheck.py | 1 - 2 files changed, 2 deletions(-) diff --git a/test/test_automated_install.py b/test/test_automated_install.py index 1bbdefa1..027e5365 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -1,4 +1,3 @@ -import pytest from textwrap import dedent SETUPVARS = { diff --git a/test/test_shellcheck.py b/test/test_shellcheck.py index b9b1cea5..43e8ad6f 100644 --- a/test/test_shellcheck.py +++ b/test/test_shellcheck.py @@ -1,4 +1,3 @@ -import pytest import testinfra run_local = testinfra.get_backend( From 1d3445bc0f019f2a01d36552c0e8a84fa5e67467 Mon Sep 17 00:00:00 2001 From: bcambl Date: Tue, 3 Jul 2018 00:33:30 -0600 Subject: [PATCH 112/163] python linting: whitespace after '(' and before ')' (E201 & E202) Signed-off-by: bcambl --- test/test_000_build_containers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_000_build_containers.py b/test/test_000_build_containers.py index 725136d8..dc365026 100644 --- a/test/test_000_build_containers.py +++ b/test/test_000_build_containers.py @@ -8,8 +8,8 @@ run_local = testinfra.get_backend( @pytest.mark.parametrize("image,tag", [ - ( 'test/debian.Dockerfile', 'pytest_pihole:debian' ), - ( 'test/centos.Dockerfile', 'pytest_pihole:centos' ), + ('test/debian.Dockerfile', 'pytest_pihole:debian'), + ('test/centos.Dockerfile', 'pytest_pihole:centos'), ]) def test_build_pihole_image(image, tag): build_cmd = run_local('docker build -f {} -t {} .'.format(image, tag)) From 064a75b21b73948a11b2af2518a3e9dc2cc01192 Mon Sep 17 00:00:00 2001 From: bcambl Date: Tue, 3 Jul 2018 00:35:16 -0600 Subject: [PATCH 113/163] python linting: missing whitespace around operator (E225) Signed-off-by: bcambl --- test/test_automated_install.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_automated_install.py b/test/test_automated_install.py index 027e5365..14c2b77a 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -8,9 +8,9 @@ SETUPVARS = { 'PIHOLE_DNS_2': '4.2.2.2' } -tick_box="[\x1b[1;32m\xe2\x9c\x93\x1b[0m]".decode("utf-8") -cross_box="[\x1b[1;31m\xe2\x9c\x97\x1b[0m]".decode("utf-8") -info_box="[i]".decode("utf-8") +tick_box = "[\x1b[1;32m\xe2\x9c\x93\x1b[0m]".decode("utf-8") +cross_box = "[\x1b[1;31m\xe2\x9c\x97\x1b[0m]".decode("utf-8") +info_box = "[i]".decode("utf-8") def test_setupVars_are_sourced_to_global_scope(Pihole): From 40537e1522b2eb9d18d07254b4b8f155d749061e Mon Sep 17 00:00:00 2001 From: bcambl Date: Tue, 3 Jul 2018 00:36:47 -0600 Subject: [PATCH 114/163] python linting: missing whitespace after ',' (E231) Signed-off-by: bcambl --- test/test_automated_install.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_automated_install.py b/test/test_automated_install.py index 14c2b77a..400a94f4 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -20,7 +20,7 @@ def test_setupVars_are_sourced_to_global_scope(Pihole): This confirms the sourced variables are in scope between functions ''' setup_var_file = 'cat < /etc/pihole/setupVars.conf\n' - for k,v in SETUPVARS.iteritems(): + for k, v in SETUPVARS.iteritems(): setup_var_file += "{}={}\n".format(k, v) setup_var_file += "EOF\n" Pihole.run(setup_var_file) @@ -45,7 +45,7 @@ def test_setupVars_are_sourced_to_global_scope(Pihole): output = run_script(Pihole, script).stdout - for k,v in SETUPVARS.iteritems(): + for k, v in SETUPVARS.iteritems(): assert "{}={}".format(k, v) in output @@ -55,7 +55,7 @@ def test_setupVars_saved_to_file(Pihole): ''' # dedent works better with this and padding matching script below set_setup_vars = '\n' - for k,v in SETUPVARS.iteritems(): + for k, v in SETUPVARS.iteritems(): set_setup_vars += " {}={}\n".format(k, v) Pihole.run(set_setup_vars).stdout @@ -73,7 +73,7 @@ def test_setupVars_saved_to_file(Pihole): output = run_script(Pihole, script).stdout - for k,v in SETUPVARS.iteritems(): + for k, v in SETUPVARS.iteritems(): assert "{}={}".format(k, v) in output From 5ca2ad6148b83ab2d072477dda95ab0c3b97332f Mon Sep 17 00:00:00 2001 From: bcambl Date: Thu, 5 Jul 2018 18:10:43 -0600 Subject: [PATCH 115/163] move test globals & mock commands to conftest Signed-off-by: bcambl --- test/conftest.py | 74 ++++++++++++++++++++++++++++++ test/test_automated_install.py | 82 ++++------------------------------ 2 files changed, 83 insertions(+), 73 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 80fea82b..7ec3832e 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,10 +1,23 @@ import pytest import testinfra +from textwrap import dedent check_output = testinfra.get_backend( "local://" ).get_module("Command").check_output +SETUPVARS = { + 'PIHOLE_INTERFACE': 'eth99', + 'IPV4_ADDRESS': '1.1.1.1', + 'IPV6_ADDRESS': 'FE80::240:D0FF:FE48:4672', + 'PIHOLE_DNS_1': '4.2.2.1', + 'PIHOLE_DNS_2': '4.2.2.2' +} + +tick_box = "[\x1b[1;32m\xe2\x9c\x93\x1b[0m]".decode("utf-8") +cross_box = "[\x1b[1;31m\xe2\x9c\x97\x1b[0m]".decode("utf-8") +info_box = "[i]".decode("utf-8") + @pytest.fixture def Pihole(Docker): @@ -80,3 +93,64 @@ def cmd(request): default to doing nothing by tailing null, but don't exit ''' return 'tail -f /dev/null' + + +# Helper functions +def mock_command(script, args, container): + ''' + Allows for setup of commands we don't really want to have to run for real + in unit tests + ''' + full_script_path = '/usr/local/bin/{}'.format(script) + mock_script = dedent('''\ + #!/bin/bash -e + echo "\$0 \$@" >> /var/log/{script} + case "\$1" in'''.format(script=script)) + for k, v in args.iteritems(): + case = dedent(''' + {arg}) + echo {res} + exit {retcode} + ;;'''.format(arg=k, res=v[0], retcode=v[1])) + mock_script += case + mock_script += dedent(''' + esac''') + container.run(''' + cat < {script}\n{content}\nEOF + chmod +x {script} + rm -f /var/log/{scriptlog}'''.format(script=full_script_path, + content=mock_script, + scriptlog=script)) + + +def mock_command_2(script, args, container): + ''' + Allows for setup of commands we don't really want to have to run for real + in unit tests + ''' + full_script_path = '/usr/local/bin/{}'.format(script) + mock_script = dedent('''\ + #!/bin/bash -e + echo "\$0 \$@" >> /var/log/{script} + case "\$1 \$2" in'''.format(script=script)) + for k, v in args.iteritems(): + case = dedent(''' + \"{arg}\") + echo \"{res}\" + exit {retcode} + ;;'''.format(arg=k, res=v[0], retcode=v[1])) + mock_script += case + mock_script += dedent(''' + esac''') + container.run(''' + cat < {script}\n{content}\nEOF + chmod +x {script} + rm -f /var/log/{scriptlog}'''.format(script=full_script_path, + content=mock_script, + scriptlog=script)) + + +def run_script(Pihole, script): + result = Pihole.run(script) + assert result.rc == 0 + return result diff --git a/test/test_automated_install.py b/test/test_automated_install.py index 400a94f4..3718282f 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -1,16 +1,13 @@ from textwrap import dedent - -SETUPVARS = { - 'PIHOLE_INTERFACE': 'eth99', - 'IPV4_ADDRESS': '1.1.1.1', - 'IPV6_ADDRESS': 'FE80::240:D0FF:FE48:4672', - 'PIHOLE_DNS_1': '4.2.2.1', - 'PIHOLE_DNS_2': '4.2.2.2' -} - -tick_box = "[\x1b[1;32m\xe2\x9c\x93\x1b[0m]".decode("utf-8") -cross_box = "[\x1b[1;31m\xe2\x9c\x97\x1b[0m]".decode("utf-8") -info_box = "[i]".decode("utf-8") +from conftest import ( + SETUPVARS, + tick_box, + info_box, + cross_box, + mock_command, + mock_command_2, + run_script +) def test_setupVars_are_sourced_to_global_scope(Pihole): @@ -632,64 +629,3 @@ def test_IPv6_ULA_GUA_test(Pihole): ''') expected_stdout = 'Found IPv6 ULA address, using it for blocking IPv6 ads' assert expected_stdout in detectPlatform.stdout - - -# Helper functions -def mock_command(script, args, container): - ''' - Allows for setup of commands we don't really want to have to run for real - in unit tests - ''' - full_script_path = '/usr/local/bin/{}'.format(script) - mock_script = dedent('''\ - #!/bin/bash -e - echo "\$0 \$@" >> /var/log/{script} - case "\$1" in'''.format(script=script)) - for k, v in args.iteritems(): - case = dedent(''' - {arg}) - echo {res} - exit {retcode} - ;;'''.format(arg=k, res=v[0], retcode=v[1])) - mock_script += case - mock_script += dedent(''' - esac''') - container.run(''' - cat < {script}\n{content}\nEOF - chmod +x {script} - rm -f /var/log/{scriptlog}'''.format(script=full_script_path, - content=mock_script, - scriptlog=script)) - - -def mock_command_2(script, args, container): - ''' - Allows for setup of commands we don't really want to have to run for real - in unit tests - ''' - full_script_path = '/usr/local/bin/{}'.format(script) - mock_script = dedent('''\ - #!/bin/bash -e - echo "\$0 \$@" >> /var/log/{script} - case "\$1 \$2" in'''.format(script=script)) - for k, v in args.iteritems(): - case = dedent(''' - \"{arg}\") - echo \"{res}\" - exit {retcode} - ;;'''.format(arg=k, res=v[0], retcode=v[1])) - mock_script += case - mock_script += dedent(''' - esac''') - container.run(''' - cat < {script}\n{content}\nEOF - chmod +x {script} - rm -f /var/log/{scriptlog}'''.format(script=full_script_path, - content=mock_script, - scriptlog=script)) - - -def run_script(Pihole, script): - result = Pihole.run(script) - assert result.rc == 0 - return result From 553aad6ed2d489bd508d7f956261d5e4b69de6a4 Mon Sep 17 00:00:00 2001 From: bcambl Date: Thu, 5 Jul 2018 18:12:39 -0600 Subject: [PATCH 116/163] add Fedora container to test matrix Signed-off-by: bcambl --- test/conftest.py | 2 +- test/fedora.Dockerfile | 16 ++++++++++++++++ test/test_000_build_containers.py | 1 + 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 test/fedora.Dockerfile diff --git a/test/conftest.py b/test/conftest.py index 7ec3832e..58530d38 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -71,7 +71,7 @@ def args(request): return '-t -d' -@pytest.fixture(params=['debian', 'centos']) +@pytest.fixture(params=['debian', 'centos', 'fedora']) def tag(request): ''' consumed by image to make the test matrix diff --git a/test/fedora.Dockerfile b/test/fedora.Dockerfile new file mode 100644 index 00000000..c4834388 --- /dev/null +++ b/test/fedora.Dockerfile @@ -0,0 +1,16 @@ +FROM fedora:latest + +ENV GITDIR /etc/.pihole +ENV SCRIPTDIR /opt/pihole + +RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole +ADD . $GITDIR +RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ +ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR + +RUN true && \ + chmod +x $SCRIPTDIR/* + +ENV PH_TEST true + +#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/test_000_build_containers.py b/test/test_000_build_containers.py index dc365026..ad243754 100644 --- a/test/test_000_build_containers.py +++ b/test/test_000_build_containers.py @@ -10,6 +10,7 @@ run_local = testinfra.get_backend( @pytest.mark.parametrize("image,tag", [ ('test/debian.Dockerfile', 'pytest_pihole:debian'), ('test/centos.Dockerfile', 'pytest_pihole:centos'), + ('test/fedora.Dockerfile', 'pytest_pihole:fedora'), ]) def test_build_pihole_image(image, tag): build_cmd = run_local('docker build -f {} -t {} .'.format(image, tag)) From a323b126e5ea89d4f54f6dfffdbb52a126a19449 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Thu, 5 Jul 2018 21:48:36 -0700 Subject: [PATCH 117/163] Finished space conversion Signed-off-by: Dan Schaper --- .editorconfig | 2 +- .idea/codeStyles/Project.xml | 6 - automated install/basic-install.sh | 4023 ++++++++++++++-------------- 3 files changed, 2010 insertions(+), 2021 deletions(-) diff --git a/.editorconfig b/.editorconfig index e5626a07..ee415d1f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -9,7 +9,7 @@ end_of_line = lf insert_final_newline = true indent_style = space indent_size = tab -tab_width = 2 +tab_width = 4 charset = utf-8 trim_trailing_whitespace = true diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml index 7704aeaa..79a710fd 100644 --- a/.idea/codeStyles/Project.xml +++ b/.idea/codeStyles/Project.xml @@ -1,11 +1,5 @@ - diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 36d717f5..b1852288 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -84,30 +84,30 @@ runUnattended=false INSTALL_WEB_SERVER=true # Check arguments for the undocumented flags for var in "$@"; do - case "$var" in - "--reconfigure" ) reconfigure=true;; - "--i_do_not_follow_recommendations" ) skipSpaceCheck=true;; - "--unattended" ) runUnattended=true;; - "--disable-install-webserver" ) INSTALL_WEB_SERVER=false;; - esac + case "$var" in + "--reconfigure" ) reconfigure=true;; + "--i_do_not_follow_recommendations" ) skipSpaceCheck=true;; + "--unattended" ) runUnattended=true;; + "--disable-install-webserver" ) INSTALL_WEB_SERVER=false;; + esac done # If the color table file exists, if [[ -f "${coltable}" ]]; then - # source it - source ${coltable} + # source it + source ${coltable} # Otherwise, else - # Set these values so the installer can still run in color - COL_NC='\e[0m' # No Color - COL_LIGHT_GREEN='\e[1;32m' - COL_LIGHT_RED='\e[1;31m' - TICK="[${COL_LIGHT_GREEN}✓${COL_NC}]" - CROSS="[${COL_LIGHT_RED}✗${COL_NC}]" - INFO="[i]" - # shellcheck disable=SC2034 - DONE="${COL_LIGHT_GREEN} done!${COL_NC}" - OVER="\\r\\033[K" + # Set these values so the installer can still run in color + COL_NC='\e[0m' # No Color + COL_LIGHT_GREEN='\e[1;32m' + COL_LIGHT_RED='\e[1;31m' + TICK="[${COL_LIGHT_GREEN}✓${COL_NC}]" + CROSS="[${COL_LIGHT_RED}✗${COL_NC}]" + INFO="[i]" + # shellcheck disable=SC2034 + DONE="${COL_LIGHT_GREEN} done!${COL_NC}" + OVER="\\r\\033[K" fi # A simple function that just echoes out our logo in ASCII format @@ -141,1033 +141,1029 @@ show_ascii_berry() { distro_check() { # If apt-get is installed, then we know it's part of the Debian family if command -v apt-get &> /dev/null; then - # Set some global variables here - # We don't set them earlier since the family might be Red Hat, so these values would be different - PKG_MANAGER="apt-get" - # A variable to store the command used to update the package cache - UPDATE_PKG_CACHE="${PKG_MANAGER} update" - # An array for something... - PKG_INSTALL=(${PKG_MANAGER} --yes --no-install-recommends install) - # grep -c will return 1 retVal on 0 matches, block this throwing the set -e with an OR TRUE - PKG_COUNT="${PKG_MANAGER} -s -o Debug::NoLocking=true upgrade | grep -c ^Inst || true" - # Some distros vary slightly so these fixes for dependencies may apply - # Debian 7 doesn't have iproute2 so if the dry run install is successful, - if ${PKG_MANAGER} install --dry-run iproute2 > /dev/null 2>&1; then - # we can install it - iproute_pkg="iproute2" - # Otherwise, - else - # use iproute - iproute_pkg="iproute" - fi - # Check for and determine version number (major and minor) of current php install - if command -v php &> /dev/null; then - phpInsVersion="$(php -v | head -n1 | grep -Po '(?<=PHP )[^ ]+')" - echo -e " ${INFO} Existing PHP installation detected : PHP version $phpInsVersion" - phpInsMajor="$(echo "$phpInsVersion" | cut -d\. -f1)" - phpInsMinor="$(echo "$phpInsVersion" | cut -d\. -f2)" - # Is installed php version 7.0 or greater - if [ "$(echo "$phpInsMajor.$phpInsMinor < 7.0" | bc )" == 0 ]; then - phpInsNewer=true - fi - fi - # Check if installed php is v 7.0, or newer to determine packages to install - if [[ "$phpInsNewer" != true ]]; then - # Prefer the php metapackage if it's there - if ${PKG_MANAGER} install --dry-run php > /dev/null 2>&1; then - phpVer="php" - # fall back on the php5 packages - else - phpVer="php5" - fi - else - # Newer php is installed, its common, cgi & sqlite counterparts are deps - phpVer="php$phpInsMajor.$phpInsMinor" - fi - # We also need the correct version for `php-sqlite` (which differs across distros) - if ${PKG_MANAGER} install --dry-run ${phpVer}-sqlite3 > /dev/null 2>&1; then - phpSqlite="sqlite3" - else - phpSqlite="sqlite" - fi - # Since our install script is so large, we need several other programs to successfully get a machine provisioned - # These programs are stored in an array so they can be looped through later - INSTALLER_DEPS=(apt-utils dialog debconf dhcpcd5 git ${iproute_pkg} whiptail) - # Pi-hole itself has several dependencies that also need to be installed - PIHOLE_DEPS=(bc cron curl dnsutils iputils-ping lsof netcat psmisc sudo unzip wget idn2 sqlite3 libcap2-bin dns-root-data resolvconf) - # The Web dashboard has some that also need to be installed - # It's useful to separate the two since our repos are also setup as "Core" code and "Web" code - PIHOLE_WEB_DEPS=(lighttpd ${phpVer}-common ${phpVer}-cgi ${phpVer}-${phpSqlite}) - # The Web server user, - LIGHTTPD_USER="www-data" - # group, - LIGHTTPD_GROUP="www-data" - # and config file - LIGHTTPD_CFG="lighttpd.conf.debian" - - # A function to check... - test_dpkg_lock() { - # An iterator used for counting loop iterations - i=0 - # fuser is a program to show which processes use the named files, sockets, or filesystems - # So while the command is true - while fuser /var/lib/dpkg/lock >/dev/null 2>&1 ; do - # Wait half a second - sleep 0.5 - # and increase the iterator - ((i=i+1)) - done - # Always return success, since we only return if there is no - # lock (anymore) - return 0 + # Set some global variables here + # We don't set them earlier since the family might be Red Hat, so these values would be different + PKG_MANAGER="apt-get" + # A variable to store the command used to update the package cache + UPDATE_PKG_CACHE="${PKG_MANAGER} update" + # An array for something... + PKG_INSTALL=(${PKG_MANAGER} --yes --no-install-recommends install) + # grep -c will return 1 retVal on 0 matches, block this throwing the set -e with an OR TRUE + PKG_COUNT="${PKG_MANAGER} -s -o Debug::NoLocking=true upgrade | grep -c ^Inst || true" + # Some distros vary slightly so these fixes for dependencies may apply + # Debian 7 doesn't have iproute2 so if the dry run install is successful, + if ${PKG_MANAGER} install --dry-run iproute2 > /dev/null 2>&1; then + # we can install it + iproute_pkg="iproute2" + # Otherwise, + else + # use iproute + iproute_pkg="iproute" + fi + # Check for and determine version number (major and minor) of current php install + if command -v php &> /dev/null; then + phpInsVersion="$(php -v | head -n1 | grep -Po '(?<=PHP )[^ ]+')" + echo -e " ${INFO} Existing PHP installation detected : PHP version $phpInsVersion" + phpInsMajor="$(echo "$phpInsVersion" | cut -d\. -f1)" + phpInsMinor="$(echo "$phpInsVersion" | cut -d\. -f2)" + # Is installed php version 7.0 or greater + if [ "$(echo "$phpInsMajor.$phpInsMinor < 7.0" | bc )" == 0 ]; then + phpInsNewer=true + fi + fi + # Check if installed php is v 7.0, or newer to determine packages to install + if [[ "$phpInsNewer" != true ]]; then + # Prefer the php metapackage if it's there + if ${PKG_MANAGER} install --dry-run php > /dev/null 2>&1; then + phpVer="php" + # fall back on the php5 packages + else + phpVer="php5" + fi + else + # Newer php is installed, its common, cgi & sqlite counterparts are deps + phpVer="php$phpInsMajor.$phpInsMinor" + fi + # We also need the correct version for `php-sqlite` (which differs across distros) + if ${PKG_MANAGER} install --dry-run ${phpVer}-sqlite3 > /dev/null 2>&1; then + phpSqlite="sqlite3" + else + phpSqlite="sqlite" + fi + # Since our install script is so large, we need several other programs to successfully get a machine provisioned + # These programs are stored in an array so they can be looped through later + INSTALLER_DEPS=(apt-utils dialog debconf dhcpcd5 git ${iproute_pkg} whiptail) + # Pi-hole itself has several dependencies that also need to be installed + PIHOLE_DEPS=(bc cron curl dnsutils iputils-ping lsof netcat psmisc sudo unzip wget idn2 sqlite3 libcap2-bin dns-root-data resolvconf) + # The Web dashboard has some that also need to be installed + # It's useful to separate the two since our repos are also setup as "Core" code and "Web" code + PIHOLE_WEB_DEPS=(lighttpd ${phpVer}-common ${phpVer}-cgi ${phpVer}-${phpSqlite}) + # The Web server user, + LIGHTTPD_USER="www-data" + # group, + LIGHTTPD_GROUP="www-data" + # and config file + LIGHTTPD_CFG="lighttpd.conf.debian" + + # A function to check... + test_dpkg_lock() { + # An iterator used for counting loop iterations + i=0 + # fuser is a program to show which processes use the named files, sockets, or filesystems + # So while the command is true + while fuser /var/lib/dpkg/lock >/dev/null 2>&1 ; do + # Wait half a second + sleep 0.5 + # and increase the iterator + ((i=i+1)) + done + # Always return success, since we only return if there is no + # lock (anymore) + return 0 } # If apt-get is not found, check for rpm to see if it's a Red Hat family OS elif command -v rpm &> /dev/null; then - # Then check if dnf or yum is the package manager - if command -v dnf &> /dev/null; then - PKG_MANAGER="dnf" - else - PKG_MANAGER="yum" - fi - - # Fedora and family update cache on every PKG_INSTALL call, no need for a separate update. - UPDATE_PKG_CACHE=":" - PKG_INSTALL=(${PKG_MANAGER} install -y) - PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l" - INSTALLER_DEPS=(dialog git iproute net-tools newt procps-ng which) - PIHOLE_DEPS=(bc bind-utils cronie curl findutils nmap-ncat sudo unzip wget libidn2 psmisc) - PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo) - LIGHTTPD_USER="lighttpd" - LIGHTTPD_GROUP="lighttpd" - LIGHTTPD_CFG="lighttpd.conf.fedora" - # If the host OS is Fedora, - if grep -qi 'fedora' /etc/redhat-release; then - # all required packages should be available by default with the latest fedora release - : # continue - # or if host OS is CentOS, - elif grep -qi 'centos' /etc/redhat-release; then - # Pi-Hole currently supports CentOS 7+ with PHP7+ - SUPPORTED_CENTOS_VERSION=7 - SUPPORTED_CENTOS_PHP_VERSION=7 - # Check current CentOS major release version - CURRENT_CENTOS_VERSION=$(rpm -q --queryformat '%{VERSION}' centos-release) - # Check if CentOS version is supported - if [[ $CURRENT_CENTOS_VERSION -lt $SUPPORTED_CENTOS_VERSION ]]; then - echo -e " ${CROSS} CentOS $CURRENT_CENTOS_VERSION is not suported." - echo -e " Please update to CentOS release $SUPPORTED_CENTOS_VERSION or later" - # exit the installer - exit - fi - # on CentOS we need to add the EPEL repository to gain access to Fedora packages - EPEL_PKG="epel-release" - rpm -q ${EPEL_PKG} &> /dev/null || rc=$? - if [[ $rc -ne 0 ]]; then - echo -e " ${INFO} Enabling EPEL package repository (https://fedoraproject.org/wiki/EPEL)" - "${PKG_INSTALL[@]}" ${EPEL_PKG} &> /dev/null - echo -e " ${TICK} Installed ${EPEL_PKG}" - fi - - # The default php on CentOS 7.x is 5.4 which is EOL - # Check if the version of PHP available via installed repositories is >= to PHP 7 - AVAILABLE_PHP_VERSION=$(${PKG_MANAGER} info php | grep -i version | grep -o '[0-9]\+' | head -1) - if [[ $AVAILABLE_PHP_VERSION -ge $SUPPORTED_CENTOS_PHP_VERSION ]]; then - # Since PHP 7 is available by default, install via default PHP package names - : # do nothing as PHP is current + # Then check if dnf or yum is the package manager + if command -v dnf &> /dev/null; then + PKG_MANAGER="dnf" else - REMI_PKG="remi-release" - REMI_REPO="remi-php72" - rpm -q ${REMI_PKG} &> /dev/null || rc=$? - if [[ $rc -ne 0 ]]; then - # The PHP version available via default repositories is older than version 7 - if ! whiptail --defaultno --title "PHP 7 Update (recommended)" --yesno "PHP 7.x is recommended for both security and language features.\\nWould you like to install PHP7 via Remi's RPM repository?\\n\\nSee: https://rpms.remirepo.net for more information" ${r} ${c}; then - # User decided to NOT update PHP from REMI, attempt to install the default available PHP version - echo -e " ${INFO} User opt-out of PHP 7 upgrade on CentOS. Deprecated PHP may be in use." - : # continue with unsupported php version - else - echo -e " ${INFO} Enabling Remi's RPM repository (https://rpms.remirepo.net)" - "${PKG_INSTALL[@]}" "https://rpms.remirepo.net/enterprise/${REMI_PKG}-$(rpm -E '%{rhel}').rpm" &> /dev/null - # enable the PHP 7 repository via yum-config-manager (provided by yum-utils) - "${PKG_INSTALL[@]}" "yum-utils" &> /dev/null - yum-config-manager --enable ${REMI_REPO} &> /dev/null - echo -e " ${TICK} Remi's RPM repository has been enabled for PHP7" + PKG_MANAGER="yum" + fi + # Fedora and family update cache on every PKG_INSTALL call, no need for a separate update. + UPDATE_PKG_CACHE=":" + PKG_INSTALL=(${PKG_MANAGER} install -y) + PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l" + INSTALLER_DEPS=(dialog git iproute net-tools newt procps-ng which) + PIHOLE_DEPS=(bc bind-utils cronie curl findutils nmap-ncat sudo unzip wget libidn2 psmisc) + PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo) + LIGHTTPD_USER="lighttpd" + LIGHTTPD_GROUP="lighttpd" + LIGHTTPD_CFG="lighttpd.conf.fedora" + # If the host OS is Fedora, + if grep -qi 'fedora' /etc/redhat-release; then + # all required packages should be available by default with the latest fedora release + : # continue + # or if host OS is CentOS, + elif grep -qi 'centos' /etc/redhat-release; then + # Pi-Hole currently supports CentOS 7+ with PHP7+ + SUPPORTED_CENTOS_VERSION=7 + SUPPORTED_CENTOS_PHP_VERSION=7 + # Check current CentOS major release version + CURRENT_CENTOS_VERSION=$(rpm -q --queryformat '%{VERSION}' centos-release) + # Check if CentOS version is supported + if [[ $CURRENT_CENTOS_VERSION -lt $SUPPORTED_CENTOS_VERSION ]]; then + echo -e " ${CROSS} CentOS $CURRENT_CENTOS_VERSION is not suported." + echo -e " Please update to CentOS release $SUPPORTED_CENTOS_VERSION or later" + # exit the installer + exit + fi + # on CentOS we need to add the EPEL repository to gain access to Fedora packages + EPEL_PKG="epel-release" + rpm -q ${EPEL_PKG} &> /dev/null || rc=$? + if [[ $rc -ne 0 ]]; then + echo -e " ${INFO} Enabling EPEL package repository (https://fedoraproject.org/wiki/EPEL)" + "${PKG_INSTALL[@]}" ${EPEL_PKG} &> /dev/null + echo -e " ${TICK} Installed ${EPEL_PKG}" fi - fi - fi - else - # If not a supported version of Fedora or CentOS, - echo -e " ${CROSS} Unsupported RPM based distribution" - # exit the installer - exit - fi + # The default php on CentOS 7.x is 5.4 which is EOL + # Check if the version of PHP available via installed repositories is >= to PHP 7 + AVAILABLE_PHP_VERSION=$(${PKG_MANAGER} info php | grep -i version | grep -o '[0-9]\+' | head -1) + if [[ $AVAILABLE_PHP_VERSION -ge $SUPPORTED_CENTOS_PHP_VERSION ]]; then + # Since PHP 7 is available by default, install via default PHP package names + : # do nothing as PHP is current + else + REMI_PKG="remi-release" + REMI_REPO="remi-php72" + rpm -q ${REMI_PKG} &> /dev/null || rc=$? + if [[ $rc -ne 0 ]]; then + # The PHP version available via default repositories is older than version 7 + if ! whiptail --defaultno --title "PHP 7 Update (recommended)" --yesno "PHP 7.x is recommended for both security and language features.\\nWould you like to install PHP7 via Remi's RPM repository?\\n\\nSee: https://rpms.remirepo.net for more information" ${r} ${c}; then + # User decided to NOT update PHP from REMI, attempt to install the default available PHP version + echo -e " ${INFO} User opt-out of PHP 7 upgrade on CentOS. Deprecated PHP may be in use." + : # continue with unsupported php version + else + echo -e " ${INFO} Enabling Remi's RPM repository (https://rpms.remirepo.net)" + "${PKG_INSTALL[@]}" "https://rpms.remirepo.net/enterprise/${REMI_PKG}-$(rpm -E '%{rhel}').rpm" &> /dev/null + # enable the PHP 7 repository via yum-config-manager (provided by yum-utils) + "${PKG_INSTALL[@]}" "yum-utils" &> /dev/null + yum-config-manager --enable ${REMI_REPO} &> /dev/null + echo -e " ${TICK} Remi's RPM repository has been enabled for PHP7" + fi + fi + fi + else + # If not a supported version of Fedora or CentOS, + echo -e " ${CROSS} Unsupported RPM based distribution" + # exit the installer + exit + fi # If neither apt-get or rmp/dnf are found else - # it's not an OS we can support, - echo -e " ${CROSS} OS distribution not supported" - # so exit the installer - exit + # it's not an OS we can support, + echo -e " ${CROSS} OS distribution not supported" + # so exit the installer + exit fi } # A function for checking if a folder is a git repository is_repo() { - # Use a named, local variable instead of the vague $1, which is the first argument passed to this function - # These local variables should always be lowercase - local directory="${1}" - # A local variable for the current directory - local curdir - # A variable to store the return code - local rc - # Assign the current directory variable by using pwd - curdir="${PWD}" - # If the first argument passed to this function is a directory, - if [[ -d "${directory}" ]]; then - # move into the directory - cd "${directory}" - # Use git to check if the folder is a repo - # git -C is not used here to support git versions older than 1.8.4 - git status --short &> /dev/null || rc=$? - # If the command was not successful, - else - # Set a non-zero return code if directory does not exist - rc=1 - fi - # Move back into the directory the user started in - cd "${curdir}" - # Return the code; if one is not set, return 0 - return "${rc:-0}" + # Use a named, local variable instead of the vague $1, which is the first argument passed to this function + # These local variables should always be lowercase + local directory="${1}" + # A local variable for the current directory + local curdir + # A variable to store the return code + local rc + # Assign the current directory variable by using pwd + curdir="${PWD}" + # If the first argument passed to this function is a directory, + if [[ -d "${directory}" ]]; then + # move into the directory + cd "${directory}" + # Use git to check if the folder is a repo + # git -C is not used here to support git versions older than 1.8.4 + git status --short &> /dev/null || rc=$? + # If the command was not successful, + else + # Set a non-zero return code if directory does not exist + rc=1 + fi + # Move back into the directory the user started in + cd "${curdir}" + # Return the code; if one is not set, return 0 + return "${rc:-0}" } # A function to clone a repo make_repo() { - # Set named variables for better readability - local directory="${1}" - local remoteRepo="${2}" - # The message to display when this function is running - str="Clone ${remoteRepo} into ${directory}" - # Display the message and use the color table to preface the message with an "info" indicator - echo -ne " ${INFO} ${str}..." - # If the directory exists, - if [[ -d "${directory}" ]]; then - # delete everything in it so git can clone into it - rm -rf "${directory}" - fi - # Clone the repo and return the return code from this command - git clone -q --depth 1 "${remoteRepo}" "${directory}" &> /dev/null || return $? - # Show a colored message showing it's status - echo -e "${OVER} ${TICK} ${str}" - # Always return 0? Not sure this is correct - return 0 + # Set named variables for better readability + local directory="${1}" + local remoteRepo="${2}" + # The message to display when this function is running + str="Clone ${remoteRepo} into ${directory}" + # Display the message and use the color table to preface the message with an "info" indicator + echo -ne " ${INFO} ${str}..." + # If the directory exists, + if [[ -d "${directory}" ]]; then + # delete everything in it so git can clone into it + rm -rf "${directory}" + fi + # Clone the repo and return the return code from this command + git clone -q --depth 1 "${remoteRepo}" "${directory}" &> /dev/null || return $? + # Show a colored message showing it's status + echo -e "${OVER} ${TICK} ${str}" + # Always return 0? Not sure this is correct + return 0 } # We need to make sure the repos are up-to-date so we can effectively install Clean out the directory if it exists for git to clone into update_repo() { - # Use named, local variables - # As you can see, these are the same variable names used in the last function, - # but since they are local, their scope does not go beyond this function - # This helps prevent the wrong value from being assigned if you were to set the variable as a GLOBAL one - local directory="${1}" - local curdir - - # A variable to store the message we want to display; - # Again, it's useful to store these in variables in case we need to reuse or change the message; - # we only need to make one change here - local str="Update repo in ${1}" - - # Make sure we know what directory we are in so we can move back into it - curdir="${PWD}" - # Move into the directory that was passed as an argument - cd "${directory}" &> /dev/null || return 1 - # Let the user know what's happening - echo -ne " ${INFO} ${str}..." - # Stash any local commits as they conflict with our working code - git stash --all --quiet &> /dev/null || true # Okay for stash failure - git clean --quiet --force -d || true # Okay for already clean directory - # Pull the latest commits - git pull --quiet &> /dev/null || return $? - # Show a completion message - echo -e "${OVER} ${TICK} ${str}" - # Move back into the original directory - cd "${curdir}" &> /dev/null || return 1 - return 0 + # Use named, local variables + # As you can see, these are the same variable names used in the last function, + # but since they are local, their scope does not go beyond this function + # This helps prevent the wrong value from being assigned if you were to set the variable as a GLOBAL one + local directory="${1}" + local curdir + + # A variable to store the message we want to display; + # Again, it's useful to store these in variables in case we need to reuse or change the message; + # we only need to make one change here + local str="Update repo in ${1}" + + # Make sure we know what directory we are in so we can move back into it + curdir="${PWD}" + # Move into the directory that was passed as an argument + cd "${directory}" &> /dev/null || return 1 + # Let the user know what's happening + echo -ne " ${INFO} ${str}..." + # Stash any local commits as they conflict with our working code + git stash --all --quiet &> /dev/null || true # Okay for stash failure + git clean --quiet --force -d || true # Okay for already clean directory + # Pull the latest commits + git pull --quiet &> /dev/null || return $? + # Show a completion message + echo -e "${OVER} ${TICK} ${str}" + # Move back into the original directory + cd "${curdir}" &> /dev/null || return 1 + return 0 } # A function that combines the functions previously made getGitFiles() { - # Setup named variables for the git repos - # We need the directory - local directory="${1}" - # as well as the repo URL - local remoteRepo="${2}" - # A local variable containing the message to be displayed - local str="Check for existing repository in ${1}" - # Show the message - echo -ne " ${INFO} ${str}..." - # Check if the directory is a repository - if is_repo "${directory}"; then - # Show that we're checking it - echo -e "${OVER} ${TICK} ${str}" - # Update the repo, returning an error message on failure - update_repo "${directory}" || { echo -e "\\n ${COL_LIGHT_RED}Error: Could not update local repository. Contact support.${COL_NC}"; exit 1; } - # If it's not a .git repo, - else - # Show an error - echo -e "${OVER} ${CROSS} ${str}" - # Attempt to make the repository, showing an error on failure - make_repo "${directory}" "${remoteRepo}" || { echo -e "\\n ${COL_LIGHT_RED}Error: Could not update local repository. Contact support.${COL_NC}"; exit 1; } - fi - # echo a blank line - echo "" - # and return success? - return 0 + # Setup named variables for the git repos + # We need the directory + local directory="${1}" + # as well as the repo URL + local remoteRepo="${2}" + # A local variable containing the message to be displayed + local str="Check for existing repository in ${1}" + # Show the message + echo -ne " ${INFO} ${str}..." + # Check if the directory is a repository + if is_repo "${directory}"; then + # Show that we're checking it + echo -e "${OVER} ${TICK} ${str}" + # Update the repo, returning an error message on failure + update_repo "${directory}" || { echo -e "\\n ${COL_LIGHT_RED}Error: Could not update local repository. Contact support.${COL_NC}"; exit 1; } + # If it's not a .git repo, + else + # Show an error + echo -e "${OVER} ${CROSS} ${str}" + # Attempt to make the repository, showing an error on failure + make_repo "${directory}" "${remoteRepo}" || { echo -e "\\n ${COL_LIGHT_RED}Error: Could not update local repository. Contact support.${COL_NC}"; exit 1; } + fi + # echo a blank line + echo "" + # and return success? + return 0 } # Reset a repo to get rid of any local changed resetRepo() { - # Use named variables for arguments - local directory="${1}" - # Move into the directory - cd "${directory}" &> /dev/null || return 1 - # Store the message in a variable - str="Resetting repository within ${1}..." - # Show the message - echo -ne " ${INFO} ${str}" - # Use git to remove the local changes - git reset --hard &> /dev/null || return $? - # And show the status - echo -e "${OVER} ${TICK} ${str}" - # Returning success anyway? - return 0 + # Use named variables for arguments + local directory="${1}" + # Move into the directory + cd "${directory}" &> /dev/null || return 1 + # Store the message in a variable + str="Resetting repository within ${1}..." + # Show the message + echo -ne " ${INFO} ${str}" + # Use git to remove the local changes + git reset --hard &> /dev/null || return $? + # And show the status + echo -e "${OVER} ${TICK} ${str}" + # Returning success anyway? + return 0 } # We need to know the IPv4 information so we can effectively setup the DNS server # Without this information, we won't know where to Pi-hole will be found find_IPv4_information() { - # Named, local variables - local route - # Find IP used to route to outside world by checking the the route to Google's public DNS server - route=$(ip route get 8.8.8.8) - # Use awk to strip out just the interface device as it is used in future commands - IPv4dev=$(awk '{for (i=1; i<=NF; i++) if ($i~/dev/) print $(i+1)}' <<< "${route}") - # Get just the IP address - IPv4bare=$(awk '{print $7}' <<< "${route}") - # Append the CIDR notation to the IP address - IPV4_ADDRESS=$(ip -o -f inet addr show | grep "${IPv4bare}" | awk '{print $4}' | awk 'END {print}') - # Get the default gateway (the way to reach the Internet) - IPv4gw=$(awk '{print $3}' <<< "${route}") - + # Named, local variables + local route + # Find IP used to route to outside world by checking the the route to Google's public DNS server + route=$(ip route get 8.8.8.8) + # Use awk to strip out just the interface device as it is used in future commands + IPv4dev=$(awk '{for (i=1; i<=NF; i++) if ($i~/dev/) print $(i+1)}' <<< "${route}") + # Get just the IP address + IPv4bare=$(awk '{print $7}' <<< "${route}") + # Append the CIDR notation to the IP address + IPV4_ADDRESS=$(ip -o -f inet addr show | grep "${IPv4bare}" | awk '{print $4}' | awk 'END {print}') + # Get the default gateway (the way to reach the Internet) + IPv4gw=$(awk '{print $3}' <<< "${route}") } # Get available interfaces that are UP get_available_interfaces() { - # There may be more than one so it's all stored in a variable - availableInterfaces=$(ip --oneline link show up | grep -v "lo" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) + # There may be more than one so it's all stored in a variable + availableInterfaces=$(ip --oneline link show up | grep -v "lo" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) } # A function for displaying the dialogs the user sees when first running the installer welcomeDialogs() { - # Display the welcome dialog using an appropriately sized window via the calculation conducted earlier in the script - whiptail --msgbox --backtitle "Welcome" --title "Pi-hole automated installer" "\\n\\nThis installer will transform your device into a network-wide ad blocker!" ${r} ${c} + # Display the welcome dialog using an appropriately sized window via the calculation conducted earlier in the script + whiptail --msgbox --backtitle "Welcome" --title "Pi-hole automated installer" "\\n\\nThis installer will transform your device into a network-wide ad blocker!" ${r} ${c} - # Request that users donate if they enjoy the software since we all work on it in our free time - whiptail --msgbox --backtitle "Plea" --title "Free and open source" "\\n\\nThe Pi-hole is free, but powered by your donations: http://pi-hole.net/donate" ${r} ${c} + # Request that users donate if they enjoy the software since we all work on it in our free time + whiptail --msgbox --backtitle "Plea" --title "Free and open source" "\\n\\nThe Pi-hole is free, but powered by your donations: http://pi-hole.net/donate" ${r} ${c} - # Explain the need for a static address - whiptail --msgbox --backtitle "Initiating network interface" --title "Static IP Needed" "\\n\\nThe Pi-hole is a SERVER so it needs a STATIC IP ADDRESS to function properly. + # Explain the need for a static address + whiptail --msgbox --backtitle "Initiating network interface" --title "Static IP Needed" "\\n\\nThe Pi-hole is a SERVER so it needs a STATIC IP ADDRESS to function properly. In the next section, you can choose to use your current network settings (DHCP) or to manually edit them." ${r} ${c} } # We need to make sure there is enough space before installing, so there is a function to check this verifyFreeDiskSpace() { - - # 50MB is the minimum space needed (45MB install (includes web admin bootstrap/jquery libraries etc) + 5MB one day of logs.) - # - Fourdee: Local ensures the variable is only created, and accessible within this function/void. Generally considered a "good" coding practice for non-global variables. - local str="Disk space check" - # Required space in KB - local required_free_kilobytes=51200 - # Calculate existing free space on this machine - local existing_free_kilobytes - existing_free_kilobytes=$(df -Pk | grep -m1 '\/$' | awk '{print $4}') - - # If the existing space is not an integer, - if ! [[ "${existing_free_kilobytes}" =~ ^([0-9])+$ ]]; then - # show an error that we can't determine the free space - echo -e " ${CROSS} ${str} - Unknown free disk space! - We were unable to determine available free disk space on this system. - You may override this check, however, it is not recommended - The option '${COL_LIGHT_RED}--i_do_not_follow_recommendations${COL_NC}' can override this - e.g: curl -L https://install.pi-hole.net | bash /dev/stdin ${COL_LIGHT_RED}