From 6830b08723c17b1c1b9a43d341b12b0951682bf9 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Mon, 24 Jul 2017 21:24:34 +1000 Subject: [PATCH 01/50] Clean up and optimise Gravity * Shellcheck validation * Made variable names, function names, comments and output clearer to understand * Quoted and braced variables and conditionals * Fix adlists.list handling logic, and remove any CR line endings * Make CP/MV/RM provide user-friendly output upon failure * Change adlists.list retrieval logic * Moved and fixed adlists.list domain parsing logic * Create gravity_ParseFileAsDomains() function to handle parsing of source files * If no changes to a list is detected, print no output * Ensure each source blocklist has a final newline * Format number output as currency * Make array of adlists domain sources unique to prevent redundant whitelisting * Merged bash IPv4/IPv6 hosts formatting IF statement into an awk one-liner * Trap Ctrl-C cancellations and run gravity_Cleanup() * Use new gravity_Cleanup() function on errors and script completion * Ensure that dnsmasq uses force-reload when gravity is invoked * Add --wildcard option to ensure dnsmasq is restarted upon b/wlisting of a wildcard Signed-off-by: WaLLy3K --- gravity.sh | 674 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 388 insertions(+), 286 deletions(-) diff --git a/gravity.sh b/gravity.sh index 168a4350..852356d7 100755 --- a/gravity.sh +++ b/gravity.sh @@ -1,18 +1,66 @@ #!/usr/bin/env bash +# shellcheck disable=SC1090 + # Pi-hole: A black hole for Internet advertisements # (c) 2017 Pi-hole, LLC (https://pi-hole.net) # Network-wide ad blocking via your own hardware. # +# Usage: "pihole -g" # Compiles a list of ad-serving domains by downloading them from multiple sources # # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. -# Run this script as root or under sudo - coltable="/opt/pihole/COL_TABLE" source ${coltable} +basename="pihole" +PIHOLE_COMMAND="/usr/local/bin/${basename}" +WHITELIST_COMMAND="${PIHOLE_COMMAND} -w" + +piholeDir="/etc/${basename}" +piholeRepo="/etc/.${basename}" + +adListFile="${piholeDir}/adlists.list" +adListDefault="${piholeDir}/adlists.default" +adListRepoDefault="${piholeRepo}/adlists.default" + +whitelistFile="${piholeDir}/whitelist.txt" +blacklistFile="${piholeDir}/blacklist.txt" +wildcardFile="/etc/dnsmasq.d/03-pihole-wildcard.conf" + +adList="${piholeDir}/gravity.list" +blackList="${piholeDir}/black.list" +localList="${piholeDir}/local.list" + +domainsExtension="domains" +matterAndLight="${basename}.0.matterandlight.txt" +supernova="${basename}.1.supernova.txt" +preEventHorizon="list.preEventHorizon" +eventHorizon="${basename}.2.supernova.txt" +accretionDisc="${basename}.3.accretionDisc.txt" + +skipDownload="false" + +# Source setupVars from install script +setupVars="${piholeDir}/setupVars.conf" +if [[ -f "${setupVars}" ]];then + source "${setupVars}" + + # Remove CIDR mask from IPv4/6 addresses + IPV4_ADDRESS="${IPV4_ADDRESS%/*}" + IPV6_ADDRESS="${IPV6_ADDRESS%/*}" +else + echo -e " ${COL_LIGHT_RED}Installation Failure: ${setupVars} does not exist! ${COL_NC} + Please run 'pihole -r', and choose the 'reconfigure' option to fix." + exit 1 +fi + +# Warn users still using pihole.conf that it no longer has any effect +if [[ -r "${piholeDir}/pihole.conf" ]]; then + echo -e " ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}" +fi + helpFunc() { echo "Usage: pihole -g Update domains from blocklists specified in adlists.list @@ -23,267 +71,307 @@ Options: exit 0 } -PIHOLE_COMMAND="/usr/local/bin/pihole" - -adListFile=/etc/pihole/adlists.list -adListDefault=/etc/pihole/adlists.default # Deprecated -adListRepoDefault=/etc/.pihole/adlists.default -whitelistScript="${PIHOLE_COMMAND} -w" -whitelistFile=/etc/pihole/whitelist.txt -blacklistFile=/etc/pihole/blacklist.txt -readonly wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf" - -# Source the setupVars from install script for the IP -setupVars=/etc/pihole/setupVars.conf -if [[ -f "${setupVars}" ]];then - . /etc/pihole/setupVars.conf -else - echo -e " ${COL_LIGHT_RED}Error: /etc/pihole/setupVars.conf missing. Possible installation failure.${COL_NC} - Please run 'pihole -r', and choose the 'reconfigure' option to reconfigure." - exit 1 -fi - -# Remove the /* from the end of the IP addresses -IPV4_ADDRESS=${IPV4_ADDRESS%/*} -IPV6_ADDRESS=${IPV6_ADDRESS%/*} - -# Variables for various stages of downloading and formatting the list -basename=pihole -piholeDir=/etc/${basename} -adList=${piholeDir}/gravity.list -blackList=${piholeDir}/black.list -localList=${piholeDir}/local.list -justDomainsExtension=domains -matterAndLight=${basename}.0.matterandlight.txt -supernova=${basename}.1.supernova.txt -preEventHorizon=list.preEventHorizon -eventHorizon=${basename}.2.supernova.txt -accretionDisc=${basename}.3.accretionDisc.txt - -skipDownload=false - -# Warn users still using pihole.conf that it no longer has any effect -if [[ -r ${piholeDir}/pihole.conf ]]; then -echo -e " ${COL_LIGHT_RED}pihole.conf file no longer supported. Overrides in this file are ignored.${COL_NC}" -fi - -########################### -# Collapse - begin formation of pihole -gravity_collapse() { - - #New Logic: - # Does /etc/pihole/adlists.list exist? If so leave it alone - # If not, cp /etc/.pihole/adlists.default /etc/pihole/adlists.list - # Read from adlists.list - - # The following two blocks will sort out any missing adlists in the /etc/pihole directory, and remove legacy adlists.default - if [[ -f "${adListDefault}" ]] && [[ -f "${adListFile}" ]]; then - rm "${adListDefault}" - fi - - if [ ! -f "${adListFile}" ]; then - cp "${adListRepoDefault}" "${adListFile}" - fi - +# Retrieve blocklist URLs from adlists.list +gravity_Collapse() { echo -e " ${INFO} Neutrino emissions detected..." - echo "" - local str="Pulling source lists into range" + + # Handle "adlists.list" and "adlists.default" files + if [[ -f "${adListDefault}" ]] && [[ -f "${adListFile}" ]]; then + # Remove superceded $adListDefault file + rm "${adListDefault}" 2> /dev/null || \ + echo -e " ${CROSS} Unable to remove ${adListDefault}" + elif [[ ! -f "${adListFile}" ]]; then + # Create "adlists.list" + cp "${adListRepoDefault}" "${adListFile}" 2> /dev/null || \ + echo -e " ${CROSS} Unable to copy ${adListFile##*/} from ${piholeRepo}" + fi + + local str="Pulling blocklist source list into range" echo -ne " ${INFO} ${str}..." - sources=() - while IFS= read -r line || [[ -n "$line" ]]; do - # Do not read commented out or blank lines - if [[ ${line} = \#* ]] || [[ ! ${line} ]]; then - echo "" > /dev/null - else - sources+=(${line}) - fi - done < ${adListFile} + # Retrieve source URLs from $adListFile + # Logic: Remove comments, CR line endings and empty lines + mapfile -t sources < <(awk '!/^[#@;!\[]/ {gsub(/\r$/, "", $0); if ($1) { print $1 } }' "${adListFile}" 2> /dev/null) - echo -e "${OVER} ${TICK} ${str}" + # Parse source domains from $sources + # Logic: Split by folder/port and remove URL protocol/password + mapfile -t sourceDomains < <( + awk -F '[/:]' '{ + gsub(/(.*:\/\/|.*:.*@)/, "", $0) + print $1 + }' <<< "$(printf '%s\n' "${sources[@]}")" 2> /dev/null + ) + + if [[ -n "${sources[*]}" ]] || [[ -n "${sourceDomains[*]}" ]]; then + echo -e "${OVER} ${TICK} ${str}" + else + echo -e "${OVER} ${CROSS} ${str}" + gravity_Cleanup "error" + fi } -# patternCheck - check to see if curl downloaded any new files. -gravity_patternCheck() { - patternBuffer=$1 - success=$2 - error=$3 - if [[ "${success}" = true ]]; then - # Check if download was successful but list has not been modified - if [[ "${error}" == "304" ]]; then - echo -e " ${TICK} No changes detected, transport skipped!" - # Check if the patternbuffer is a non-zero length file - elif [[ -s "${patternBuffer}" ]]; then - # Some blocklists are copyright, they need to be downloaded and stored - # as is. They can be processed for content after they have been saved. - mv "${patternBuffer}" "${saveLocation}" - echo -e " ${TICK} List updated, transport successful!" - else - # Empty file -> use previously downloaded list - echo -e " ${INFO} Received empty file, ${COL_LIGHT_GREEN}using cached one${COL_NC} (list not updated!)" - fi +# Parse source file into domains-only format +gravity_ParseFileAsDomains() { + local source destination hostsFilter firstLine abpFilter + source="${1}" + destination="${2}" + + # Determine how to parse source file + if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then + # Symbols used as comments: "#;@![/" + commentPattern="[#;@![\\/]" + + # Parse consolidated file by removing comments and hosts IP's + # Logic: Process lines which do not begin with comments + awk '!/^'"${commentPattern}"'/ { + # If there are multiple words seperated by space + if (NF>1) { + # Remove comments (Inc. prefixed spaces/tabs) + if ($0 ~ /'"${commentPattern}"'/) { gsub("( | )'"${commentPattern}"'.*", "", $0) } + # Print consecutive domains + if ($3) { + $1="" + gsub("^ ", "", $0) + print $0 + # Print single domain + } else if ($2) { + print $2 + } + # Print single domain + } else if($1) { + print $1 + } + }' "${source}" 2> /dev/null > "${destination}" else - # Check if cached list exists - if [[ -r "${saveLocation}" ]]; then - echo -e " ${CROSS} List download failed, using cached list (list not updated!)" + # Individual file parsing + # Logic: comments are kept, and domains are extracted from each line + read -r firstLine < "${source}" + + # Determine how to parse individual source file + if [[ "${firstLine,,}" =~ "adblock" ]] || [[ "${firstLine,,}" =~ "ublock" ]] || [[ "${firstLine,,}" =~ "! checksum" ]]; then + # Parse Adblock domains & comments: https://adblockplus.org/filter-cheatsheet + abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/" + awk ''"${abpFilter}"' { + # Remove valid adblock type options + gsub(/~?(important|third-party|popup|subdocument|websocket),?/, "", $0) + # Remove starting domain name anchor "||" and ending seperator "^$" ($ optional) + gsub(/(\|\||\^\$?$)/, "", $0) + # Remove lines which are only IPv4 addresses or contain "^/*" + if ($0 ~ /(^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$|[\\^\/\*])/) { $0="" } + # Print if not empty + if ($0) { print $0 } + }' "${source}" 2> /dev/null > "${destination}" + echo -e " ${TICK} Format: Adblock" + elif grep -q -E "^(https?://|([0-9]{1,3}\.){3}[0-9]{1,3}$)" "${source}" &> /dev/null; then + # Parse URLs + awk '{ + # Remove URL protocol, optional "username:password@", and ":?/;" + if ($0 ~ /[:?\/;]/) { gsub(/(^.*:\/\/(.*:.*@)?|[:?\/;].*)/, "", $0) } + # Remove lines which are only IPv4 addresses + if ($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" } + if ($0) { print $0 } + }' "${source}" 2> /dev/null > "${destination}" + echo -e " ${TICK} Format: URL" else - echo -e " ${CROSS} Download failed and no cached list available (list will not be considered)" + # Keep hosts/domains file in same format as it was downloaded + output=$( { mv "${source}" "${destination}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e " ${CROSS} Unable to move tmp file to ${piholeDir} + ${output}" + gravity_Cleanup "error" + fi fi fi } -# transport - curl the specified url with any needed command extentions -gravity_transport() { - url=$1 - cmd_ext=$2 - agent=$3 +# Determine output based on gravity_Transport() status +gravity_AdvancedTransport() { + local patternBuffer success error output status + patternBuffer="${1}" + success="${2}" + error="${3}" - # tmp file, so we don't have to store the (long!) lists in RAM + if [[ "${success}" = true ]]; then + if [[ "${error}" == "304" ]]; then + : # Print no output + # Check if the patternbuffer is a non-zero length file + elif [[ -s "${patternBuffer}" ]]; then + # Parse Adblock/URL format blocklists and include comments + # HOSTS format is moved as-is + gravity_ParseFileAsDomains "${patternBuffer}" "${saveLocation}" "1" + else + # Fall back to previously cached list if current $patternBuffer is empty + echo -e " ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" + fi + else + # Determine if cached list exists + if [[ -r "${saveLocation}" ]]; then + echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" + else + echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" + fi + fi +} + +# Curl the specified URL with any necessary command extentions +gravity_Transport() { + local url cmd_ext agent + url="${1}" + cmd_ext="${2}" + agent="${3}" + + # Store downloaded content to temp file instead of RAM patternBuffer=$(mktemp) heisenbergCompensator="" - if [[ -r ${saveLocation} ]]; then + if [[ -r "${saveLocation}" ]]; then # If domain has been saved, add file for date check to only download newer heisenbergCompensator="-z ${saveLocation}" fi - # Silently curl url - echo -e "${OVER} ${TICK} ${str}" local str="Status:" - echo -ne " ${INFO} ${str} Pending" - err=$(curl -s -L ${cmd_ext} ${heisenbergCompensator} -w %{http_code} -A "${agent}" ${url} -o ${patternBuffer}) + echo -ne " ${INFO} ${str} Pending..." + # shellcheck disable=SC2086 + httpCode=$(curl -s -L ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${patternBuffer}" 2> /dev/null) - # Analyze http response - case "$err" in - "200" ) echo -e "${OVER} ${TICK} ${str} Success (OK)"; success=true;; - "304" ) echo -e "${OVER} ${TICK} ${str} Not modified"; success=true;; - "403" ) echo -e "${OVER} ${CROSS} ${str} Forbidden"; success=false;; - "404" ) echo -e "${OVER} ${CROSS} ${str} Not found"; success=false;; - "408" ) echo -e "${OVER} ${CROSS} ${str} Time-out"; success=false;; - "451" ) echo -e "${OVER} ${CROSS} ${str} Unavailable For Legal Reasons"; success=false;; - "521" ) echo -e "${OVER} ${CROSS} ${str} Web Server Is Down (Cloudflare)"; success=false;; - "522" ) echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Cloudflare)"; success=false;; - "500" ) echo -e "${OVER} ${CROSS} ${str} Internal Server Error"; success=false;; - * ) echo -e "${OVER} ${CROSS} ${str} Status $err"; success=false;; + # Determine "Status:" output based on HTTP response + case "$httpCode" in + "200" ) echo -e "${OVER} ${TICK} ${str} Transport successful"; success=true;; + "304" ) echo -e "${OVER} ${TICK} ${str} No changes detected"; success=true;; + "403" ) echo -e "${OVER} ${CROSS} ${str} Forbidden"; success=false;; + "404" ) echo -e "${OVER} ${CROSS} ${str} Not found"; success=false;; + "408" ) echo -e "${OVER} ${CROSS} ${str} Time-out"; success=false;; + "451" ) echo -e "${OVER} ${CROSS} ${str} Unavailable For Legal Reasons"; success=false;; + "521" ) echo -e "${OVER} ${CROSS} ${str} Web Server Is Down (Cloudflare)"; success=false;; + "522" ) echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Cloudflare)"; success=false;; + "500" ) echo -e "${OVER} ${CROSS} ${str} Internal Server Error"; success=false;; + * ) echo -e "${OVER} ${CROSS} ${str} Status $httpCode"; success=false;; esac - # Process result - gravity_patternCheck "${patternBuffer}" "${success}" "${err}" + # Output additional info if success=false + gravity_AdvancedTransport "${patternBuffer}" "${success}" "${httpCode}" - # Delete temp file if it hasn't been moved + # Delete temp file if it has not been moved if [[ -f "${patternBuffer}" ]]; then rm "${patternBuffer}" fi } -# spinup - main gravity function -gravity_spinup() { +# Define User Agent and options for each blocklist +gravity_Pull() { + local agent url domain cmd_ext str + echo "" - # Loop through domain list. Download each one and remove commented lines (lines beginning with '# 'or '/') and # blank lines + + # Loop through $sources to download each one for ((i = 0; i < "${#sources[@]}"; i++)); do - url=${sources[$i]} - # Get just the domain from the URL - domain=$(cut -d'/' -f3 <<< "${url}") + url="${sources[$i]}" + domain="${sourceDomains[$i]}" # Save the file as list.#.domain - saveLocation=${piholeDir}/list.${i}.${domain}.${justDomainsExtension} - activeDomains[$i]=${saveLocation} + saveLocation="${piholeDir}/list.${i}.${domain}.${domainsExtension}" + activeDomains[$i]="${saveLocation}" agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36" - # Use a case statement to download lists that need special cURL commands - # to complete properly and reset the user agent when required + # Use a case statement to download lists that need special commands case "${domain}" in - "pgl.yoyo.org") - cmd_ext="-d mimetype=plaintext -d hostformat=hosts" - ;; - - # Default is a simple request - *) cmd_ext="" + "pgl.yoyo.org") cmd_ext="-d mimetype=plaintext -d hostformat=hosts";; + *) cmd_ext="";; esac if [[ "${skipDownload}" == false ]]; then - local str="Aiming tractor beam at $domain" - echo -ne " ${INFO} ${str}..." + str="Target: $domain (${url##*/})" + echo -e " ${INFO} ${str}" + + gravity_Transport "$url" "$cmd_ext" "$agent" "$str" - gravity_transport "$url" "$cmd_ext" "$agent" "$str" echo "" fi done } -# Schwarzchild - aggregate domains to one list and add blacklisted domains -gravity_Schwarzchild() { - echo "" - # Find all active domains and compile them into one file and remove CRs - local str="Aggregating list of domains" +# Consolidate domains to one list and add blacklisted domains +gravity_Schwarzschild() { + local str lastLine + + str="Consolidating blocklists" echo -ne " ${INFO} ${str}..." - - truncate -s 0 ${piholeDir}/${matterAndLight} + + # Compile all blacklisted domains into one file and remove CRs + truncate -s 0 "${piholeDir}/${matterAndLight}" for i in "${activeDomains[@]}"; do # Only assimilate list if it is available (download might have failed permanently) if [[ -r "${i}" ]]; then - cat "${i}" | tr -d '\r' >> ${piholeDir}/${matterAndLight} + tr -d '\r' < "${i}" >> "${piholeDir}/${matterAndLight}" + + # Ensure each source blocklist has a final newline + lastLine=$(tail -1 "${piholeDir}/${matterAndLight}") + [[ "${#lastLine}" -gt 0 ]] && echo "" >> "${piholeDir}/${matterAndLight}" fi done echo -e "${OVER} ${TICK} ${str}" } +# Append blacklist entries to eventHorizon if they exist gravity_Blacklist() { - # Append blacklist entries to eventHorizon if they exist + local numBlacklisted plural str + if [[ -f "${blacklistFile}" ]]; then - numBlacklisted=$(wc -l < "${blacklistFile}") - plural=; [[ "$numBlacklisted" != "1" ]] && plural=s - local str="Exact blocked domain${plural}: $numBlacklisted" + numBlacklisted=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")") + plural=; [[ "${numBlacklisted}" != "1" ]] && plural=s + str="Exact blocked domain${plural}: $numBlacklisted" echo -e " ${INFO} ${str}" else echo -e " ${INFO} Nothing to blacklist!" fi } +# Return number of wildcards in output gravity_Wildcard() { - # Return number of wildcards in output - don't actually handle wildcards - if [[ -f "${wildcardlist}" ]]; then - numWildcards=$(grep -c ^ "${wildcardlist}") + local numWildcards plural + + if [[ -f "${wildcardFile}" ]]; then + numWildcards=$(grep -c ^ "${wildcardFile}") if [[ -n "${IPV4_ADDRESS}" ]] && [[ -n "${IPV6_ADDRESS}" ]];then let numWildcards/=2 fi - plural=; [[ "$numWildcards" != "1" ]] && plural=s + plural=; [[ "${numWildcards}" != "1" ]] && plural=s echo -e " ${INFO} Wildcard blocked domain${plural}: $numWildcards" else echo -e " ${INFO} No wildcards used!" fi - } +# Prevent the domains of adlist sources from being blacklisted by other blocklists gravity_Whitelist() { + local plural str + echo "" - # Prevent our sources from being pulled into the hole - plural=; [[ "${sources[@]}" != "1" ]] && plural=s - local str="Adding adlist source${plural} to the whitelist" + plural=; [[ "${#sources[*]}" != "1" ]] && plural=s + str="Adding blocklist source${plural} to the whitelist" echo -ne " ${INFO} ${str}..." - urls=() - for url in "${sources[@]}"; do - tmp=$(awk -F '/' '{print $3}' <<< "${url}") - urls=("${urls[@]}" ${tmp}) - done + # Create array of unique $sourceDomains + # shellcheck disable=SC2046 + read -r -a uniqDomains <<< $(awk '{ if(!a[$1]++) { print $1 } }' <<< "$(printf '%s\n' "${sourceDomains[@]}")") + + ${WHITELIST_COMMAND} -nr -q "${uniqDomains[*]}" > /dev/null echo -e "${OVER} ${TICK} ${str}" - # Ensure adlist domains are in whitelist.txt - ${whitelistScript} -nr -q "${urls[@]}" > /dev/null - - # Check whitelist.txt exists. + # Test existence of whitelist.txt if [[ -f "${whitelistFile}" ]]; then # Remove anything in whitelist.txt from the Event Horizon numWhitelisted=$(wc -l < "${whitelistFile}") - plural=; [[ "$numWhitelisted" != "1" ]] && plural=s + plural=; [[ "${numWhitelisted}" != "1" ]] && plural=s local str="Whitelisting $numWhitelisted domain${plural}" echo -ne " ${INFO} ${str}..." # Print everything from preEventHorizon into eventHorizon EXCEPT domains in whitelist.txt - grep -F -x -v -f ${whitelistFile} ${piholeDir}/${preEventHorizon} > ${piholeDir}/${eventHorizon} + grep -F -x -v -f "${whitelistFile}" "${piholeDir}/${preEventHorizon}" > "${piholeDir}/${eventHorizon}" echo -e "${OVER} ${TICK} ${str}" else @@ -291,140 +379,155 @@ gravity_Whitelist() { fi } -gravity_unique() { - # Sort and remove duplicates - local str="Removing duplicate domains" +# Sort and remove duplicate blacklisted domains +gravity_Unique() { + local str numberOf + + str="Removing duplicate domains" echo -ne " ${INFO} ${str}..." - - sort -u ${piholeDir}/${supernova} > ${piholeDir}/${preEventHorizon} - + sort -u "${piholeDir}/${supernova}" > "${piholeDir}/${preEventHorizon}" echo -e "${OVER} ${TICK} ${str}" - numberOf=$(wc -l < ${piholeDir}/${preEventHorizon}) - echo -e " ${INFO} ${COL_LIGHT_BLUE}${numberOf}${COL_NC} unique domains trapped in the event horizon." + + numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") + echo -e " ${INFO} ${COL_LIGHT_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon" } -gravity_doHostFormat() { - # Check vars from setupVars.conf to see if we're using IPv4, IPv6, Or both. - if [[ -n "${IPV4_ADDRESS}" ]] && [[ -n "${IPV6_ADDRESS}" ]];then - # Both IPv4 and IPv6 - awk -v ipv4addr="$IPV4_ADDRESS" -v ipv6addr="$IPV6_ADDRESS" '{sub(/\r$/,""); print ipv4addr" "$0"\n"ipv6addr" "$0}' >> "${2}" < "${1}" - elif [[ -n "${IPV4_ADDRESS}" ]] && [[ -z "${IPV6_ADDRESS}" ]];then - # Only IPv4 - awk -v ipv4addr="$IPV4_ADDRESS" '{sub(/\r$/,""); print ipv4addr" "$0}' >> "${2}" < "${1}" - elif [[ -z "${IPV4_ADDRESS}" ]] && [[ -n "${IPV6_ADDRESS}" ]];then - # Only IPv6 - awk -v ipv6addr="$IPV6_ADDRESS" '{sub(/\r$/,""); print ipv6addr" "$0}' >> "${2}" < "${1}" - elif [[ -z "${IPV4_ADDRESS}" ]] &&[[ -z "${IPV6_ADDRESS}" ]];then +# Parse list of domains into hosts format +gravity_ParseDomainsIntoHosts() { + if [[ -n "${IPV4_ADDRESS}" ]] || [[ -n "${IPV6_ADDRESS}" ]]; then + awk -v ipv4addr="$IPV4_ADDRESS" -v ipv6addr="$IPV6_ADDRESS" \ + '{sub(/\r$/,""); if(ipv4addr) { print ipv4addr" "$0; }; if(ipv6addr) { print ipv6addr" "$0; }}' >> "${2}" < "${1}" + else echo -e "${OVER} ${CROSS} ${str}" - echo -e " ${COL_LIGHT_RED}No IP Values found! Please run 'pihole -r' and choose reconfigure to restore values${COL_NC}" - exit 1 + echo -e " ${COL_LIGHT_RED}No IP addresses found! Please run 'pihole -r' to reconfigure${COL_NC}\\n" + gravity_Cleanup "error" fi } -gravity_hostFormatLocal() { - # Format domain list as "192.168.x.x domain.com" - +# Create "localhost" entries +gravity_ParseLocalDomains() { if [[ -f "/etc/hostname" ]]; then - hostname=$(< /etc/hostname) - elif [ -x "$(command -v hostname)" ]; then + hostname=$(< "/etc/hostname") + elif command -v hostname &> /dev/null; then hostname=$(hostname -f) else echo -e " ${CROSS} Unable to determine fully qualified domain name of host" fi - echo -e "${hostname}\npi.hole" > "${localList}.tmp" + echo -e "${hostname}\\npi.hole" > "${localList}.tmp" + # Copy the file over as /etc/pihole/local.list so dnsmasq can use it - rm "${localList}" - gravity_doHostFormat "${localList}.tmp" "${localList}" - rm "${localList}.tmp" + rm "${localList}" 2> /dev/null || \ + echo -e " ${CROSS} Unable to remove ${localList}" + gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}" + rm "${localList}.tmp" 2> /dev/null || \ + echo -e " ${CROSS} Unable to remove ${localList}.tmp" } -gravity_hostFormatGravity() { - # Format domain list as "192.168.x.x domain.com" - echo "" > "${piholeDir}/${accretionDisc}" - gravity_doHostFormat "${piholeDir}/${eventHorizon}" "${piholeDir}/${accretionDisc}" +# Create primary blacklist entries +gravity_ParseBlacklistDomains() { + # Create $accretionDisc + [[ ! -f "${piholeDir}/${accretionDisc}" ]] && echo "" > "${piholeDir}/${accretionDisc}" + + gravity_ParseDomainsIntoHosts "${piholeDir}/${eventHorizon}" "${piholeDir}/${accretionDisc}" + # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it - mv "${piholeDir}/${accretionDisc}" "${adList}" + output=$( { mv "${piholeDir}/${accretionDisc}" "${adList}"; } 2>&1 ) + status="$?" + if [[ "${status}" -ne 0 ]]; then + echo -e " ${CROSS} Unable to move ${accretionDisc} from ${piholeDir} + ${output}" + gravity_Cleanup "error" + fi } -gravity_hostFormatBlack() { +# Create user-added blacklist entries +gravity_ParseUserDomains() { if [[ -f "${blacklistFile}" ]]; then - numBlacklisted=$(wc -l < "${blacklistFile}") - # Format domain list as "192.168.x.x domain.com" - gravity_doHostFormat "${blacklistFile}" "${blackList}.tmp" + numBlacklisted=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")") + gravity_ParseDomainsIntoHosts "${blacklistFile}" "${blackList}.tmp" # Copy the file over as /etc/pihole/black.list so dnsmasq can use it - mv "${blackList}.tmp" "${blackList}" + mv "${blackList}.tmp" "${blackList}" 2> /dev/null || \ + echo -e " ${CROSS} Unable to move ${blackList##*/}.tmp to ${piholeDir}" else echo -e " ${INFO} Nothing to blacklist!" fi } -# blackbody - remove any remnant files from script processes -gravity_blackbody() { - # Loop through list files - for file in ${piholeDir}/*.${justDomainsExtension}; do - # If list is in active array then leave it (noop) else rm the list - if [[ " ${activeDomains[@]} " =~ ${file} ]]; then - : - else - rm -f "${file}" - fi - done -} - -gravity_advanced() { - # Remove comments and print only the domain name - # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious - # This helps with that and makes it easier to read - # It also helps with debugging so each stage of the script can be researched more in depth - local str="Formatting list of domains to remove comments" +# Parse consolidated blocklist into domains-only format +gravity_Advanced() { + local str="Extracting domains from blocklists" echo -ne " ${INFO} ${str}..." - #awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' ${piholeDir}/${matterAndLight} | sed -nr -e 's/\.{2,}/./g' -e '/\./p' > ${piholeDir}/${supernova} - #Above line does not correctly grab domains where comment is on the same line (e.g 'addomain.com #comment') - #Awk -F splits on given IFS, we grab the right hand side (chops trailing #coments and /'s to grab the domain only. - #Last awk command takes non-commented lines and if they have 2 fields, take the left field (the domain) and leave - #+ the right (IP address), otherwise grab the single field. - cat ${piholeDir}/${matterAndLight} | \ - awk -F '#' '{print $1}' | \ - awk -F '/' '{print $1}' | \ - awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' | \ - sed -nr -e 's/\.{2,}/./g' -e '/\./p' > ${piholeDir}/${supernova} + # Parse files as Hosts + gravity_ParseFileAsDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${supernova}" - echo -e "${OVER} ${TICK} ${str}" + numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${supernova}")") + echo -e "${OVER} ${TICK} ${str} + ${INFO} ${COL_LIGHT_BLUE}${numberOf}${COL_NC} domains being pulled in by gravity" - numberOf=$(wc -l < ${piholeDir}/${supernova}) - echo -e " ${INFO} ${COL_LIGHT_BLUE}${numberOf}${COL_NC} domains being pulled in by gravity" - - gravity_unique + gravity_Unique } -gravity_reload() { - # Reload hosts file - echo "" - local str="Refreshing lists in dnsmasq" - echo -e " ${INFO} ${str}..." +# Trap Ctrl-C +gravity_Trap() { + trap '{ echo -e "\\n\\n ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT +} - # Ensure /etc/dnsmasq.d/01-pihole.conf is pointing at the correct list! - # First escape forward slashes in the path: - adList=${adList//\//\\\/} - # Now replace the line in dnsmasq file - # sed -i "s/^addn-hosts.*/addn-hosts=$adList/" /etc/dnsmasq.d/01-pihole.conf +# Clean up after Gravity +gravity_Cleanup() { + local error="${1:-}" - "${PIHOLE_COMMAND}" restartdns + str="Cleaning up debris" + echo -ne " ${INFO} ${str}..." + + rm ${piholeDir}/pihole.*.txt 2> /dev/null + rm ${piholeDir}/*.tmp 2> /dev/null + + # Remove any unused .domains files + for file in ${piholeDir}/*.${domainsExtension}; do + # If list is in active array then leave it (noop) else rm the list + if [[ "${activeDomains[*]}" =~ ${file} ]]; then + : + else + rm -f "${file}" 2> /dev/null || \ + echo -e " ${CROSS} Failed to remove ${file##*/}" + fi + done + + echo -e "${OVER} ${TICK} ${str}" + + [[ -n "$error" ]] && echo "" + + # Only restart DNS service if offline + if ! pidof dnsmasq &> /dev/null; then + "${PIHOLE_COMMAND}" restartdns + fi + + if [[ -n "$error" ]]; then + "${PIHOLE_COMMAND}" status + exit 1 + fi } for var in "$@"; do case "${var}" in - "-f" | "--force" ) forceGrav=true;; - "-h" | "--help" ) helpFunc;; - "-sd" | "--skip-download" ) skipDownload=true;; - "-b" | "--blacklist-only" ) blackListOnly=true;; + "-f" | "--force" ) forceDelete=true;; + "-h" | "--help" ) helpFunc;; + "-sd" | "--skip-download" ) skipDownload=true;; + "-b" | "--blacklist-only" ) blackListOnly=true;; + "-w" | "--wildcard" ) dnsRestart="restart";; esac done -if [[ "${forceGrav}" == true ]]; then +# Main Gravity Execution +gravity_Trap + +# Use "force-reload" when restarting dnsmasq for Blacklists and Whitelists +[[ -z "${dnsRestart}" ]] && dnsRestart="force-reload" + +if [[ "${forceDelete}" == true ]]; then str="Deleting exising list cache" echo -ne "${INFO} ${str}..." @@ -435,42 +538,41 @@ if [[ "${forceGrav}" == true ]]; then fi fi +# If $blackListOnly is true, only run essential functions if [[ ! "${blackListOnly}" == true ]]; then - gravity_collapse - gravity_spinup + gravity_Collapse + gravity_Pull + if [[ "${skipDownload}" == false ]]; then - gravity_Schwarzchild - gravity_advanced + gravity_Schwarzschild + gravity_Advanced else echo -e " ${INFO} Using cached Event Horizon list..." - numberOf=$(wc -l < ${piholeDir}/${preEventHorizon}) - echo -e " ${INFO} ${COL_LIGHT_BLUE}$numberOf${COL_NC} unique domains trapped in the event horizon." + numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") + echo -e " ${INFO} ${COL_LIGHT_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon" fi + gravity_Whitelist fi + gravity_Blacklist gravity_Wildcard -str="Formatting domains into a HOSTS file" +str="Parsing domains into hosts format" echo -ne " ${INFO} ${str}..." + if [[ ! "${blackListOnly}" == true ]]; then - gravity_hostFormatLocal - gravity_hostFormatGravity + gravity_ParseLocalDomains + gravity_ParseBlacklistDomains fi -gravity_hostFormatBlack + +gravity_ParseUserDomains echo -e "${OVER} ${TICK} ${str}" -gravity_blackbody - if [[ ! "${blackListOnly}" == true ]]; then - # Clear no longer needed files... - str="Cleaning up un-needed files" - echo -ne " ${INFO} ${str}..." - - rm ${piholeDir}/pihole.*.txt 2> /dev/null - - echo -e "${OVER} ${TICK} ${str}" + gravity_Cleanup fi -gravity_reload +echo "" +"${PIHOLE_COMMAND}" restartdns "${dnsRestart}" "${PIHOLE_COMMAND}" status From a2825be81961cee4ddbb37f9e984fe6a27faa99a Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Mon, 24 Jul 2017 21:25:04 +1000 Subject: [PATCH 02/50] Add dnsmasq "force-reload" option * Made use of $PI_HOLE_SCRIPT_DIR * Fix #1610 & #1624 * Add "$2" to restartDNS options Signed-off-by: WaLLy3K --- pihole | 58 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/pihole b/pihole index 8da911d8..7ea2cc38 100755 --- a/pihole +++ b/pihole @@ -8,12 +8,12 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. -colfile="/opt/pihole/COL_TABLE" -source ${colfile} - readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" readonly wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf" +colfile="${PI_HOLE_SCRIPT_DIR}/COL_TABLE" +source "${colfile}" + # Must be root to use this tool if [[ ! $EUID -eq 0 ]];then if [[ -x "$(command -v sudo)" ]]; then @@ -26,7 +26,7 @@ if [[ ! $EUID -eq 0 ]];then fi webpageFunc() { - source /opt/pihole/webpage.sh + source "${PI_HOLE_SCRIPT_DIR}/webpage.sh" main "$@" exit 0 } @@ -340,33 +340,33 @@ versionFunc() { } restartDNS() { - dnsmasqPid=$(pidof dnsmasq) - local str="Restarting DNS service" - echo -ne " ${INFO} ${str}" - if [[ "${dnsmasqPid}" ]]; then - # Service already running - reload config - if [[ -x "$(command -v systemctl)" ]]; then - output=$( { systemctl restart dnsmasq; } 2>&1 ) + local svcOption str output status over + + # Force-reload is used for "/etc/pihole/*.list" files + # Restart is needed for "/etc/dnsmasq.d/*.conf" files + svcOption="${1:-}" + + if [[ -z "${svcOption}" ]]; then + # Get PID of dnsmasq to determine if it needs to start or restart + if pidof dnsmasq &> /dev/null; then + svcOption="restart" else - output=$( { service dnsmasq restart; } 2>&1 ) - fi - if [[ -z "${output}" ]]; then - echo -e "${OVER} ${TICK} ${str}" - else - echo -e "${OVER} ${CROSS} ${output}" + svcOption="start" fi + fi + + # Only print output to Terminal, not Web Admin + str="${svcOption^}ing DNS service" + [[ -t 1 ]] && echo -ne " ${INFO} ${str}..." + + output=$( { service dnsmasq "${svcOption}"; } 2>&1 ) + status="$?" + + if [[ "$?" -eq 0 ]]; then + [[ -t 1 ]] && echo -e "${OVER} ${TICK} ${str}" else - # Service not running, start it up - if [[ -x "$(command -v systemctl)" ]]; then - output=$( { systemctl start dnsmasq; } 2>&1 ) - else - output=$( { service dnsmasq start; } 2>&1 ) - fi - if [[ -z "${output}" ]]; then - echo -e "${OVER} ${TICK} ${str}" - else - echo -e "${OVER} ${CROSS} ${output}" - fi + [[ ! -t 1 ]] && OVER="" + echo -e "${OVER} ${CROSS} ${output}" fi } @@ -645,7 +645,7 @@ case "${1}" in "enable" ) piholeEnable 1;; "disable" ) piholeEnable 0 "$2";; "status" ) piholeStatus "$2";; - "restartdns" ) restartDNS;; + "restartdns" ) restartDNS "$2";; "-a" | "admin" ) webpageFunc "$@";; "-t" | "tail" ) tailFunc;; "checkout" ) piholeCheckoutFunc "$@";; From 406098e55a61c8cf72ba06e3de682cb938a401e1 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Mon, 24 Jul 2017 21:26:39 +1000 Subject: [PATCH 03/50] Speed up refresh time * Add "--blacklist-only" to only run essential gravity functions * Pass "--wildcard" option to `gravity.sh` to ensure dnsmasq is restarted Signed-off-by: WaLLy3K --- advanced/Scripts/list.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 86589083..8b6a6e5b 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -138,7 +138,7 @@ AddDomain() { if [[ "${verbose}" == true ]]; then echo -e " ${INFO} Adding $1 to wildcard blacklist..." fi - reload=true + reload="restart" echo "address=/$1/${IPV4_ADDRESS}" >> "${wildcardlist}" if [[ "${#IPV6_ADDRESS}" > 0 ]]; then echo "address=/$1/${IPV6_ADDRESS}" >> "${wildcardlist}" @@ -183,7 +183,7 @@ RemoveDomain() { echo -e " ${INFO} Removing $1 from $listname..." # /I flag: search case-insensitive sed -i "/address=\/${domain}/Id" "${list}" - reload=true + reload="restart" else if [[ "${verbose}" == true ]]; then echo -e " ${INFO} ${1} does not exist in ${listname}, no need to remove!" @@ -192,12 +192,16 @@ RemoveDomain() { fi } +# Update Gravity Reload() { - # Reload hosts file echo "" - echo -e " ${INFO} Updating gravity..." - echo "" - pihole -g -sd + + # Ensure that "restart" is used for Wildcard updates + if [[ "${1}" == "restart" ]]; then + local type="--wildcard" + fi + + pihole -g --skip-download --blacklist-only "${type:-}" } Displaylist() { @@ -243,6 +247,7 @@ fi PoplistFile -if ${reload}; then - Reload +if [[ "${reload}" != false ]]; then + # Ensure that "restart" is used for Wildcard updates + Reload "${reload}" fi From 61ff0452e135445b6782e24a8c5783c3aad97eff Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Mon, 24 Jul 2017 21:27:12 +1000 Subject: [PATCH 04/50] Remove duplicate code * Make RestartDNS() use `pihole restartdns` Signed-off-by: WaLLy3K --- advanced/Scripts/webpage.sh | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 42272122..1f90f9c3 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -221,20 +221,7 @@ Reboot() { } RestartDNS() { - local str="Restarting DNS service" - [[ -t 1 ]] && echo -ne " ${INFO} ${str}" - if command -v systemctl &> /dev/null; then - output=$( { systemctl restart dnsmasq; } 2>&1 ) - else - output=$( { service dnsmasq restart; } 2>&1 ) - fi - - if [[ -z "${output}" ]]; then - [[ -t 1 ]] && echo -e "${OVER} ${TICK} ${str}" - else - [[ ! -t 1 ]] && OVER="" - echo -e "${OVER} ${CROSS} ${output}" - fi + /usr/local/bin/pihole restartdns } SetQueryLogOptions() { From 2d8fff099fbaf2e99faab031ef9e928bcea14906 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Thu, 27 Jul 2017 12:34:26 +1000 Subject: [PATCH 05/50] Pass correct options to gravity.sh * Optimise $validDomain function by using bashism and `grep` * Add black/white/wildcard variables to pass to Reload() * Revert reload variable behaviour * Ensure Reload() function passes correct options to gravity.sh Signed-off-by: WaLLy3K --- advanced/Scripts/list.sh | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 8b6a6e5b..a3f3261a 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -66,15 +66,15 @@ HandleOther() { domain="${1,,}" # Check validity of domain - validDomain=$(perl -lne 'print if /^((-|_)*[a-z\d]((-|_)*[a-z\d])*(-|_)*)(\.(-|_)*([a-z\d]((-|_)*[a-z\d])*))*$/' <<< "${domain}") # Valid chars check - validDomain=$(perl -lne 'print if /^.{1,253}$/' <<< "${validDomain}") # Overall length check - validDomain=$(perl -lne 'print if /^[^\.]{1,63}(\.[^\.]{1,63})*$/' <<< "${validDomain}") # Length of each label + if [[ "${#domain}" -le 253 ]]; then + validDomain=$(grep -P "^((-|_)*[a-z\d]((-|_)*[a-z\d])*(-|_)*)(\.(-|_)*([a-z\d]((-|_)*[a-z\d])*))*$" <<< "${domain}") # Valid chars check + validDomain=$(grep -P "^[^\.]{1,63}(\.[^\.]{1,63})*$" <<< "${validDomain}") # Length of each label + fi - if [[ -z "${validDomain}" ]]; then - echo -e " ${CROSS} $1 is not a valid argument or domain name!" - else - echo -e " ${TICK} $1 is a valid domain name!" + if [[ -n "${validDomain}" ]]; then domList=("${domList[@]}" ${validDomain}) + else + echo -e " ${CROSS} ${domain} is not a valid argument or domain name!" fi } @@ -107,6 +107,8 @@ AddDomain() { [[ "${list}" == "${wildcardlist}" ]] && listname="wildcard blacklist" if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then + [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only" + [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only" bool=true # Is the domain in the list we want to add it to? grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false @@ -129,7 +131,7 @@ AddDomain() { # Remove the /* from the end of the IP addresses IPV4_ADDRESS=${IPV4_ADDRESS%/*} IPV6_ADDRESS=${IPV6_ADDRESS%/*} - + [[ -z "${type}" ]] && type="--wildcard-only" bool=true # Is the domain in the list? grep -e "address=\/${domain}\/" "${wildcardlist}" > /dev/null 2>&1 || bool=false @@ -161,6 +163,8 @@ RemoveDomain() { if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then bool=true + [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only" + [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only" # Is it in the list? Logic follows that if its whitelisted it should not be blacklisted and vice versa grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false if [[ "${bool}" == true ]]; then @@ -175,6 +179,7 @@ RemoveDomain() { fi fi elif [[ "${list}" == "${wildcardlist}" ]]; then + [[ -z "${type}" ]] && type="--wildcard-only" bool=true # Is it in the list? grep -e "address=\/${domain}\/" "${wildcardlist}" > /dev/null 2>&1 || bool=false @@ -183,7 +188,7 @@ RemoveDomain() { echo -e " ${INFO} Removing $1 from $listname..." # /I flag: search case-insensitive sed -i "/address=\/${domain}/Id" "${list}" - reload="restart" + reload=true else if [[ "${verbose}" == true ]]; then echo -e " ${INFO} ${1} does not exist in ${listname}, no need to remove!" @@ -195,13 +200,7 @@ RemoveDomain() { # Update Gravity Reload() { echo "" - - # Ensure that "restart" is used for Wildcard updates - if [[ "${1}" == "restart" ]]; then - local type="--wildcard" - fi - - pihole -g --skip-download --blacklist-only "${type:-}" + pihole -g --skip-download "${type:-}" } Displaylist() { From f24ab8508eb1fa2b87e25f0f1db29ed869562bce Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Thu, 27 Jul 2017 12:34:35 +1000 Subject: [PATCH 06/50] WIP cleanup * Changed supernova/eventHorizon variables to match their purpose * Add gravity_DNSLookup() function * Ensure all comments are clear and relevant * Use && instead of || in gravity_Collapse() * Renamed existing functions, and placed them in order of script execution * Use \t instead of literal tab in gravity_ParseFileIntoDomains() * Replace instances of "truncate" with : > (e.g: gravity_Schwarzschild()) * Ensure correct variables are local'd * Use phrase "Cleaning up stray matter" when gravity_Cleanup() is called * Add black/white/wildcard switches for list.sh * Ensure necessary functions are called when modifying black/white/wildcards Signed-off-by: WaLLy3K --- gravity.sh | 582 ++++++++++++++++++++++++++++------------------------- 1 file changed, 310 insertions(+), 272 deletions(-) diff --git a/gravity.sh b/gravity.sh index 852356d7..b222886b 100755 --- a/gravity.sh +++ b/gravity.sh @@ -12,7 +12,7 @@ # Please see LICENSE file for your rights under this license. coltable="/opt/pihole/COL_TABLE" -source ${coltable} +source "${coltable}" basename="pihole" PIHOLE_COMMAND="/usr/local/bin/${basename}" @@ -35,13 +35,16 @@ localList="${piholeDir}/local.list" domainsExtension="domains" matterAndLight="${basename}.0.matterandlight.txt" -supernova="${basename}.1.supernova.txt" -preEventHorizon="list.preEventHorizon" -eventHorizon="${basename}.2.supernova.txt" +parsedMatter="${basename}.1.parsedmatter.txt" +whitelistMatter="${basename}.2.whitelistmatter.txt" accretionDisc="${basename}.3.accretionDisc.txt" +preEventHorizon="list.preEventHorizon" skipDownload="false" +# Use "force-reload" when restarting dnsmasq for everything but Wildcards +dnsRestart="force-reload" + # Source setupVars from install script setupVars="${piholeDir}/setupVars.conf" if [[ -f "${setupVars}" ]];then @@ -61,17 +64,39 @@ if [[ -r "${piholeDir}/pihole.conf" ]]; then echo -e " ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}" fi -helpFunc() { - echo "Usage: pihole -g -Update domains from blocklists specified in adlists.list +# Attempt to resolve DNS before proceeding with blocklist generation +gravity_DNSLookup() { + local plu -Options: - -f, --force Force the download of all specified blocklists - -h, --help Show this help dialog" - exit 0 + # Determine if github.com can be resolved + if ! timeout 2 nslookup github.com &> /dev/null; then + if [[ -n "${secs}" ]]; then + echo -e "${OVER} ${CROSS} DNS resolution is still unavailable, cancelling" + exit 1 + fi + + # Determine error output message + if pidof dnsmasq &> /dev/null; then + echo -e " ${CROSS} DNS resolution is temporarily unavailable" + else + echo -e " ${CROSS} DNS service is not running" + "${PIHOLE_COMMAND}" restartdns + fi + + secs="15" + while [[ "${secs}" -ne 0 ]]; do + [[ "${secs}" -ne 1 ]] && plu="s" || plu="" + echo -ne "${OVER} ${INFO} Waiting $secs second${plu} before continuing..." + sleep 1 + : $((secs--)) + done + + # Try again + gravity_DNSLookup + fi } -# Retrieve blocklist URLs from adlists.list +# Retrieve blocklist URLs and parse domains from adlists.list gravity_Collapse() { echo -e " ${INFO} Neutrino emissions detected..." @@ -94,7 +119,7 @@ gravity_Collapse() { mapfile -t sources < <(awk '!/^[#@;!\[]/ {gsub(/\r$/, "", $0); if ($1) { print $1 } }' "${adListFile}" 2> /dev/null) # Parse source domains from $sources - # Logic: Split by folder/port and remove URL protocol/password + # Logic: Split by folder/port, remove URL protocol & optional username:password@ mapfile -t sourceDomains < <( awk -F '[/:]' '{ gsub(/(.*:\/\/|.*:.*@)/, "", $0) @@ -102,7 +127,7 @@ gravity_Collapse() { }' <<< "$(printf '%s\n' "${sources[@]}")" 2> /dev/null ) - if [[ -n "${sources[*]}" ]] || [[ -n "${sourceDomains[*]}" ]]; then + if [[ -n "${sources[*]}" ]] && [[ -n "${sourceDomains[*]}" ]]; then echo -e "${OVER} ${TICK} ${str}" else echo -e "${OVER} ${CROSS} ${str}" @@ -110,157 +135,9 @@ gravity_Collapse() { fi } -# Parse source file into domains-only format -gravity_ParseFileAsDomains() { - local source destination hostsFilter firstLine abpFilter - source="${1}" - destination="${2}" - - # Determine how to parse source file - if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then - # Symbols used as comments: "#;@![/" - commentPattern="[#;@![\\/]" - - # Parse consolidated file by removing comments and hosts IP's - # Logic: Process lines which do not begin with comments - awk '!/^'"${commentPattern}"'/ { - # If there are multiple words seperated by space - if (NF>1) { - # Remove comments (Inc. prefixed spaces/tabs) - if ($0 ~ /'"${commentPattern}"'/) { gsub("( | )'"${commentPattern}"'.*", "", $0) } - # Print consecutive domains - if ($3) { - $1="" - gsub("^ ", "", $0) - print $0 - # Print single domain - } else if ($2) { - print $2 - } - # Print single domain - } else if($1) { - print $1 - } - }' "${source}" 2> /dev/null > "${destination}" - else - # Individual file parsing - # Logic: comments are kept, and domains are extracted from each line - read -r firstLine < "${source}" - - # Determine how to parse individual source file - if [[ "${firstLine,,}" =~ "adblock" ]] || [[ "${firstLine,,}" =~ "ublock" ]] || [[ "${firstLine,,}" =~ "! checksum" ]]; then - # Parse Adblock domains & comments: https://adblockplus.org/filter-cheatsheet - abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/" - awk ''"${abpFilter}"' { - # Remove valid adblock type options - gsub(/~?(important|third-party|popup|subdocument|websocket),?/, "", $0) - # Remove starting domain name anchor "||" and ending seperator "^$" ($ optional) - gsub(/(\|\||\^\$?$)/, "", $0) - # Remove lines which are only IPv4 addresses or contain "^/*" - if ($0 ~ /(^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$|[\\^\/\*])/) { $0="" } - # Print if not empty - if ($0) { print $0 } - }' "${source}" 2> /dev/null > "${destination}" - echo -e " ${TICK} Format: Adblock" - elif grep -q -E "^(https?://|([0-9]{1,3}\.){3}[0-9]{1,3}$)" "${source}" &> /dev/null; then - # Parse URLs - awk '{ - # Remove URL protocol, optional "username:password@", and ":?/;" - if ($0 ~ /[:?\/;]/) { gsub(/(^.*:\/\/(.*:.*@)?|[:?\/;].*)/, "", $0) } - # Remove lines which are only IPv4 addresses - if ($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" } - if ($0) { print $0 } - }' "${source}" 2> /dev/null > "${destination}" - echo -e " ${TICK} Format: URL" - else - # Keep hosts/domains file in same format as it was downloaded - output=$( { mv "${source}" "${destination}"; } 2>&1 ) - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e " ${CROSS} Unable to move tmp file to ${piholeDir} - ${output}" - gravity_Cleanup "error" - fi - fi - fi -} - -# Determine output based on gravity_Transport() status -gravity_AdvancedTransport() { - local patternBuffer success error output status - patternBuffer="${1}" - success="${2}" - error="${3}" - - if [[ "${success}" = true ]]; then - if [[ "${error}" == "304" ]]; then - : # Print no output - # Check if the patternbuffer is a non-zero length file - elif [[ -s "${patternBuffer}" ]]; then - # Parse Adblock/URL format blocklists and include comments - # HOSTS format is moved as-is - gravity_ParseFileAsDomains "${patternBuffer}" "${saveLocation}" "1" - else - # Fall back to previously cached list if current $patternBuffer is empty - echo -e " ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" - fi - else - # Determine if cached list exists - if [[ -r "${saveLocation}" ]]; then - echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" - else - echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" - fi - fi -} - -# Curl the specified URL with any necessary command extentions -gravity_Transport() { - local url cmd_ext agent - url="${1}" - cmd_ext="${2}" - agent="${3}" - - # Store downloaded content to temp file instead of RAM - patternBuffer=$(mktemp) - heisenbergCompensator="" - if [[ -r "${saveLocation}" ]]; then - # If domain has been saved, add file for date check to only download newer - heisenbergCompensator="-z ${saveLocation}" - fi - - local str="Status:" - echo -ne " ${INFO} ${str} Pending..." - # shellcheck disable=SC2086 - httpCode=$(curl -s -L ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${patternBuffer}" 2> /dev/null) - - # Determine "Status:" output based on HTTP response - case "$httpCode" in - "200" ) echo -e "${OVER} ${TICK} ${str} Transport successful"; success=true;; - "304" ) echo -e "${OVER} ${TICK} ${str} No changes detected"; success=true;; - "403" ) echo -e "${OVER} ${CROSS} ${str} Forbidden"; success=false;; - "404" ) echo -e "${OVER} ${CROSS} ${str} Not found"; success=false;; - "408" ) echo -e "${OVER} ${CROSS} ${str} Time-out"; success=false;; - "451" ) echo -e "${OVER} ${CROSS} ${str} Unavailable For Legal Reasons"; success=false;; - "521" ) echo -e "${OVER} ${CROSS} ${str} Web Server Is Down (Cloudflare)"; success=false;; - "522" ) echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Cloudflare)"; success=false;; - "500" ) echo -e "${OVER} ${CROSS} ${str} Internal Server Error"; success=false;; - * ) echo -e "${OVER} ${CROSS} ${str} Status $httpCode"; success=false;; - esac - - # Output additional info if success=false - gravity_AdvancedTransport "${patternBuffer}" "${success}" "${httpCode}" - - # Delete temp file if it has not been moved - if [[ -f "${patternBuffer}" ]]; then - rm "${patternBuffer}" - fi -} - -# Define User Agent and options for each blocklist -gravity_Pull() { - local agent url domain cmd_ext str +# Define options for when retrieving blocklists +gravity_Supernova() { + local url domain agent cmd_ext str echo "" @@ -282,28 +159,167 @@ gravity_Pull() { esac if [[ "${skipDownload}" == false ]]; then - str="Target: $domain (${url##*/})" + str="Target: ${domain} (${url##*/})" echo -e " ${INFO} ${str}" - gravity_Transport "$url" "$cmd_ext" "$agent" "$str" + gravity_Pull "${url}" "${cmd_ext}" "${agent}" "${str}" echo "" fi done } -# Consolidate domains to one list and add blacklisted domains +# Download specified URL and perform QA +gravity_Pull() { + local url cmd_ext agent heisenbergCompensator patternBuffer str httpCode success + + url="${1}" + cmd_ext="${2}" + agent="${3}" + + # Store downloaded content to temp file instead of RAM + patternBuffer=$(mktemp) + heisenbergCompensator="" + if [[ -r "${saveLocation}" ]]; then + # If domain has been saved, add file for date check to only download newer + heisenbergCompensator="-z ${saveLocation}" + fi + + str="Status:" + echo -ne " ${INFO} ${str} Pending..." + # shellcheck disable=SC2086 + httpCode=$(curl -s -L ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${patternBuffer}" 2> /dev/null) + + # Determine "Status:" output based on HTTP response + case "$httpCode" in + "200" ) echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success="true";; + "304" ) echo -e "${OVER} ${TICK} ${str} No changes detected"; success="true";; + "403" ) echo -e "${OVER} ${CROSS} ${str} Forbidden"; success="false";; + "404" ) echo -e "${OVER} ${CROSS} ${str} Not found"; success="false";; + "408" ) echo -e "${OVER} ${CROSS} ${str} Time-out"; success="false";; + "451" ) echo -e "${OVER} ${CROSS} ${str} Unavailable For Legal Reasons"; success="false";; + "521" ) echo -e "${OVER} ${CROSS} ${str} Web Server Is Down (Cloudflare)"; success="false";; + "522" ) echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Cloudflare)"; success="false";; + "500" ) echo -e "${OVER} ${CROSS} ${str} Internal Server Error"; success="false";; + * ) echo -e "${OVER} ${CROSS} ${str} Status $httpCode"; success="false";; + esac + + # Determine if the blocklist was downloaded and saved correctly + if [[ "${success}" == "true" ]]; then + if [[ "${httpCode}" == "304" ]]; then + : # Do nothing + # Check if patternbuffer is a non-zero length file + elif [[ -s "${patternBuffer}" ]]; then + # Determine if blocklist is non-standard and parse as appropriate + gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}" + else + # Fall back to previously cached list if patternBuffer is empty + echo -e " ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" + fi + else + # Determine if cached list exists + if [[ -r "${saveLocation}" ]]; then + echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" + else + echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" + fi + fi + + # Delete temp file if it has not been moved + if [[ -f "${patternBuffer}" ]]; then + rm "${patternBuffer}" 2> /dev/null || \ + echo -e " ${CROSS} Unable to remove ${patternBuffer}" + fi +} + +# Parse non-standard source files into domains-only format +gravity_ParseFileIntoDomains() { + local source destination commentPattern firstLine abpFilter + source="${1}" + destination="${2}" + + # Determine how to parse source file + if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then + # Consolidated list parsing: Remove comments and hosts IP's + + # Symbols used as comments: #;@![/ + commentPattern="[#;@![\\/]" + + # Logic: Process lines which do not begin with comments + awk '!/^'"${commentPattern}"'/ { + # If there are multiple words seperated by space + if (NF>1) { + # Remove comments (Inc. prefixed spaces/tabs) + if ($0 ~ /'"${commentPattern}"'/) { gsub("( |\t)'"${commentPattern}"'.*", "", $0) } + # Print consecutive domains + if ($3) { + $1="" + # Remove space which is left in $0 when removing $1 + gsub("^ ", "", $0) + print $0 + # Print single domain + } else if ($2) { + print $2 + } + # If there are no words seperated by space + } else if($1) { + print $1 + } + }' "${source}" 2> /dev/null > "${destination}" + else + # Individual file parsing: Keep comments, while parsing domains from each line + read -r firstLine < "${source}" + + # Determine how to parse individual source file formats + if [[ "${firstLine,,}" =~ "adblock" ]] || [[ "${firstLine,,}" =~ "ublock" ]] || [[ "${firstLine,,}" =~ "! checksum" ]]; then + # Logic: Parse Adblock domains & comments: https://adblockplus.org/filter-cheatsheet + abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/" + awk ''"${abpFilter}"' { + # Remove valid adblock type options + gsub(/~?(important|third-party|popup|subdocument|websocket),?/, "", $0) + # Remove starting domain name anchor "||" and ending seperator "^$" ($ optional) + gsub(/(\|\||\^\$?$)/, "", $0) + # Remove lines which are only IPv4 addresses or contain "^/*" + if ($0 ~ /(^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$|[\\^\/\*])/) { $0="" } + if ($0) { print $0 } + }' "${source}" 2> /dev/null > "${destination}" + elif grep -q -E "^(https?://|([0-9]{1,3}\\.){3}[0-9]{1,3}$)" "${source}" &> /dev/null; then + # Parse URLs if source file contains http:// or IPv4 + awk '{ + # Remove URL protocol, optional "username:password@", and ":?/;" + if ($0 ~ /[:?\/;]/) { gsub(/(^.*:\/\/(.*:.*@)?|[:?\/;].*)/, "", $0) } + # Remove lines which are only IPv4 addresses + if ($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" } + if ($0) { print $0 } + }' "${source}" 2> /dev/null > "${destination}" + else + # Keep hosts/domains file in same format as it was downloaded + output=$( { mv "${source}" "${destination}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e " ${CROSS} Unable to move tmp file to ${piholeDir} + ${output}" + gravity_Cleanup "error" + fi + fi + fi +} + +# Create unfiltered "Matter and Light" consolidated list gravity_Schwarzschild() { local str lastLine str="Consolidating blocklists" echo -ne " ${INFO} ${str}..." - # Compile all blacklisted domains into one file and remove CRs - truncate -s 0 "${piholeDir}/${matterAndLight}" + # Empty $matterAndLight if it already exists, otherwise, create it + : > "${piholeDir}/${matterAndLight}" + for i in "${activeDomains[@]}"; do # Only assimilate list if it is available (download might have failed permanently) if [[ -r "${i}" ]]; then + # Compile all blacklisted domains into one file and remove CRs tr -d '\r' < "${i}" >> "${piholeDir}/${matterAndLight}" # Ensure each source blocklist has a final newline @@ -315,39 +331,39 @@ gravity_Schwarzschild() { echo -e "${OVER} ${TICK} ${str}" } -# Append blacklist entries to eventHorizon if they exist -gravity_Blacklist() { - local numBlacklisted plural str +# Parse unfiltered consolidated blocklist into filtered domains-only format +gravity_Filter() { + local str num - if [[ -f "${blacklistFile}" ]]; then - numBlacklisted=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")") - plural=; [[ "${numBlacklisted}" != "1" ]] && plural=s - str="Exact blocked domain${plural}: $numBlacklisted" - echo -e " ${INFO} ${str}" - else - echo -e " ${INFO} Nothing to blacklist!" - fi + str="Extracting domains from blocklists" + echo -ne " ${INFO} ${str}..." + + # Parse files as hosts + gravity_ParseFileIntoDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${parsedMatter}" + + num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${parsedMatter}")") + echo -e "${OVER} ${TICK} ${str} + ${INFO} ${COL_LIGHT_BLUE}${num}${COL_NC} domains being pulled in by gravity" + + gravity_Unique } -# Return number of wildcards in output -gravity_Wildcard() { - local numWildcards plural +# Sort and remove duplicate blacklisted domains +gravity_Unique() { + local str num - if [[ -f "${wildcardFile}" ]]; then - numWildcards=$(grep -c ^ "${wildcardFile}") - if [[ -n "${IPV4_ADDRESS}" ]] && [[ -n "${IPV6_ADDRESS}" ]];then - let numWildcards/=2 - fi - plural=; [[ "${numWildcards}" != "1" ]] && plural=s - echo -e " ${INFO} Wildcard blocked domain${plural}: $numWildcards" - else - echo -e " ${INFO} No wildcards used!" - fi + str="Removing duplicate domains" + echo -ne " ${INFO} ${str}..." + sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}" + echo -e "${OVER} ${TICK} ${str}" + + num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") + echo -e " ${INFO} ${COL_LIGHT_BLUE}${num}${COL_NC} unique domains trapped in the Event Horizon" } -# Prevent the domains of adlist sources from being blacklisted by other blocklists -gravity_Whitelist() { - local plural str +# Whitelist blocklist domain sources +gravity_WhitelistBLD() { + local plural str uniqDomains echo "" plural=; [[ "${#sources[*]}" != "1" ]] && plural=s @@ -361,17 +377,22 @@ gravity_Whitelist() { ${WHITELIST_COMMAND} -nr -q "${uniqDomains[*]}" > /dev/null echo -e "${OVER} ${TICK} ${str}" +} + +# Whitelist user-defined domains +gravity_Whitelist() { + local plural str num # Test existence of whitelist.txt if [[ -f "${whitelistFile}" ]]; then # Remove anything in whitelist.txt from the Event Horizon - numWhitelisted=$(wc -l < "${whitelistFile}") - plural=; [[ "${numWhitelisted}" != "1" ]] && plural=s - local str="Whitelisting $numWhitelisted domain${plural}" + num=$(wc -l < "${whitelistFile}") + plural=; [[ "${num}" != "1" ]] && plural=s + local str="Whitelisting ${num} domain${plural}" echo -ne " ${INFO} ${str}..." - # Print everything from preEventHorizon into eventHorizon EXCEPT domains in whitelist.txt - grep -F -x -v -f "${whitelistFile}" "${piholeDir}/${preEventHorizon}" > "${piholeDir}/${eventHorizon}" + # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in whitelist.txt + grep -F -x -v -f "${whitelistFile}" "${piholeDir}/${preEventHorizon}" > "${piholeDir}/${whitelistMatter}" echo -e "${OVER} ${TICK} ${str}" else @@ -379,24 +400,40 @@ gravity_Whitelist() { fi } -# Sort and remove duplicate blacklisted domains -gravity_Unique() { - local str numberOf +# Output count of blacklisted domains and wildcards +gravity_ShowBlockCount() { + local num plural str - str="Removing duplicate domains" - echo -ne " ${INFO} ${str}..." - sort -u "${piholeDir}/${supernova}" > "${piholeDir}/${preEventHorizon}" - echo -e "${OVER} ${TICK} ${str}" + if [[ -f "${blacklistFile}" ]]; then + num=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")") + plural=; [[ "${num}" != "1" ]] && plural=s + str="Exact blocked domain${plural}: ${num}" + echo -e " ${INFO} ${str}" + else + echo -e " ${INFO} Nothing to blacklist!" + fi - numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") - echo -e " ${INFO} ${COL_LIGHT_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon" + if [[ -f "${wildcardFile}" ]]; then + num=$(grep -c "^" "${wildcardFile}") + # If IPv4 and IPv6 is used, divide total wildcard count by 2 + if [[ -n "${IPV4_ADDRESS}" ]] && [[ -n "${IPV6_ADDRESS}" ]];then + num=$(( num/2 )) + fi + plural=; [[ "${num}" != "1" ]] && plural=s + echo -e " ${INFO} Wildcard blocked domain${plural}: ${num}" + else + echo -e " ${INFO} No wildcards used!" + fi } # Parse list of domains into hosts format gravity_ParseDomainsIntoHosts() { if [[ -n "${IPV4_ADDRESS}" ]] || [[ -n "${IPV6_ADDRESS}" ]]; then - awk -v ipv4addr="$IPV4_ADDRESS" -v ipv6addr="$IPV6_ADDRESS" \ - '{sub(/\r$/,""); if(ipv4addr) { print ipv4addr" "$0; }; if(ipv6addr) { print ipv6addr" "$0; }}' >> "${2}" < "${1}" + awk -v ipv4addr="$IPV4_ADDRESS" -v ipv6addr="$IPV6_ADDRESS" '{ + sub(/\r$/, "") + if(ipv4addr) { print ipv4addr" "$0; } + if(ipv6addr) { print ipv6addr" "$0; } + }' >> "${2}" < "${1}" else echo -e "${OVER} ${CROSS} ${str}" echo -e " ${COL_LIGHT_RED}No IP addresses found! Please run 'pihole -r' to reconfigure${COL_NC}\\n" @@ -406,6 +443,8 @@ gravity_ParseDomainsIntoHosts() { # Create "localhost" entries gravity_ParseLocalDomains() { + local hostname + if [[ -f "/etc/hostname" ]]; then hostname=$(< "/etc/hostname") elif command -v hostname &> /dev/null; then @@ -415,23 +454,19 @@ gravity_ParseLocalDomains() { fi echo -e "${hostname}\\npi.hole" > "${localList}.tmp" - - # Copy the file over as /etc/pihole/local.list so dnsmasq can use it - rm "${localList}" 2> /dev/null || \ - echo -e " ${CROSS} Unable to remove ${localList}" gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}" - rm "${localList}.tmp" 2> /dev/null || \ - echo -e " ${CROSS} Unable to remove ${localList}.tmp" } # Create primary blacklist entries gravity_ParseBlacklistDomains() { - # Create $accretionDisc - [[ ! -f "${piholeDir}/${accretionDisc}" ]] && echo "" > "${piholeDir}/${accretionDisc}" + local output status - gravity_ParseDomainsIntoHosts "${piholeDir}/${eventHorizon}" "${piholeDir}/${accretionDisc}" + # Empty $accretionDisc if it already exists, otherwise, create it + : > "${piholeDir}/${accretionDisc}" - # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it + gravity_ParseDomainsIntoHosts "${piholeDir}/${whitelistMatter}" "${piholeDir}/${accretionDisc}" + + # Move the file over as /etc/pihole/gravity.list so dnsmasq can use it output=$( { mv "${piholeDir}/${accretionDisc}" "${adList}"; } 2>&1 ) status="$?" @@ -445,7 +480,6 @@ gravity_ParseBlacklistDomains() { # Create user-added blacklist entries gravity_ParseUserDomains() { if [[ -f "${blacklistFile}" ]]; then - numBlacklisted=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")") gravity_ParseDomainsIntoHosts "${blacklistFile}" "${blackList}.tmp" # Copy the file over as /etc/pihole/black.list so dnsmasq can use it mv "${blackList}.tmp" "${blackList}" 2> /dev/null || \ @@ -455,21 +489,6 @@ gravity_ParseUserDomains() { fi } -# Parse consolidated blocklist into domains-only format -gravity_Advanced() { - local str="Extracting domains from blocklists" - echo -ne " ${INFO} ${str}..." - - # Parse files as Hosts - gravity_ParseFileAsDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${supernova}" - - numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${supernova}")") - echo -e "${OVER} ${TICK} ${str} - ${INFO} ${COL_LIGHT_BLUE}${numberOf}${COL_NC} domains being pulled in by gravity" - - gravity_Unique -} - # Trap Ctrl-C gravity_Trap() { trap '{ echo -e "\\n\\n ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT @@ -479,7 +498,7 @@ gravity_Trap() { gravity_Cleanup() { local error="${1:-}" - str="Cleaning up debris" + str="Cleaning up stray matter" echo -ne " ${INFO} ${str}..." rm ${piholeDir}/pihole.*.txt 2> /dev/null @@ -487,10 +506,8 @@ gravity_Cleanup() { # Remove any unused .domains files for file in ${piholeDir}/*.${domainsExtension}; do - # If list is in active array then leave it (noop) else rm the list - if [[ "${activeDomains[*]}" =~ ${file} ]]; then - : - else + # If list is not in active array, then remove it + if [[ ! "${activeDomains[*]}" =~ ${file} ]]; then rm -f "${file}" 2> /dev/null || \ echo -e " ${CROSS} Failed to remove ${file##*/}" fi @@ -505,27 +522,38 @@ gravity_Cleanup() { "${PIHOLE_COMMAND}" restartdns fi + # Print Pi-hole status if an error occured if [[ -n "$error" ]]; then "${PIHOLE_COMMAND}" status exit 1 fi } +helpFunc() { + echo "Usage: pihole -g +Update domains from blocklists specified in adlists.list + +Options: + -f, --force Force the download of all specified blocklists + -h, --help Show this help dialog" + exit 0 +} + for var in "$@"; do case "${var}" in "-f" | "--force" ) forceDelete=true;; "-h" | "--help" ) helpFunc;; "-sd" | "--skip-download" ) skipDownload=true;; - "-b" | "--blacklist-only" ) blackListOnly=true;; - "-w" | "--wildcard" ) dnsRestart="restart";; + "-b" | "--blacklist-only" ) listType="blacklist";; + "-w" | "--whitelist-only" ) listType="whitelist";; + "-wild" | "--wildcard-only" ) listType="wildcard";; esac done -# Main Gravity Execution gravity_Trap -# Use "force-reload" when restarting dnsmasq for Blacklists and Whitelists -[[ -z "${dnsRestart}" ]] && dnsRestart="force-reload" +# Ensure dnsmasq is restarted when modifying wildcards +[[ "${listType}" == "wildcard" ]] && dnsRestart="restart" if [[ "${forceDelete}" == true ]]; then str="Deleting exising list cache" @@ -535,41 +563,51 @@ if [[ "${forceDelete}" == true ]]; then echo -e "${OVER} ${TICK} ${str}" else echo -e "${OVER} ${CROSS} ${str}" + exit 1 fi fi -# If $blackListOnly is true, only run essential functions -if [[ ! "${blackListOnly}" == true ]]; then +# Determine which functions to run +if [[ "${skipDownload}" == false ]]; then + # Gravity needs to download blocklists + gravity_DNSLookup gravity_Collapse - gravity_Pull - - if [[ "${skipDownload}" == false ]]; then - gravity_Schwarzschild - gravity_Advanced - else - echo -e " ${INFO} Using cached Event Horizon list..." - numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") - echo -e " ${INFO} ${COL_LIGHT_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon" - fi + gravity_Supernova + gravity_Schwarzschild + gravity_Filter + gravity_WhitelistBLD +else + # Gravity needs to modify Blacklist/Whitelist/Wildcards + echo -e " ${INFO} Using cached Event Horizon list..." + numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") + echo -e " ${INFO} ${COL_LIGHT_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon" +fi +# Perform when downloading blocklists, or modifying the whitelist +if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "whitelist" ]]; then gravity_Whitelist fi -gravity_Blacklist -gravity_Wildcard +gravity_ShowBlockCount -str="Parsing domains into hosts format" -echo -ne " ${INFO} ${str}..." +# Perform when downloading blocklists, or modifying the blacklist +if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "blacklist" ]]; then + str="Parsing domains into hosts format" + echo -ne " ${INFO} ${str}..." -if [[ ! "${blackListOnly}" == true ]]; then - gravity_ParseLocalDomains - gravity_ParseBlacklistDomains + gravity_ParseUserDomains + + # Perform when downloading blocklists + if [[ ! "${listType}" == "blacklist" ]]; then + gravity_ParseLocalDomains + gravity_ParseBlacklistDomains + fi + + echo -e "${OVER} ${TICK} ${str}" fi -gravity_ParseUserDomains -echo -e "${OVER} ${TICK} ${str}" - -if [[ ! "${blackListOnly}" == true ]]; then +# Perform when downloading blocklists +if [[ "${skipDownload}" == false ]]; then gravity_Cleanup fi From cc4ada99d83fb002634fc320763b2ab565e47843 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Tue, 1 Aug 2017 20:48:43 +1000 Subject: [PATCH 07/50] Use SIGHUP instead of force-reload --- pihole | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pihole b/pihole index 1579bdbf..6d875d85 100755 --- a/pihole +++ b/pihole @@ -347,32 +347,33 @@ versionFunc() { } restartDNS() { - local svcOption str output status over - - # Force-reload is used for "/etc/pihole/*.list" files - # Restart is needed for "/etc/dnsmasq.d/*.conf" files + local svcOption svc str output status svcOption="${1:-}" - if [[ -z "${svcOption}" ]]; then + if [[ "${svcOption}" =~ "reload" ]]; then + # SIGHUP does NOT re-read any *.conf files + svc="killall -s SIGHUP dnsmasq" + elif [[ -z "${svcOption}" ]]; then # Get PID of dnsmasq to determine if it needs to start or restart if pidof dnsmasq &> /dev/null; then svcOption="restart" else svcOption="start" fi + svc="service dnsmasq ${svcOption}" fi - # Only print output to Terminal, not Web Admin + # Print output to Terminal, not Web Admin str="${svcOption^}ing DNS service" [[ -t 1 ]] && echo -ne " ${INFO} ${str}..." - output=$( { service dnsmasq "${svcOption}"; } 2>&1 ) + output=$( { ${svc}; } 2>&1 ) status="$?" - if [[ "$?" -eq 0 ]]; then + if [[ "${status}" -eq 0 ]]; then [[ -t 1 ]] && echo -e "${OVER} ${TICK} ${str}" else - [[ ! -t 1 ]] && OVER="" + [[ ! -t 1 ]] && local OVER="" echo -e "${OVER} ${CROSS} ${output}" fi } From 95e3a5944d355f8eea6cba5cede4c60885bcf579 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Thu, 17 Aug 2017 19:53:43 +0100 Subject: [PATCH 08/50] Initial overhaul of uninstall script sourcing `distro_check()` from `basic-install.sh` Totally untested as yet... Signed-off-by: Adam Warner --- automated install/uninstall.sh | 38 ++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 336bb953..f2c42f24 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -36,6 +36,24 @@ else fi fi +readonly PI_HOLE_FILES_DIR="/etc/.pihole" +source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh" +# setupVars set in basic-install.sh +source "${setupVars}" + +# distro_check() sourced from basic-install.sh +distro_check + +# Install packages used by the Pi-hole +if [[ "${INSTALL_WEB}" == true ]]; then + # Install the Web dependencies + DEPS=("${PIHOLE_DEPS[@]}" "${PIHOLE_WEB_DEPS[@]}") +# Otherwise, +else + # just install the Core dependencies + DEPS=("${PIHOLE_DEPS[@]}") +fi + # Compatability if [ -x "$(command -v rpm)" ]; then # Fedora Family @@ -45,7 +63,6 @@ if [ -x "$(command -v rpm)" ]; then PKG_MANAGER="yum" fi PKG_REMOVE="${PKG_MANAGER} remove -y" - PIHOLE_DEPS=( bind-utils bc dnsmasq lighttpd lighttpd-fastcgi php-common git curl unzip wget findutils ) package_check() { rpm -qa | grep ^$1- > /dev/null } @@ -56,7 +73,6 @@ elif [ -x "$(command -v apt-get)" ]; then # Debian Family PKG_MANAGER="apt-get" PKG_REMOVE="${PKG_MANAGER} -y remove --purge" - PIHOLE_DEPS=( dnsutils bc dnsmasq lighttpd php5-common git curl unzip wget ) package_check() { dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -c "ok installed" } @@ -72,7 +88,7 @@ fi removeAndPurge() { # Purge dependencies echo "" - for i in "${PIHOLE_DEPS[@]}"; do + for i in "${DEPS[@]}"; do package_check ${i} > /dev/null if [[ "$?" -eq 0 ]]; then while true; do @@ -94,12 +110,12 @@ removeAndPurge() { # Remove dnsmasq config files ${SUDO} rm /etc/dnsmasq.conf /etc/dnsmasq.conf.orig /etc/dnsmasq.d/01-pihole.conf &> /dev/null echo -e " ${TICK} Removing dnsmasq config files" - + # Take care of any additional package cleaning echo -ne " ${INFO} Removing & cleaning remaining dependencies..." package_cleanup &> /dev/null echo -e "${OVER} ${TICK} Removed & cleaned up remaining dependencies" - + # Call removeNoPurge to remove Pi-hole specific files removeNoPurge } @@ -145,7 +161,7 @@ removeNoPurge() { ${SUDO} mv /etc/lighttpd/lighttpd.conf.orig /etc/lighttpd/lighttpd.conf fi fi - + ${SUDO} rm /etc/dnsmasq.d/adList.conf &> /dev/null ${SUDO} rm /etc/dnsmasq.d/01-pihole.conf &> /dev/null ${SUDO} rm -rf /var/log/*pihole* &> /dev/null @@ -156,23 +172,23 @@ removeNoPurge() { ${SUDO} rm /etc/bash_completion.d/pihole &> /dev/null ${SUDO} rm /etc/sudoers.d/pihole &> /dev/null echo -e " ${TICK} Removed config files" - + # Remove FTL if command -v pihole-FTL &> /dev/null; then echo -ne " ${INFO} Removing pihole-FTL..." - + if [[ -x "$(command -v systemctl)" ]]; then systemctl stop pihole-FTL else service pihole-FTL stop fi - + ${SUDO} rm /etc/init.d/pihole-FTL ${SUDO} rm /usr/bin/pihole-FTL - + echo -e "${OVER} ${TICK} Removed pihole-FTL" fi - + # If the pihole user exists, then remove if id "pihole" &> /dev/null; then ${SUDO} userdel -r pihole 2> /dev/null From 3c250944952cf9ab622c3f015af300ce640ce0e9 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Thu, 17 Aug 2017 19:56:36 +0100 Subject: [PATCH 09/50] No need to declare `PKG_MANAGER`, as it's already declared in basic-install Signed-off-by: Adam Warner --- automated install/uninstall.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index f2c42f24..aee0869a 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -57,11 +57,6 @@ fi # Compatability if [ -x "$(command -v rpm)" ]; then # Fedora Family - if [ -x "$(command -v dnf)" ]; then - PKG_MANAGER="dnf" - else - PKG_MANAGER="yum" - fi PKG_REMOVE="${PKG_MANAGER} remove -y" package_check() { rpm -qa | grep ^$1- > /dev/null @@ -71,7 +66,6 @@ if [ -x "$(command -v rpm)" ]; then } elif [ -x "$(command -v apt-get)" ]; then # Debian Family - PKG_MANAGER="apt-get" PKG_REMOVE="${PKG_MANAGER} -y remove --purge" package_check() { dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -c "ok installed" From 07e5e8e67aef0cc6678aab2d59fc080102b4378c Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Thu, 17 Aug 2017 20:03:19 +0100 Subject: [PATCH 10/50] need to include `PH_TEST="true"` so that the install script isn't run when sourcing it.. doh Signed-off-by: Adam Warner --- automated install/uninstall.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index aee0869a..d1dda225 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -37,6 +37,7 @@ else fi readonly PI_HOLE_FILES_DIR="/etc/.pihole" +PH_TEST="true" source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh" # setupVars set in basic-install.sh source "${setupVars}" From 54f4f6d2d76fce90dbc77de4f8f4cffb61af036e Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Thu, 17 Aug 2017 20:17:49 +0100 Subject: [PATCH 11/50] Add "${INSTALLER_DEPS[@]}" to package array add `-f` to rm commands so that `set -e` is not kicked Signed-off-by: Adam Warner --- automated install/uninstall.sh | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index d1dda225..ffc14c9e 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -48,11 +48,11 @@ distro_check # Install packages used by the Pi-hole if [[ "${INSTALL_WEB}" == true ]]; then # Install the Web dependencies - DEPS=("${PIHOLE_DEPS[@]}" "${PIHOLE_WEB_DEPS[@]}") + DEPS=("${INSTALLER_DEPS[@]}" "${PIHOLE_DEPS[@]}" "${PIHOLE_WEB_DEPS[@]}") # Otherwise, else # just install the Core dependencies - DEPS=("${PIHOLE_DEPS[@]}") + DEPS=("${INSTALLER_DEPS[@]}" "${PIHOLE_DEPS[@]}") fi # Compatability @@ -103,7 +103,7 @@ removeAndPurge() { done # Remove dnsmasq config files - ${SUDO} rm /etc/dnsmasq.conf /etc/dnsmasq.conf.orig /etc/dnsmasq.d/01-pihole.conf &> /dev/null + ${SUDO} rm -f /etc/dnsmasq.conf /etc/dnsmasq.conf.orig /etc/dnsmasq.d/01-pihole.conf &> /dev/null echo -e " ${TICK} Removing dnsmasq config files" # Take care of any additional package cleaning @@ -120,7 +120,7 @@ removeNoPurge() { echo -ne " ${INFO} Removing Web Interface..." ${SUDO} rm -rf /var/www/html/admin &> /dev/null ${SUDO} rm -rf /var/www/html/pihole &> /dev/null - ${SUDO} rm /var/www/html/index.lighttpd.orig &> /dev/null + ${SUDO} rm -f /var/www/html/index.lighttpd.orig &> /dev/null # If the web directory is empty after removing these files, then the parent html folder can be removed. if [ -d "/var/www/html" ]; then @@ -143,7 +143,7 @@ removeNoPurge() { # Attempt to preserve backwards compatibility with older versions if [[ -f /etc/cron.d/pihole ]];then - ${SUDO} rm /etc/cron.d/pihole &> /dev/null + ${SUDO} rm -f /etc/cron.d/pihole &> /dev/null echo -e " ${TICK} Removed /etc/cron.d/pihole" fi @@ -157,15 +157,15 @@ removeNoPurge() { fi fi - ${SUDO} rm /etc/dnsmasq.d/adList.conf &> /dev/null - ${SUDO} rm /etc/dnsmasq.d/01-pihole.conf &> /dev/null + ${SUDO} rm -f /etc/dnsmasq.d/adList.conf &> /dev/null + ${SUDO} rm -f /etc/dnsmasq.d/01-pihole.conf &> /dev/null ${SUDO} rm -rf /var/log/*pihole* &> /dev/null ${SUDO} rm -rf /etc/pihole/ &> /dev/null ${SUDO} rm -rf /etc/.pihole/ &> /dev/null ${SUDO} rm -rf /opt/pihole/ &> /dev/null - ${SUDO} rm /usr/local/bin/pihole &> /dev/null - ${SUDO} rm /etc/bash_completion.d/pihole &> /dev/null - ${SUDO} rm /etc/sudoers.d/pihole &> /dev/null + ${SUDO} rm -f /usr/local/bin/pihole &> /dev/null + ${SUDO} rm -f /etc/bash_completion.d/pihole &> /dev/null + ${SUDO} rm -f /etc/sudoers.d/pihole &> /dev/null echo -e " ${TICK} Removed config files" # Remove FTL @@ -178,8 +178,8 @@ removeNoPurge() { service pihole-FTL stop fi - ${SUDO} rm /etc/init.d/pihole-FTL - ${SUDO} rm /usr/bin/pihole-FTL + ${SUDO} rm -f /etc/init.d/pihole-FTL + ${SUDO} rm -f /usr/bin/pihole-FTL echo -e "${OVER} ${TICK} Removed pihole-FTL" fi From 1b3428626491e372aa2d1e2b4a04560efbb09741 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Thu, 17 Aug 2017 20:33:53 +0100 Subject: [PATCH 12/50] verbiage Signed-off-by: Adam Warner --- automated install/uninstall.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index ffc14c9e..88eca716 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -209,7 +209,13 @@ else echo -e " ${INFO} Be sure to confirm if any dependencies should not be removed" fi while true; do - read -rp " ${QST} Do you wish to go through each dependency for removal? [Y/n] " yn + echo -e "${COL_YELLOW} ${INFO} The following dependencies may have been added to the system by Pi-hole install:" + echo -n " " + for i in "${DEPS[@]}"; do + echo -n "${i} " + done + echo "${COL_NC}" + read -rp " ${QST} Do you wish to go through each dependency for removal? (Choosing No will leave all dependencies installed)[Y/n] " yn case ${yn} in [Yy]* ) removeAndPurge; break;; [Nn]* ) removeNoPurge; break;; From e938132c53af84575c5efa8cf2351a48274126c1 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Thu, 17 Aug 2017 21:02:10 +0100 Subject: [PATCH 13/50] accidentally a space Signed-off-by: Adam Warner --- automated install/uninstall.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 88eca716..47a3aa5c 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -214,8 +214,8 @@ while true; do for i in "${DEPS[@]}"; do echo -n "${i} " done - echo "${COL_NC}" - read -rp " ${QST} Do you wish to go through each dependency for removal? (Choosing No will leave all dependencies installed)[Y/n] " yn + echo "${COL_NC}"git pu + read -rp " ${QST} Do you wish to go through each dependency for removal? (Choosing No will leave all dependencies installed) [Y/n] " yn case ${yn} in [Yy]* ) removeAndPurge; break;; [Nn]* ) removeNoPurge; break;; From 939c99cdbf57252d0bee07c3d453ccfb835cef25 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 23 Aug 2017 11:48:11 +0100 Subject: [PATCH 14/50] accidentally an entire group of letters. --- automated install/uninstall.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 47a3aa5c..cd4670b8 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -214,7 +214,7 @@ while true; do for i in "${DEPS[@]}"; do echo -n "${i} " done - echo "${COL_NC}"git pu + echo "${COL_NC}" read -rp " ${QST} Do you wish to go through each dependency for removal? (Choosing No will leave all dependencies installed) [Y/n] " yn case ${yn} in [Yy]* ) removeAndPurge; break;; From 0a00936e99481204154190f3d554926ed8992e48 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Mon, 28 Aug 2017 11:36:02 +1000 Subject: [PATCH 15/50] Update resolver test & added more comments * Add/update code comments * Change resolver check to test for pi.hole * Make resolver check timeout after 10 seconds * Use > instead of &> where appropriate * Make resolver check sleep for 30 seconds (effectively waiting up to 50s for dnsmasq to be resolvable) * Provide confirmation upon success of resolver check availability * Quotes and Braced remaining variables as appropriate * Removed duplicate local --- gravity.sh | 58 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/gravity.sh b/gravity.sh index b222886b..c1254bb2 100755 --- a/gravity.sh +++ b/gravity.sh @@ -64,26 +64,27 @@ if [[ -r "${piholeDir}/pihole.conf" ]]; then echo -e " ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}" fi -# Attempt to resolve DNS before proceeding with blocklist generation +# Determine if DNS resolution is available before proceeding with retrieving blocklists gravity_DNSLookup() { local plu - # Determine if github.com can be resolved - if ! timeout 2 nslookup github.com &> /dev/null; then + # Determine if pi.hole can be resolved + if ! timeout 10 nslookup pi.hole > /dev/null; then if [[ -n "${secs}" ]]; then echo -e "${OVER} ${CROSS} DNS resolution is still unavailable, cancelling" exit 1 fi # Determine error output message - if pidof dnsmasq &> /dev/null; then + if pidof dnsmasq > /dev/null; then echo -e " ${CROSS} DNS resolution is temporarily unavailable" else echo -e " ${CROSS} DNS service is not running" "${PIHOLE_COMMAND}" restartdns fi - secs="15" + # Give time for dnsmasq to be resolvable + secs="30" while [[ "${secs}" -ne 0 ]]; do [[ "${secs}" -ne 1 ]] && plu="s" || plu="" echo -ne "${OVER} ${INFO} Waiting $secs second${plu} before continuing..." @@ -93,6 +94,9 @@ gravity_DNSLookup() { # Try again gravity_DNSLookup + elif [[ -n "${secs}" ]]; then + # Print confirmation of resolvability if it had previously failed + echo -e "${OVER} ${TICK} DNS resolution is now available\\n" fi } @@ -106,7 +110,7 @@ gravity_Collapse() { rm "${adListDefault}" 2> /dev/null || \ echo -e " ${CROSS} Unable to remove ${adListDefault}" elif [[ ! -f "${adListFile}" ]]; then - # Create "adlists.list" + # Create "adlists.list" by copying "adlists.default" from internal Pi-hole repo cp "${adListRepoDefault}" "${adListFile}" 2> /dev/null || \ echo -e " ${CROSS} Unable to copy ${adListFile##*/} from ${piholeRepo}" fi @@ -115,11 +119,11 @@ gravity_Collapse() { echo -ne " ${INFO} ${str}..." # Retrieve source URLs from $adListFile - # Logic: Remove comments, CR line endings and empty lines + # Awk Logic: Remove comments, CR line endings and empty lines mapfile -t sources < <(awk '!/^[#@;!\[]/ {gsub(/\r$/, "", $0); if ($1) { print $1 } }' "${adListFile}" 2> /dev/null) # Parse source domains from $sources - # Logic: Split by folder/port, remove URL protocol & optional username:password@ + # Awk Logic: Split by folder/port, remove URL protocol & optional username:password@ mapfile -t sourceDomains < <( awk -F '[/:]' '{ gsub(/(.*:\/\/|.*:.*@)/, "", $0) @@ -150,9 +154,10 @@ gravity_Supernova() { saveLocation="${piholeDir}/list.${i}.${domain}.${domainsExtension}" activeDomains[$i]="${saveLocation}" + # Default user-agent (for Cloudflare's Browser Integrity Check: https://support.cloudflare.com/hc/en-us/articles/200170086-What-does-the-Browser-Integrity-Check-do-) agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36" - # Use a case statement to download lists that need special commands + # Provide special commands for blocklists which may need them case "${domain}" in "pgl.yoyo.org") cmd_ext="-d mimetype=plaintext -d hostformat=hosts";; *) cmd_ext="";; @@ -179,9 +184,10 @@ gravity_Pull() { # Store downloaded content to temp file instead of RAM patternBuffer=$(mktemp) + heisenbergCompensator="" if [[ -r "${saveLocation}" ]]; then - # If domain has been saved, add file for date check to only download newer + # Allow curl to determine if a remote file has been modified since last retrieval heisenbergCompensator="-z ${saveLocation}" fi @@ -191,7 +197,7 @@ gravity_Pull() { httpCode=$(curl -s -L ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${patternBuffer}" 2> /dev/null) # Determine "Status:" output based on HTTP response - case "$httpCode" in + case "${httpCode}" in "200" ) echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success="true";; "304" ) echo -e "${OVER} ${TICK} ${str} No changes detected"; success="true";; "403" ) echo -e "${OVER} ${CROSS} ${str} Forbidden"; success="false";; @@ -201,7 +207,7 @@ gravity_Pull() { "521" ) echo -e "${OVER} ${CROSS} ${str} Web Server Is Down (Cloudflare)"; success="false";; "522" ) echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Cloudflare)"; success="false";; "500" ) echo -e "${OVER} ${CROSS} ${str} Internal Server Error"; success="false";; - * ) echo -e "${OVER} ${CROSS} ${str} Status $httpCode"; success="false";; + * ) echo -e "${OVER} ${CROSS} ${str} Status ${httpCode}"; success="false";; esac # Determine if the blocklist was downloaded and saved correctly @@ -242,14 +248,14 @@ gravity_ParseFileIntoDomains() { if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then # Consolidated list parsing: Remove comments and hosts IP's - # Symbols used as comments: #;@![/ + # Define symbols used as comments: #;@![/ commentPattern="[#;@![\\/]" - # Logic: Process lines which do not begin with comments + # Awk Logic: Process lines which do not begin with comments awk '!/^'"${commentPattern}"'/ { # If there are multiple words seperated by space if (NF>1) { - # Remove comments (Inc. prefixed spaces/tabs) + # Remove comments (including prefixed spaces/tabs) if ($0 ~ /'"${commentPattern}"'/) { gsub("( |\t)'"${commentPattern}"'.*", "", $0) } # Print consecutive domains if ($3) { @@ -271,8 +277,9 @@ gravity_ParseFileIntoDomains() { read -r firstLine < "${source}" # Determine how to parse individual source file formats + # Lists may not capitalise the first line correctly, so compare strings against lower case if [[ "${firstLine,,}" =~ "adblock" ]] || [[ "${firstLine,,}" =~ "ublock" ]] || [[ "${firstLine,,}" =~ "! checksum" ]]; then - # Logic: Parse Adblock domains & comments: https://adblockplus.org/filter-cheatsheet + # Awk Logic: Parse Adblock domains & comments: https://adblockplus.org/filter-cheatsheet abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/" awk ''"${abpFilter}"' { # Remove valid adblock type options @@ -338,9 +345,10 @@ gravity_Filter() { str="Extracting domains from blocklists" echo -ne " ${INFO} ${str}..." - # Parse files as hosts + # Parse into hosts file gravity_ParseFileIntoDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${parsedMatter}" + # Format file line count as currency num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${parsedMatter}")") echo -e "${OVER} ${TICK} ${str} ${INFO} ${COL_LIGHT_BLUE}${num}${COL_NC} domains being pulled in by gravity" @@ -357,6 +365,7 @@ gravity_Unique() { sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}" echo -e "${OVER} ${TICK} ${str}" + # Format file line count as currency num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") echo -e " ${INFO} ${COL_LIGHT_BLUE}${num}${COL_NC} unique domains trapped in the Event Horizon" } @@ -371,6 +380,7 @@ gravity_WhitelistBLD() { echo -ne " ${INFO} ${str}..." # Create array of unique $sourceDomains + # Disable SC2046 as quoting will only return first domain # shellcheck disable=SC2046 read -r -a uniqDomains <<< $(awk '{ if(!a[$1]++) { print $1 } }' <<< "$(printf '%s\n' "${sourceDomains[@]}")") @@ -388,7 +398,7 @@ gravity_Whitelist() { # Remove anything in whitelist.txt from the Event Horizon num=$(wc -l < "${whitelistFile}") plural=; [[ "${num}" != "1" ]] && plural=s - local str="Whitelisting ${num} domain${plural}" + str="Whitelisting ${num} domain${plural}" echo -ne " ${INFO} ${str}..." # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in whitelist.txt @@ -429,6 +439,7 @@ gravity_ShowBlockCount() { # Parse list of domains into hosts format gravity_ParseDomainsIntoHosts() { if [[ -n "${IPV4_ADDRESS}" ]] || [[ -n "${IPV6_ADDRESS}" ]]; then + # Awk Logic: Remove CR line endings and print IP before domain if IPv4/6 is used awk -v ipv4addr="$IPV4_ADDRESS" -v ipv6addr="$IPV6_ADDRESS" '{ sub(/\r$/, "") if(ipv4addr) { print ipv4addr" "$0; } @@ -447,7 +458,7 @@ gravity_ParseLocalDomains() { if [[ -f "/etc/hostname" ]]; then hostname=$(< "/etc/hostname") - elif command -v hostname &> /dev/null; then + elif command -v hostname > /dev/null; then hostname=$(hostname -f) else echo -e " ${CROSS} Unable to determine fully qualified domain name of host" @@ -471,8 +482,7 @@ gravity_ParseBlacklistDomains() { status="$?" if [[ "${status}" -ne 0 ]]; then - echo -e " ${CROSS} Unable to move ${accretionDisc} from ${piholeDir} - ${output}" + echo -e " ${CROSS} Unable to move ${accretionDisc} from ${piholeDir}\\n ${output}" gravity_Cleanup "error" fi } @@ -507,7 +517,7 @@ gravity_Cleanup() { # Remove any unused .domains files for file in ${piholeDir}/*.${domainsExtension}; do # If list is not in active array, then remove it - if [[ ! "${activeDomains[*]}" =~ ${file} ]]; then + if [[ ! "${activeDomains[*]}" =~ "${file}" ]]; then rm -f "${file}" 2> /dev/null || \ echo -e " ${CROSS} Failed to remove ${file##*/}" fi @@ -515,7 +525,7 @@ gravity_Cleanup() { echo -e "${OVER} ${TICK} ${str}" - [[ -n "$error" ]] && echo "" + [[ -n "${error}" ]] && echo "" # Only restart DNS service if offline if ! pidof dnsmasq &> /dev/null; then @@ -523,7 +533,7 @@ gravity_Cleanup() { fi # Print Pi-hole status if an error occured - if [[ -n "$error" ]]; then + if [[ -n "${error}" ]]; then "${PIHOLE_COMMAND}" status exit 1 fi From 4d39ab9753f1df520c50b86b9d126353382f6e13 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Thu, 14 Sep 2017 16:39:25 +1000 Subject: [PATCH 16/50] Allow force-reload to be used when restarting DNS * Remove duplicate coltable variable definition and source * Minor comment modifications * Add "$2" to restartdns --- pihole | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pihole b/pihole index 61ef0e20..625ae8bb 100755 --- a/pihole +++ b/pihole @@ -12,10 +12,6 @@ readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" readonly wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf" readonly colfile="${PI_HOLE_SCRIPT_DIR}/COL_TABLE" - -source ${colfile} - -colfile="${PI_HOLE_SCRIPT_DIR}/COL_TABLE" source "${colfile}" # Must be root to use this tool @@ -354,8 +350,9 @@ restartDNS() { local svcOption svc str output status svcOption="${1:-}" + # Determine if we should reload or restart dnsmasq if [[ "${svcOption}" =~ "reload" ]]; then - # SIGHUP does NOT re-read any *.conf files + # Using SIGHUP will NOT re-read any *.conf files svc="killall -s SIGHUP dnsmasq" elif [[ -z "${svcOption}" ]]; then # Get PID of dnsmasq to determine if it needs to start or restart @@ -367,7 +364,7 @@ restartDNS() { svc="service dnsmasq ${svcOption}" fi - # Print output to Terminal, not Web Admin + # Print output to Terminal, but not to Web Admin str="${svcOption^}ing DNS service" [[ -t 1 ]] && echo -ne " ${INFO} ${str}..." @@ -659,7 +656,7 @@ case "${1}" in "enable" ) piholeEnable 1;; "disable" ) piholeEnable 0 "$2";; "status" ) statusFunc "$2";; - "restartdns" ) restartDNS;; + "restartdns" ) restartDNS "$2";; "-a" | "admin" ) webpageFunc "$@";; "-t" | "tail" ) tailFunc;; "checkout" ) piholeCheckoutFunc "$@";; From 8191ec01e5ca0699a29f07076f51396049440577 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Thu, 14 Sep 2017 16:39:30 +1000 Subject: [PATCH 17/50] Gravity Bugfixes * Merge development OpenVPN code * Determine which domain to resolve depending on existence of $localList * Re-add code to remove $localList, preventing duplicate local entries * Minor shellcheck validation fix --- gravity.sh | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/gravity.sh b/gravity.sh index c1254bb2..354bb51e 100755 --- a/gravity.sh +++ b/gravity.sh @@ -32,6 +32,7 @@ wildcardFile="/etc/dnsmasq.d/03-pihole-wildcard.conf" adList="${piholeDir}/gravity.list" blackList="${piholeDir}/black.list" localList="${piholeDir}/local.list" +VPNList="/etc/openvpn/ipp.txt" domainsExtension="domains" matterAndLight="${basename}.0.matterandlight.txt" @@ -66,10 +67,17 @@ fi # Determine if DNS resolution is available before proceeding with retrieving blocklists gravity_DNSLookup() { - local plu + local lookupDomain plu - # Determine if pi.hole can be resolved - if ! timeout 10 nslookup pi.hole > /dev/null; then + # Determine which domain to resolve depending on existence of $localList + if [[ -e "${localList}" ]]; then + lookupDomain="pi.hole" + else + lookupDomain="raw.githubusercontent.com" + fi + + # Determine if domain can be resolved + if ! timeout 10 nslookup "${lookupDomain}" > /dev/null; then if [[ -n "${secs}" ]]; then echo -e "${OVER} ${CROSS} DNS resolution is still unavailable, cancelling" exit 1 @@ -452,20 +460,28 @@ gravity_ParseDomainsIntoHosts() { fi } -# Create "localhost" entries +# Create "localhost" entries into hosts format gravity_ParseLocalDomains() { local hostname if [[ -f "/etc/hostname" ]]; then hostname=$(< "/etc/hostname") - elif command -v hostname > /dev/null; then + elif command -v hostname &> /dev/null; then hostname=$(hostname -f) else echo -e " ${CROSS} Unable to determine fully qualified domain name of host" fi echo -e "${hostname}\\npi.hole" > "${localList}.tmp" + # Copy the file over as /etc/pihole/local.list so dnsmasq can use it + rm "${localList}" 2> /dev/null || true gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}" + rm "${localList}.tmp" 2> /dev/null || true + + # Add additional local hosts provided by OpenVPN (if available) + if [[ -f "${VPNList}" ]]; then + awk -F, '{printf $2"\t"$1"\n"}' "${VPNList}" >> "${localList}" + fi } # Create primary blacklist entries @@ -494,8 +510,6 @@ gravity_ParseUserDomains() { # Copy the file over as /etc/pihole/black.list so dnsmasq can use it mv "${blackList}.tmp" "${blackList}" 2> /dev/null || \ echo -e " ${CROSS} Unable to move ${blackList##*/}.tmp to ${piholeDir}" - else - echo -e " ${INFO} Nothing to blacklist!" fi } @@ -517,7 +531,7 @@ gravity_Cleanup() { # Remove any unused .domains files for file in ${piholeDir}/*.${domainsExtension}; do # If list is not in active array, then remove it - if [[ ! "${activeDomains[*]}" =~ "${file}" ]]; then + if [[ ! "${activeDomains[*]}" == *"${file}"* ]]; then rm -f "${file}" 2> /dev/null || \ echo -e " ${CROSS} Failed to remove ${file##*/}" fi From d3073e5e2391e4338251fc11bf2b52ddbcc22234 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Thu, 14 Sep 2017 17:09:52 +1000 Subject: [PATCH 18/50] Fix array of unique $sourceDomains --- gravity.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index 354bb51e..39a0e75b 100755 --- a/gravity.sh +++ b/gravity.sh @@ -388,9 +388,7 @@ gravity_WhitelistBLD() { echo -ne " ${INFO} ${str}..." # Create array of unique $sourceDomains - # Disable SC2046 as quoting will only return first domain - # shellcheck disable=SC2046 - read -r -a uniqDomains <<< $(awk '{ if(!a[$1]++) { print $1 } }' <<< "$(printf '%s\n' "${sourceDomains[@]}")") + mapfile -t uniqDomains <<< "$(awk '{ if(!a[$1]++) { print $1 } }' <<< "$(printf '%s\n' "${sourceDomains[@]}")")" ${WHITELIST_COMMAND} -nr -q "${uniqDomains[*]}" > /dev/null From ff5411a93a08c78d95e7811239214fbd0708e74c Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Thu, 14 Sep 2017 20:23:49 +1000 Subject: [PATCH 19/50] Add 'Connection Refused' for 000 status * Shift default dnsRestart value into unset default parameter expansion value * Change nslookup timeout to 5 seconds * Use &> instead of > * Standardise plural code * Update some comments * Add "000" when connection is refused * Condense adblock detection logic * Add Dnsmasq format detection and parsing logic * Removed unnecessary echo * Add dnsWasOffline variable to ensure that if DNS service has been stopped, that it doesn't start and also get reloaded --- gravity.sh | 89 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 38 deletions(-) diff --git a/gravity.sh b/gravity.sh index 39a0e75b..a2f5d978 100755 --- a/gravity.sh +++ b/gravity.sh @@ -43,9 +43,6 @@ preEventHorizon="list.preEventHorizon" skipDownload="false" -# Use "force-reload" when restarting dnsmasq for everything but Wildcards -dnsRestart="force-reload" - # Source setupVars from install script setupVars="${piholeDir}/setupVars.conf" if [[ -f "${setupVars}" ]];then @@ -67,9 +64,10 @@ fi # Determine if DNS resolution is available before proceeding with retrieving blocklists gravity_DNSLookup() { - local lookupDomain plu + local lookupDomain plural - # Determine which domain to resolve depending on existence of $localList + # Determine which domain should be resolved + # "pi.hole" is not always available (e.g: new install), but FTL will not log it if [[ -e "${localList}" ]]; then lookupDomain="pi.hole" else @@ -77,25 +75,25 @@ gravity_DNSLookup() { fi # Determine if domain can be resolved - if ! timeout 10 nslookup "${lookupDomain}" > /dev/null; then + if ! timeout 5 nslookup "${lookupDomain}" &> /dev/null; then if [[ -n "${secs}" ]]; then echo -e "${OVER} ${CROSS} DNS resolution is still unavailable, cancelling" exit 1 fi # Determine error output message - if pidof dnsmasq > /dev/null; then + if pidof dnsmasq &> /dev/null; then echo -e " ${CROSS} DNS resolution is temporarily unavailable" else echo -e " ${CROSS} DNS service is not running" "${PIHOLE_COMMAND}" restartdns fi - # Give time for dnsmasq to be resolvable + # Give time for DNS server to be resolvable secs="30" while [[ "${secs}" -ne 0 ]]; do - [[ "${secs}" -ne 1 ]] && plu="s" || plu="" - echo -ne "${OVER} ${INFO} Waiting $secs second${plu} before continuing..." + plural=; [[ "${secs}" -ne 1 ]] && plural="s" + echo -ne "${OVER} ${INFO} Waiting $secs second${plural} before continuing..." sleep 1 : $((secs--)) done @@ -127,7 +125,7 @@ gravity_Collapse() { echo -ne " ${INFO} ${str}..." # Retrieve source URLs from $adListFile - # Awk Logic: Remove comments, CR line endings and empty lines + # Awk Logic: Remove comments (#@;![), CR (windows) line endings and empty lines mapfile -t sources < <(awk '!/^[#@;!\[]/ {gsub(/\r$/, "", $0); if ($1) { print $1 } }' "${adListFile}" 2> /dev/null) # Parse source domains from $sources @@ -182,7 +180,7 @@ gravity_Supernova() { done } -# Download specified URL and perform QA +# Download specified URL and perform checks on HTTP status and file content gravity_Pull() { local url cmd_ext agent heisenbergCompensator patternBuffer str httpCode success @@ -195,7 +193,8 @@ gravity_Pull() { heisenbergCompensator="" if [[ -r "${saveLocation}" ]]; then - # Allow curl to determine if a remote file has been modified since last retrieval + # Make curl determine if a remote file has been modified since last retrieval + # Uses "Last-Modified" header, which certain web servers do not provide (e.g: raw github links) heisenbergCompensator="-z ${saveLocation}" fi @@ -206,28 +205,29 @@ gravity_Pull() { # Determine "Status:" output based on HTTP response case "${httpCode}" in - "200" ) echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success="true";; - "304" ) echo -e "${OVER} ${TICK} ${str} No changes detected"; success="true";; - "403" ) echo -e "${OVER} ${CROSS} ${str} Forbidden"; success="false";; - "404" ) echo -e "${OVER} ${CROSS} ${str} Not found"; success="false";; - "408" ) echo -e "${OVER} ${CROSS} ${str} Time-out"; success="false";; - "451" ) echo -e "${OVER} ${CROSS} ${str} Unavailable For Legal Reasons"; success="false";; - "521" ) echo -e "${OVER} ${CROSS} ${str} Web Server Is Down (Cloudflare)"; success="false";; - "522" ) echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Cloudflare)"; success="false";; - "500" ) echo -e "${OVER} ${CROSS} ${str} Internal Server Error"; success="false";; - * ) echo -e "${OVER} ${CROSS} ${str} Status ${httpCode}"; success="false";; + "200") echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success="true";; + "304") echo -e "${OVER} ${TICK} ${str} No changes detected"; success="true";; + "000") echo -e "${OVER} ${CROSS} ${str} Connection Refused"; success="false";; + "403") echo -e "${OVER} ${CROSS} ${str} Forbidden"; success="false";; + "404") echo -e "${OVER} ${CROSS} ${str} Not found"; success="false";; + "408") echo -e "${OVER} ${CROSS} ${str} Time-out"; success="false";; + "451") echo -e "${OVER} ${CROSS} ${str} Unavailable For Legal Reasons"; success="false";; + "521") echo -e "${OVER} ${CROSS} ${str} Web Server Is Down (Cloudflare)"; success="false";; + "522") echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Cloudflare)"; success="false";; + "500") echo -e "${OVER} ${CROSS} ${str} Internal Server Error"; success="false";; + * ) echo -e "${OVER} ${CROSS} ${str} ${httpCode}"; success="false";; esac # Determine if the blocklist was downloaded and saved correctly if [[ "${success}" == "true" ]]; then if [[ "${httpCode}" == "304" ]]; then - : # Do nothing - # Check if patternbuffer is a non-zero length file + : # Do not attempt to re-parse file + # Check if $patternbuffer is a non-zero length file elif [[ -s "${patternBuffer}" ]]; then # Determine if blocklist is non-standard and parse as appropriate gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}" else - # Fall back to previously cached list if patternBuffer is empty + # Fall back to previously cached list if $patternBuffer is empty echo -e " ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" fi else @@ -248,9 +248,7 @@ gravity_Pull() { # Parse non-standard source files into domains-only format gravity_ParseFileIntoDomains() { - local source destination commentPattern firstLine abpFilter - source="${1}" - destination="${2}" + local source="${1}" destination="${2}" commentPattern firstLine abpFilter # Determine how to parse source file if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then @@ -286,7 +284,7 @@ gravity_ParseFileIntoDomains() { # Determine how to parse individual source file formats # Lists may not capitalise the first line correctly, so compare strings against lower case - if [[ "${firstLine,,}" =~ "adblock" ]] || [[ "${firstLine,,}" =~ "ublock" ]] || [[ "${firstLine,,}" =~ "! checksum" ]]; then + if [[ "${firstLine,,}" =~ (adblock|ublock|!checksum) ]]; then # Awk Logic: Parse Adblock domains & comments: https://adblockplus.org/filter-cheatsheet abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/" awk ''"${abpFilter}"' { @@ -298,8 +296,8 @@ gravity_ParseFileIntoDomains() { if ($0 ~ /(^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$|[\\^\/\*])/) { $0="" } if ($0) { print $0 } }' "${source}" 2> /dev/null > "${destination}" + # Parse URL list if source file contains http:// or IPv4 elif grep -q -E "^(https?://|([0-9]{1,3}\\.){3}[0-9]{1,3}$)" "${source}" &> /dev/null; then - # Parse URLs if source file contains http:// or IPv4 awk '{ # Remove URL protocol, optional "username:password@", and ":?/;" if ($0 ~ /[:?\/;]/) { gsub(/(^.*:\/\/(.*:.*@)?|[:?\/;].*)/, "", $0) } @@ -307,6 +305,17 @@ gravity_ParseFileIntoDomains() { if ($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" } if ($0) { print $0 } }' "${source}" 2> /dev/null > "${destination}" + # Parse Dnsmasq format lists + elif grep -q "^address=/" "${source}" &> /dev/null; then + awk -F/ '{ + # Print comments + if ($0 ~ "#") { + print $0 + # Print domains + } else if ($2) { + print $2 + } + }' "${source}" 2> /dev/null > "${destination}" else # Keep hosts/domains file in same format as it was downloaded output=$( { mv "${source}" "${destination}"; } 2>&1 ) @@ -383,7 +392,7 @@ gravity_WhitelistBLD() { local plural str uniqDomains echo "" - plural=; [[ "${#sources[*]}" != "1" ]] && plural=s + plural=; [[ "${#sources[*]}" != "1" ]] && plural="s" str="Adding blocklist source${plural} to the whitelist" echo -ne " ${INFO} ${str}..." @@ -403,7 +412,7 @@ gravity_Whitelist() { if [[ -f "${whitelistFile}" ]]; then # Remove anything in whitelist.txt from the Event Horizon num=$(wc -l < "${whitelistFile}") - plural=; [[ "${num}" != "1" ]] && plural=s + plural=; [[ "${num}" -ne 1 ]] && plural="s" str="Whitelisting ${num} domain${plural}" echo -ne " ${INFO} ${str}..." @@ -422,7 +431,7 @@ gravity_ShowBlockCount() { if [[ -f "${blacklistFile}" ]]; then num=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")") - plural=; [[ "${num}" != "1" ]] && plural=s + plural=; [[ "${num}" -ne 1 ]] && plural="s" str="Exact blocked domain${plural}: ${num}" echo -e " ${INFO} ${str}" else @@ -435,7 +444,7 @@ gravity_ShowBlockCount() { if [[ -n "${IPV4_ADDRESS}" ]] && [[ -n "${IPV6_ADDRESS}" ]];then num=$(( num/2 )) fi - plural=; [[ "${num}" != "1" ]] && plural=s + plural=; [[ "${num}" -ne 1 ]] && plural="s" echo -e " ${INFO} Wildcard blocked domain${plural}: ${num}" else echo -e " ${INFO} No wildcards used!" @@ -536,12 +545,11 @@ gravity_Cleanup() { done echo -e "${OVER} ${TICK} ${str}" - - [[ -n "${error}" ]] && echo "" # Only restart DNS service if offline if ! pidof dnsmasq &> /dev/null; then "${PIHOLE_COMMAND}" restartdns + dnsWasOffline=true fi # Print Pi-hole status if an error occured @@ -634,5 +642,10 @@ if [[ "${skipDownload}" == false ]]; then fi echo "" -"${PIHOLE_COMMAND}" restartdns "${dnsRestart}" + +# Determine if DNS has been restarted by this instance of gravity +if [[ -z "${dnsWasOffline:-}" ]]; then + # Use "force-reload" when restarting dnsmasq for everything but Wildcards + "${PIHOLE_COMMAND}" restartdns "${dnsRestart:-force-reload}" +fi "${PIHOLE_COMMAND}" status From 28063aa7f7ae0331ce45b172f9c6a84323b58163 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Fri, 15 Sep 2017 00:24:29 +1000 Subject: [PATCH 20/50] Add administrative contact address functionality * Disable shellcheck for SC1090 --- advanced/Scripts/webpage.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index fbba3f74..4d572f49 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# shellcheck disable=SC1090 + # Pi-hole: A black hole for Internet advertisements # (c) 2017 Pi-hole, LLC (https://pi-hole.net) # Network-wide ad blocking via your own hardware. @@ -30,6 +32,7 @@ Options: -f, fahrenheit Set Fahrenheit as preferred temperature unit -k, kelvin Set Kelvin as preferred temperature unit -r, hostrecord Add a name to the DNS associated to an IPv4/IPv6 address + -e, email Set an administrative contact address for the Block Page -h, --help Show this help dialog -i, interface Specify dnsmasq's interface listening behavior Add '-h' for more info on interface usage" @@ -427,6 +430,27 @@ Options: RestartDNS } +SetAdminEmail() { + if [[ "${1}" == *"-h"* ]]; then + echo "Usage: pihole -a email
+Example: 'pihole -a email admin@address.com' +Set an administrative contact address for the Block Page + +Options: + \"\" Empty: Remove admin contact + -h, --help Show this help dialog" + exit 0 + fi + + if [[ -n "${args[2]}" ]]; then + change_setting "ADMIN_EMAIL" "${args[2]}" + echo -e " ${TICK} Setting admin contact to ${args[2]}" + else + change_setting "ADMIN_EMAIL" "" + echo -e " ${TICK} Removing admin contact" + fi +} + SetListeningMode() { source "${setupVars}" @@ -497,6 +521,7 @@ main() { "addstaticdhcp" ) AddDHCPStaticAddress;; "removestaticdhcp" ) RemoveDHCPStaticAddress;; "-r" | "hostrecord" ) SetHostRecord "$3";; + "-e" | "email" ) SetAdminEmail "$3";; "-i" | "interface" ) SetListeningMode "$@";; "-t" | "teleporter" ) Teleporter;; "adlist" ) CustomizeAdLists;; From c957124fad96efd0a0462afa0e17289fb3fd6059 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Fri, 15 Sep 2017 22:39:17 +1000 Subject: [PATCH 21/50] Optimised parsing of domains on IPv6 servers * Remove WHITELIST_COMMAND * Place IPv4/IPv6 availability test underneath setupVars.conf source * Improved clarity on comments * Define default lookupDomain on local line * Use `getent hosts` instead of nslookup (faster) * Make gravity_DNSLookup() function more readable * Use bold on "Neutrino emissions detected" * Swap conditionals around on adlists file handling * Add comments to both gravity_Collapse() `awk`s * Removed unnecessary "${str}" from gravity_Pull() * Merge function variables into local line * Place .phgbp suffice on mktemp, so patternbuffers can be cleaned up all at once in gravity_Cleanup() * Removed success="false" from $httpCode case, placed empty success var in local * Reordered $httpCode case numerically because I can * Provide error if Dnsmasq format list is being parsed * Remove IPv4 check when determining URL list (too slow on large lists) * Check ${#sources[@]} to ensure we're checking the number of entries and not the character count * Define empty plural in local line, removing unnecessary plural=; * Optimised readability of gravity_Whitelist() * Removed uninformative "Nothing to blacklist"/"No wildcards used" text * Optimised parsing of domains into hosts format on IPv6 enabled servers * Ensure /etc/hostname is non-zero * Use `: >` instead of `rm` as consistent with the rest of the script * Ensured that gravity_Cleanup() removes ${localList}.tmp * Optimised readability of gravity_ParseUserDomains() * Moved dnsRestart to ${var} case statement, renaming it to dnsRestartType for readability * Set default $listType to ensure script passes "bash strict mode" --- gravity.sh | 420 ++++++++++++++++++++++++++--------------------------- 1 file changed, 207 insertions(+), 213 deletions(-) diff --git a/gravity.sh b/gravity.sh index a2f5d978..c1295852 100755 --- a/gravity.sh +++ b/gravity.sh @@ -16,7 +16,6 @@ source "${coltable}" basename="pihole" PIHOLE_COMMAND="/usr/local/bin/${basename}" -WHITELIST_COMMAND="${PIHOLE_COMMAND} -w" piholeDir="/etc/${basename}" piholeRepo="/etc/.${basename}" @@ -51,87 +50,99 @@ if [[ -f "${setupVars}" ]];then # Remove CIDR mask from IPv4/6 addresses IPV4_ADDRESS="${IPV4_ADDRESS%/*}" IPV6_ADDRESS="${IPV6_ADDRESS%/*}" + + # Determine if IPv4/6 addresses exist + if [[ -z "${IPV4_ADDRESS}" ]] && [[ -z "${IPV6_ADDRESS}" ]]; then + echo -e " ${COL_LIGHT_RED}No IP addresses found! Please run 'pihole -r' to reconfigure${COL_NC}" + exit 1 + fi else echo -e " ${COL_LIGHT_RED}Installation Failure: ${setupVars} does not exist! ${COL_NC} Please run 'pihole -r', and choose the 'reconfigure' option to fix." exit 1 fi -# Warn users still using pihole.conf that it no longer has any effect +# Determine if superseded pihole.conf exists if [[ -r "${piholeDir}/pihole.conf" ]]; then echo -e " ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}" fi -# Determine if DNS resolution is available before proceeding with retrieving blocklists +# Determine if DNS resolution is available before proceeding gravity_DNSLookup() { - local lookupDomain plural + local lookupDomain="pi.hole" plural="" - # Determine which domain should be resolved - # "pi.hole" is not always available (e.g: new install), but FTL will not log it - if [[ -e "${localList}" ]]; then - lookupDomain="pi.hole" - else + # Determine if $localList does not exist + if [[ ! -e "${localList}" ]]; then lookupDomain="raw.githubusercontent.com" fi - # Determine if domain can be resolved - if ! timeout 5 nslookup "${lookupDomain}" &> /dev/null; then - if [[ -n "${secs}" ]]; then - echo -e "${OVER} ${CROSS} DNS resolution is still unavailable, cancelling" - exit 1 - fi - - # Determine error output message - if pidof dnsmasq &> /dev/null; then - echo -e " ${CROSS} DNS resolution is temporarily unavailable" - else - echo -e " ${CROSS} DNS service is not running" - "${PIHOLE_COMMAND}" restartdns - fi - - # Give time for DNS server to be resolvable - secs="30" - while [[ "${secs}" -ne 0 ]]; do - plural=; [[ "${secs}" -ne 1 ]] && plural="s" - echo -ne "${OVER} ${INFO} Waiting $secs second${plural} before continuing..." - sleep 1 - : $((secs--)) - done - - # Try again - gravity_DNSLookup - elif [[ -n "${secs}" ]]; then + # Determine if $lookupDomain is resolvable + if timeout 5 getent hosts "${lookupDomain}" &> /dev/null; then # Print confirmation of resolvability if it had previously failed - echo -e "${OVER} ${TICK} DNS resolution is now available\\n" + if [[ -n "${secs:-}" ]]; then + echo -e "${OVER} ${TICK} DNS resolution is now available\\n" + fi + return 0 + elif [[ -n "${secs:-}" ]]; then + echo -e "${OVER} ${CROSS} DNS resolution is still unavailable, cancelling" + exit 1 fi + + # Determine error output message + if pidof dnsmasq &> /dev/null; then + echo -e " ${CROSS} DNS resolution is currently unavailable" + else + echo -e " ${CROSS} DNS service is not running" + "${PIHOLE_COMMAND}" restartdns + fi + + # Give time for DNS server to be resolvable + secs="30" + while [[ "${secs}" -ne 0 ]]; do + [[ "${secs}" -ne 1 ]] && plural="s" + echo -ne "${OVER} ${INFO} Waiting $secs second${plural} before continuing..." + sleep 1 + : $((secs--)) + done + + # Try again + gravity_DNSLookup } # Retrieve blocklist URLs and parse domains from adlists.list gravity_Collapse() { - echo -e " ${INFO} Neutrino emissions detected..." + echo -e " ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..." - # Handle "adlists.list" and "adlists.default" files - if [[ -f "${adListDefault}" ]] && [[ -f "${adListFile}" ]]; then + # Determine if adlists file needs handling + if [[ ! -f "${adListFile}" ]]; then + # Create "adlists.list" by copying "adlists.default" from internal core repo + cp "${adListRepoDefault}" "${adListFile}" 2> /dev/null || \ + echo -e " ${CROSS} Unable to copy ${adListFile##*/} from ${piholeRepo}" + elif [[ -f "${adListDefault}" ]] && [[ -f "${adListFile}" ]]; then # Remove superceded $adListDefault file rm "${adListDefault}" 2> /dev/null || \ echo -e " ${CROSS} Unable to remove ${adListDefault}" - elif [[ ! -f "${adListFile}" ]]; then - # Create "adlists.list" by copying "adlists.default" from internal Pi-hole repo - cp "${adListRepoDefault}" "${adListFile}" 2> /dev/null || \ - echo -e " ${CROSS} Unable to copy ${adListFile##*/} from ${piholeRepo}" fi local str="Pulling blocklist source list into range" echo -ne " ${INFO} ${str}..." # Retrieve source URLs from $adListFile - # Awk Logic: Remove comments (#@;![), CR (windows) line endings and empty lines - mapfile -t sources < <(awk '!/^[#@;!\[]/ {gsub(/\r$/, "", $0); if ($1) { print $1 } }' "${adListFile}" 2> /dev/null) + mapfile -t sources <<< $( + # Logic: Remove comments (#@;![) + awk '!/^[#@;!\[]/ { + # Remove windows CR line endings + gsub(/\r$/, "", $0) + # Print non-empty line + if ($1) { print $1 } + }' "${adListFile}" 2> /dev/null + ) # Parse source domains from $sources - # Awk Logic: Split by folder/port, remove URL protocol & optional username:password@ - mapfile -t sourceDomains < <( + mapfile -t sourceDomains <<< $( + # Logic: Split by folder/port awk -F '[/:]' '{ + # Remove URL protocol & optional username:password@ gsub(/(.*:\/\/|.*:.*@)/, "", $0) print $1 }' <<< "$(printf '%s\n' "${sources[@]}")" 2> /dev/null @@ -151,7 +162,7 @@ gravity_Supernova() { echo "" - # Loop through $sources to download each one + # Loop through $sources and download each one for ((i = 0; i < "${#sources[@]}"; i++)); do url="${sources[$i]}" domain="${sourceDomains[$i]}" @@ -170,11 +181,8 @@ gravity_Supernova() { esac if [[ "${skipDownload}" == false ]]; then - str="Target: ${domain} (${url##*/})" - echo -e " ${INFO} ${str}" - - gravity_Pull "${url}" "${cmd_ext}" "${agent}" "${str}" - + echo -e " ${INFO} Target: ${domain} (${url##*/})" + gravity_Pull "${url}" "${cmd_ext}" "${agent}" echo "" fi done @@ -182,19 +190,15 @@ gravity_Supernova() { # Download specified URL and perform checks on HTTP status and file content gravity_Pull() { - local url cmd_ext agent heisenbergCompensator patternBuffer str httpCode success + local url="${1}" cmd_ext="${2}" agent="${3}" heisenbergCompensator="" patternBuffer str httpCode success="" - url="${1}" - cmd_ext="${2}" - agent="${3}" + # Create temp file to store content on disk instead of RAM + patternBuffer=$(mktemp -p "/tmp" --suffix=".phgpb") - # Store downloaded content to temp file instead of RAM - patternBuffer=$(mktemp) - - heisenbergCompensator="" + # Determine if $saveLocation has read permission if [[ -r "${saveLocation}" ]]; then - # Make curl determine if a remote file has been modified since last retrieval - # Uses "Last-Modified" header, which certain web servers do not provide (e.g: raw github links) + # Have curl determine if a remote file has been modified since last retrieval + # Uses "Last-Modified" header, which certain web servers do not provide (e.g: raw github urls) heisenbergCompensator="-z ${saveLocation}" fi @@ -205,21 +209,21 @@ gravity_Pull() { # Determine "Status:" output based on HTTP response case "${httpCode}" in - "200") echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success="true";; - "304") echo -e "${OVER} ${TICK} ${str} No changes detected"; success="true";; - "000") echo -e "${OVER} ${CROSS} ${str} Connection Refused"; success="false";; - "403") echo -e "${OVER} ${CROSS} ${str} Forbidden"; success="false";; - "404") echo -e "${OVER} ${CROSS} ${str} Not found"; success="false";; - "408") echo -e "${OVER} ${CROSS} ${str} Time-out"; success="false";; - "451") echo -e "${OVER} ${CROSS} ${str} Unavailable For Legal Reasons"; success="false";; - "521") echo -e "${OVER} ${CROSS} ${str} Web Server Is Down (Cloudflare)"; success="false";; - "522") echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Cloudflare)"; success="false";; - "500") echo -e "${OVER} ${CROSS} ${str} Internal Server Error"; success="false";; - * ) echo -e "${OVER} ${CROSS} ${str} ${httpCode}"; success="false";; + "200") echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true;; + "304") echo -e "${OVER} ${TICK} ${str} No changes detected"; success=true;; + "000") echo -e "${OVER} ${CROSS} ${str} Connection Refused";; + "403") echo -e "${OVER} ${CROSS} ${str} Forbidden";; + "404") echo -e "${OVER} ${CROSS} ${str} Not found";; + "408") echo -e "${OVER} ${CROSS} ${str} Time-out";; + "451") echo -e "${OVER} ${CROSS} ${str} Unavailable For Legal Reasons";; + "500") echo -e "${OVER} ${CROSS} ${str} Internal Server Error";; + "521") echo -e "${OVER} ${CROSS} ${str} Web Server Is Down (Cloudflare)";; + "522") echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Cloudflare)";; + * ) echo -e "${OVER} ${CROSS} ${str} ${httpCode}";; esac # Determine if the blocklist was downloaded and saved correctly - if [[ "${success}" == "true" ]]; then + if [[ "${success}" == true ]]; then if [[ "${httpCode}" == "304" ]]; then : # Do not attempt to re-parse file # Check if $patternbuffer is a non-zero length file @@ -231,46 +235,40 @@ gravity_Pull() { echo -e " ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" fi else - # Determine if cached list exists + # Determine if cached list has read permission if [[ -r "${saveLocation}" ]]; then echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" else echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" fi fi - - # Delete temp file if it has not been moved - if [[ -f "${patternBuffer}" ]]; then - rm "${patternBuffer}" 2> /dev/null || \ - echo -e " ${CROSS} Unable to remove ${patternBuffer}" - fi } -# Parse non-standard source files into domains-only format +# Parse source files into domains format gravity_ParseFileIntoDomains() { local source="${1}" destination="${2}" commentPattern firstLine abpFilter - # Determine how to parse source file + # Determine if we are parsing a consolidated list if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then - # Consolidated list parsing: Remove comments and hosts IP's - # Define symbols used as comments: #;@![/ commentPattern="[#;@![\\/]" - # Awk Logic: Process lines which do not begin with comments + # Parse Domains/Hosts files by removing comments & host IPs + # Logic: Ignore lines which begin with comments awk '!/^'"${commentPattern}"'/ { - # If there are multiple words seperated by space - if (NF>1) { + # Determine if there are multiple words seperated by a space + if(NF>1) { # Remove comments (including prefixed spaces/tabs) - if ($0 ~ /'"${commentPattern}"'/) { gsub("( |\t)'"${commentPattern}"'.*", "", $0) } - # Print consecutive domains - if ($3) { + if($0 ~ /'"${commentPattern}"'/) { gsub("( |\t)'"${commentPattern}"'.*", "", $0) } + # Determine if there are aliased domains + if($3) { + # Remove IP address $1="" # Remove space which is left in $0 when removing $1 gsub("^ ", "", $0) print $0 - # Print single domain - } else if ($2) { + } else if($2) { + # Print single domain without IP print $2 } # If there are no words seperated by space @@ -278,59 +276,60 @@ gravity_ParseFileIntoDomains() { print $1 } }' "${source}" 2> /dev/null > "${destination}" - else - # Individual file parsing: Keep comments, while parsing domains from each line - read -r firstLine < "${source}" + return 0 + fi - # Determine how to parse individual source file formats - # Lists may not capitalise the first line correctly, so compare strings against lower case - if [[ "${firstLine,,}" =~ (adblock|ublock|!checksum) ]]; then - # Awk Logic: Parse Adblock domains & comments: https://adblockplus.org/filter-cheatsheet - abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/" - awk ''"${abpFilter}"' { - # Remove valid adblock type options - gsub(/~?(important|third-party|popup|subdocument|websocket),?/, "", $0) - # Remove starting domain name anchor "||" and ending seperator "^$" ($ optional) - gsub(/(\|\||\^\$?$)/, "", $0) - # Remove lines which are only IPv4 addresses or contain "^/*" - if ($0 ~ /(^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$|[\\^\/\*])/) { $0="" } - if ($0) { print $0 } - }' "${source}" 2> /dev/null > "${destination}" - # Parse URL list if source file contains http:// or IPv4 - elif grep -q -E "^(https?://|([0-9]{1,3}\\.){3}[0-9]{1,3}$)" "${source}" &> /dev/null; then - awk '{ - # Remove URL protocol, optional "username:password@", and ":?/;" - if ($0 ~ /[:?\/;]/) { gsub(/(^.*:\/\/(.*:.*@)?|[:?\/;].*)/, "", $0) } - # Remove lines which are only IPv4 addresses - if ($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" } - if ($0) { print $0 } - }' "${source}" 2> /dev/null > "${destination}" + # Individual file parsing: Keep comments, while parsing domains from each line + # We keep comments to respect the list maintainer's licensing + read -r firstLine < "${source}" + + # Determine how to parse individual source file formats + if [[ "${firstLine,,}" =~ (adblock|ublock|^!) ]]; then + # Compare $firstLine against lower case words found in Adblock lists + + # Define symbols used as comments: [! + # "||.*^" includes the "Example 2" domains we can extract + # https://adblockplus.org/filter-cheatsheet + abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/" + + # Parse Adblock lists by extracting "Example 2" domains + # Logic: Ignore lines which do not include comments or domain name anchor + awk ''"${abpFilter}"' { + # Remove valid adblock type options + gsub(/~?(important|third-party|popup|subdocument|websocket),?/, "", $0) + # Remove starting domain name anchor "||" and ending seperator "^$" ($ optional) + gsub(/(\|\||\^\$?$)/, "", $0) + # Remove lines which are only IPv4 addresses or contain "^/*" + if($0 ~ /(^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$|[\\^\/\*])/) { $0="" } + if($0) { print $0 } + }' "${source}" 2> /dev/null > "${destination}" + elif grep -q "^address=/" "${source}" &> /dev/null; then # Parse Dnsmasq format lists - elif grep -q "^address=/" "${source}" &> /dev/null; then - awk -F/ '{ - # Print comments - if ($0 ~ "#") { - print $0 - # Print domains - } else if ($2) { - print $2 - } - }' "${source}" 2> /dev/null > "${destination}" - else - # Keep hosts/domains file in same format as it was downloaded - output=$( { mv "${source}" "${destination}"; } 2>&1 ) - status="$?" + echo -e " ${CROSS} ${COL_BOLD}dnsmasq${COL_NC} format lists are not supported" + elif grep -q -E "^(https?://|www\\.)" "${source}" &> /dev/null; then + # Parse URL list if source file contains "http://" or "www." + # Scanning for "^IPv4$" is too slow with large (1M) lists on low-end hardware - if [[ "${status}" -ne 0 ]]; then - echo -e " ${CROSS} Unable to move tmp file to ${piholeDir} - ${output}" - gravity_Cleanup "error" - fi + awk '{ + # Remove URL protocol, optional "username:password@", and ":?/;" + if ($0 ~ /[:?\/;]/) { gsub(/(^.*:\/\/(.*:.*@)?|[:?\/;].*)/, "", $0) } + # Remove lines which are only IPv4 addresses + if ($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" } + if ($0) { print $0 } + }' "${source}" 2> /dev/null > "${destination}" + else + # Default: Keep hosts/domains file in same format as it was downloaded + output=$( { mv "${source}" "${destination}"; } 2>&1 ) + + if [[ ! -e "${destination}" ]]; then + echo -e " ${CROSS} Unable to move tmp file to ${piholeDir} + ${output}" + gravity_Cleanup "error" fi fi } -# Create unfiltered "Matter and Light" consolidated list +# Create (unfiltered) "Matter and Light" consolidated list gravity_Schwarzschild() { local str lastLine @@ -340,22 +339,25 @@ gravity_Schwarzschild() { # Empty $matterAndLight if it already exists, otherwise, create it : > "${piholeDir}/${matterAndLight}" + # Loop through each *.domains file for i in "${activeDomains[@]}"; do - # Only assimilate list if it is available (download might have failed permanently) + # Determine if file has read permissions, as download might have failed if [[ -r "${i}" ]]; then - # Compile all blacklisted domains into one file and remove CRs + # Remove windows CRs from file, and append into $matterAndLight tr -d '\r' < "${i}" >> "${piholeDir}/${matterAndLight}" - # Ensure each source blocklist has a final newline + # Ensure that the first line of a new list is on a new line lastLine=$(tail -1 "${piholeDir}/${matterAndLight}") - [[ "${#lastLine}" -gt 0 ]] && echo "" >> "${piholeDir}/${matterAndLight}" + if [[ "${#lastLine}" -gt 0 ]]; then + echo "" >> "${piholeDir}/${matterAndLight}" + fi fi done echo -e "${OVER} ${TICK} ${str}" } -# Parse unfiltered consolidated blocklist into filtered domains-only format +# Parse consolidated list into (filtered, unique) domains-only format gravity_Filter() { local str num @@ -365,77 +367,67 @@ gravity_Filter() { # Parse into hosts file gravity_ParseFileIntoDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${parsedMatter}" - # Format file line count as currency + # Format $parsedMatter line total as currency num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${parsedMatter}")") echo -e "${OVER} ${TICK} ${str} ${INFO} ${COL_LIGHT_BLUE}${num}${COL_NC} domains being pulled in by gravity" - gravity_Unique -} - -# Sort and remove duplicate blacklisted domains -gravity_Unique() { - local str num - str="Removing duplicate domains" echo -ne " ${INFO} ${str}..." sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}" echo -e "${OVER} ${TICK} ${str}" - # Format file line count as currency + # Format $preEventHorizon line total as currency num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") echo -e " ${INFO} ${COL_LIGHT_BLUE}${num}${COL_NC} unique domains trapped in the Event Horizon" } -# Whitelist blocklist domain sources +# Whitelist unique blocklist domain sources gravity_WhitelistBLD() { - local plural str uniqDomains + local plural="" str uniqDomains echo "" - plural=; [[ "${#sources[*]}" != "1" ]] && plural="s" - str="Adding blocklist source${plural} to the whitelist" + [[ "${#sources[@]}" -ne 1 ]] && plural="s" + str="Adding blocklist domain source${plural} to the whitelist" echo -ne " ${INFO} ${str}..." # Create array of unique $sourceDomains mapfile -t uniqDomains <<< "$(awk '{ if(!a[$1]++) { print $1 } }' <<< "$(printf '%s\n' "${sourceDomains[@]}")")" - ${WHITELIST_COMMAND} -nr -q "${uniqDomains[*]}" > /dev/null + # Whitelist $uniqDomains + "${PIHOLE_COMMAND}" -w -nr -q "${uniqDomains[*]}" &> /dev/null echo -e "${OVER} ${TICK} ${str}" } # Whitelist user-defined domains gravity_Whitelist() { - local plural str num + local num plural="" str - # Test existence of whitelist.txt - if [[ -f "${whitelistFile}" ]]; then - # Remove anything in whitelist.txt from the Event Horizon - num=$(wc -l < "${whitelistFile}") - plural=; [[ "${num}" -ne 1 ]] && plural="s" - str="Whitelisting ${num} domain${plural}" - echo -ne " ${INFO} ${str}..." - - # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in whitelist.txt - grep -F -x -v -f "${whitelistFile}" "${piholeDir}/${preEventHorizon}" > "${piholeDir}/${whitelistMatter}" - - echo -e "${OVER} ${TICK} ${str}" - else + if [[ ! -f "${whitelistFile}" ]]; then echo -e " ${INFO} Nothing to whitelist!" + return 0 fi + + num=$(wc -l < "${whitelistFile}") + [[ "${num}" -ne 1 ]] && plural="s" + str="Whitelisting ${num} domain${plural}" + echo -ne " ${INFO} ${str}..." + + # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in $whitelistFile + grep -F -x -v -f "${whitelistFile}" "${piholeDir}/${preEventHorizon}" > "${piholeDir}/${whitelistMatter}" + + echo -e "${OVER} ${TICK} ${str}" } # Output count of blacklisted domains and wildcards gravity_ShowBlockCount() { - local num plural str + local num plural if [[ -f "${blacklistFile}" ]]; then num=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")") plural=; [[ "${num}" -ne 1 ]] && plural="s" - str="Exact blocked domain${plural}: ${num}" - echo -e " ${INFO} ${str}" - else - echo -e " ${INFO} Nothing to blacklist!" + echo -e " ${INFO} Blacklisted ${num} domain${plural}" fi if [[ -f "${wildcardFile}" ]]; then @@ -445,47 +437,47 @@ gravity_ShowBlockCount() { num=$(( num/2 )) fi plural=; [[ "${num}" -ne 1 ]] && plural="s" - echo -e " ${INFO} Wildcard blocked domain${plural}: ${num}" - else - echo -e " ${INFO} No wildcards used!" + echo -e " ${INFO} Wildcard blocked ${num} domain${plural}" fi } # Parse list of domains into hosts format gravity_ParseDomainsIntoHosts() { - if [[ -n "${IPV4_ADDRESS}" ]] || [[ -n "${IPV6_ADDRESS}" ]]; then - # Awk Logic: Remove CR line endings and print IP before domain if IPv4/6 is used - awk -v ipv4addr="$IPV4_ADDRESS" -v ipv6addr="$IPV6_ADDRESS" '{ - sub(/\r$/, "") - if(ipv4addr) { print ipv4addr" "$0; } - if(ipv6addr) { print ipv6addr" "$0; } - }' >> "${2}" < "${1}" - else - echo -e "${OVER} ${CROSS} ${str}" - echo -e " ${COL_LIGHT_RED}No IP addresses found! Please run 'pihole -r' to reconfigure${COL_NC}\\n" - gravity_Cleanup "error" - fi + awk -v ipv4="$IPV4_ADDRESS" -v ipv6="$IPV6_ADDRESS" '{ + # Remove windows CR line endings + sub(/\r$/, "") + # Parse each line as "ipaddr domain" + if(ipv6 && ipv4) { + print ipv4" "$0"\n"ipv6" "$0 + } else if(!ipv6) { + print ipv4" "$0 + } else { + print ipv6" "$0 + } + }' >> "${2}" < "${1}" } # Create "localhost" entries into hosts format gravity_ParseLocalDomains() { local hostname - if [[ -f "/etc/hostname" ]]; then + if [[ -s "/etc/hostname" ]]; then hostname=$(< "/etc/hostname") elif command -v hostname &> /dev/null; then hostname=$(hostname -f) else echo -e " ${CROSS} Unable to determine fully qualified domain name of host" + return 0 fi echo -e "${hostname}\\npi.hole" > "${localList}.tmp" - # Copy the file over as /etc/pihole/local.list so dnsmasq can use it - rm "${localList}" 2> /dev/null || true - gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}" - rm "${localList}.tmp" 2> /dev/null || true - # Add additional local hosts provided by OpenVPN (if available) + # Empty $localList if it already exists, otherwise, create it + : > "${localList}" + + gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}" + + # Add additional LAN hosts provided by OpenVPN (if available) if [[ -f "${VPNList}" ]]; then awk -F, '{printf $2"\t"$1"\n"}' "${VPNList}" >> "${localList}" fi @@ -512,12 +504,14 @@ gravity_ParseBlacklistDomains() { # Create user-added blacklist entries gravity_ParseUserDomains() { - if [[ -f "${blacklistFile}" ]]; then - gravity_ParseDomainsIntoHosts "${blacklistFile}" "${blackList}.tmp" - # Copy the file over as /etc/pihole/black.list so dnsmasq can use it - mv "${blackList}.tmp" "${blackList}" 2> /dev/null || \ - echo -e " ${CROSS} Unable to move ${blackList##*/}.tmp to ${piholeDir}" + if [[ ! -f "${blacklistFile}" ]]; then + return 0 fi + + gravity_ParseDomainsIntoHosts "${blacklistFile}" "${blackList}.tmp" + # Copy the file over as /etc/pihole/black.list so dnsmasq can use it + mv "${blackList}.tmp" "${blackList}" 2> /dev/null || \ + echo -e " ${CROSS} Unable to move ${blackList##*/}.tmp to ${piholeDir}" } # Trap Ctrl-C @@ -525,15 +519,17 @@ gravity_Trap() { trap '{ echo -e "\\n\\n ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT } -# Clean up after Gravity +# Clean up after Gravity upon exit or cancellation gravity_Cleanup() { local error="${1:-}" str="Cleaning up stray matter" echo -ne " ${INFO} ${str}..." + # Delete tmp content generated by Gravity rm ${piholeDir}/pihole.*.txt 2> /dev/null rm ${piholeDir}/*.tmp 2> /dev/null + rm /tmp/*.phgpb 2> /dev/null # Remove any unused .domains files for file in ${piholeDir}/*.${domainsExtension}; do @@ -576,16 +572,14 @@ for var in "$@"; do "-sd" | "--skip-download" ) skipDownload=true;; "-b" | "--blacklist-only" ) listType="blacklist";; "-w" | "--whitelist-only" ) listType="whitelist";; - "-wild" | "--wildcard-only" ) listType="wildcard";; + "-wild" | "--wildcard-only" ) listType="wildcard"; dnsRestartType="restart";; esac done +# Trap Ctrl-C gravity_Trap -# Ensure dnsmasq is restarted when modifying wildcards -[[ "${listType}" == "wildcard" ]] && dnsRestart="restart" - -if [[ "${forceDelete}" == true ]]; then +if [[ "${forceDelete:-}" == true ]]; then str="Deleting exising list cache" echo -ne "${INFO} ${str}..." @@ -628,7 +622,7 @@ if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "blacklist" ]]; then gravity_ParseUserDomains # Perform when downloading blocklists - if [[ ! "${listType}" == "blacklist" ]]; then + if [[ ! "${listType:-}" == "blacklist" ]]; then gravity_ParseLocalDomains gravity_ParseBlacklistDomains fi @@ -646,6 +640,6 @@ echo "" # Determine if DNS has been restarted by this instance of gravity if [[ -z "${dnsWasOffline:-}" ]]; then # Use "force-reload" when restarting dnsmasq for everything but Wildcards - "${PIHOLE_COMMAND}" restartdns "${dnsRestart:-force-reload}" + "${PIHOLE_COMMAND}" restartdns "${dnsRestartType:-force-reload}" fi "${PIHOLE_COMMAND}" status From c2d3e99ddd7a3fbe65d2cfc2ebb8e5ad917b78de Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Mon, 18 Sep 2017 00:41:26 +1000 Subject: [PATCH 22/50] Slow query fix & queryFunc optimisation * Validate modified code using Shellcheck & Strict Bash * Added and/or elaborated more comments * scanlist() should exit if /etc/pihole is not available * Add `export LC_CTYPE=C` to prevent extreme grep slowdown * Consider "domain.com#comment" an exact match * Add specialised wildcard searching grep * Optimise and simplify queryFunc() for readability * Replace IDN `python` parsing with `idn2`, as `python` is not guaranteed to be available * Use ${COL_BOLD} when printing filenames --- pihole | 315 +++++++++++++++++++++++++++------------------------------ 1 file changed, 148 insertions(+), 167 deletions(-) diff --git a/pihole b/pihole index ded79f71..63de5a7c 100755 --- a/pihole +++ b/pihole @@ -84,21 +84,27 @@ updateGravityFunc() { exit 0 } +# Scan an array of files for matching strings scanList(){ - domain="${1}" - list="${2}" - method="${3}" + local domain="${1}" lists="${2}" type="${3:-}" - # Switch folder, preventing grep from printing file path - cd "/etc/pihole" || return 1 + # Prevent grep from printing file path + cd "/etc/pihole" || exit 1 - if [[ -n "${method}" ]]; then - grep -i -E -l "(^|\s|\/)${domain}($|\s|\/)" ${list} /dev/null 2> /dev/null - else - grep -i "${domain}" ${list} /dev/null 2> /dev/null - fi + # Prevent grep -i matching slowly: http://bit.ly/2xFXtUX + export LC_CTYPE=C + + # /dev/null forces filename to be printed when only one list has been generated + # shellcheck disable=SC2086 + case "${type}" in + "exact" ) grep -i -E -l "(^|\\s)${domain}($|\\s|#)" ${lists} /dev/null;; + "wc" ) grep -i -o -m 1 "/${domain}/" ${lists};; + * ) grep -i "${domain}" ${lists} /dev/null;; + esac } +# Print each subdomain +# e.g: foo.bar.baz.com = "foo.bar.baz.com bar.baz.com baz.com com" processWildcards() { IFS="." read -r -a array <<< "${1}" for (( i=${#array[@]}-1; i>=0; i-- )); do @@ -115,8 +121,8 @@ processWildcards() { } queryFunc() { - options="$*" - options="${options/-q /}" + shift + local options="$*" adlist="" all="" exact="" blockpage="" matchType="match" if [[ "${options}" == "-h" ]] || [[ "${options}" == "--help" ]]; then echo "Usage: pihole -q [option] @@ -131,201 +137,176 @@ Options: exit 0 fi - if [[ "${options}" == *"-exact"* ]]; then - method="exact" - exact=true - fi - - if [[ "${options}" == *"-adlist"* ]]; then - adlist=true - fi - - if [[ "${options}" == *"-bp"* ]]; then - method="exact" - blockpage=true - fi - - if [[ "${options}" == *"-all"* ]]; then - all=true - fi - - # Strip valid options, leaving only the domain and invalid options - options=$(sed 's/ \?-\(exact\|adlist\(s\)\?\|bp\|all\) \?//g' <<< "$options") - - # Handle errors - if [[ "${options}" == *" "* ]]; then - error=true - str="Unknown option specified" - elif [[ "${options}" == "-q" ]]; then - error=true - str="No domain specified" - fi - - if [[ -n "${error}" ]]; then - echo -e " ${COL_LIGHT_RED}${str}${COL_NC} - Try 'pihole -q --help' for more information." + if [[ ! -e "/etc/pihole/adlists.list" ]]; then + echo -e "${COL_LIGHT_RED}The file '/etc/pihole/adlists.list' was not found${COL_NC}" exit 1 fi - # If domain contains non ASCII characters, convert domain to punycode if python is available - # Cr: https://serverfault.com/a/335079 - if [[ "$options" = *[![:ascii:]]* ]]; then - if command -v python &> /dev/null; then - query=$(python -c 'import sys;print sys.argv[1].decode("utf-8").encode("idna")' "${options}") - fi + # Handle valid options + if [[ "${options}" == *"-bp"* ]]; then + exact="exact"; blockpage=true else - query="${options}" + [[ "${options}" == *"-adlist"* ]] && adlist=true + [[ "${options}" == *"-all"* ]] && all=true + if [[ "${options}" == *"-exact"* ]]; then + exact="exact"; matchType="exact ${matchType}" + fi + fi + + # Strip valid options, leaving only the domain and invalid options + # This allows users to place the options before or after the domain + options=$(sed -E 's/ ?-(bp|adlists?|all|exact)//g' <<< "${options}") + + # Handle remaining options + # If $options contain non ASCII characters, convert to punycode + case "${options}" in + "" ) str="No domain specified";; + *$'\n'* ) str="Unknown query option specified";; + *[![:ascii:]]* ) domainQuery=$(idn2 "${options}");; + * ) domainQuery="${options}";; + esac + + if [[ -n "${str:-}" ]]; then + echo -e "${str}${COL_NC}\\nTry 'pihole -q --help' for more information." + exit 1 fi # Scan Whitelist and Blacklist lists="whitelist.txt blacklist.txt" - results=($(scanList "${query}" "${lists}" "${method}")) + mapfile -t results <<< "$(scanList "${domainQuery}" "${lists}" "${exact}")" if [[ -n "${results[*]}" ]]; then - blResult=true - # Loop through each scanList line to print appropriate title + wbMatch=true + + # Loop through each result in order to print unique file title once for result in "${results[@]}"; do - filename="${result/:*/}" - if [[ -n "$exact" ]]; then - printf " Exact result in %s\n" "${filename}" - elif [[ -n "$blockpage" ]]; then - printf "π %s\n" "${filename}" + fileName="${result%%.*}" + + if [[ -n "${blockpage}" ]]; then + echo "π ${fileName}" + exit 0 + elif [[ -n "${exact}" ]]; then + echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" else - domain="${result/*:/}" - if [[ ! "${filename}" == "${filename_prev:-}" ]]; then - printf " Result from %s\n" "${filename}" + # Only print filename title once per file + if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then + echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" + fileName_prev="${fileName}" fi - printf " %s\n" "${domain}" - filename_prev="${filename}" + echo " ${result#*:}" fi done fi # Scan Wildcards if [[ -e "${wildcardlist}" ]]; then - wildcards=($(processWildcards "${query}")) + # Determine all subdomains, domain and TLDs + mapfile -t wildcards <<< "$(processWildcards "${domainQuery}")" for match in "${wildcards[@]}"; do - results=($(scanList "\/${match}\/" ${wildcardlist})) + # Search wildcard list for matches + mapfile -t results <<< "$(scanList "${match}" "${wildcardlist}" "wc")" if [[ -n "${results[*]}" ]]; then - # Remove empty lines before couting number of results - count=$(sed '/^\s*$/d' <<< "${results[@]}" | wc -l) - if [[ "${count}" -ge 0 ]]; then - blResult=true - if [[ -z "${blockpage}" ]]; then - printf " Wildcard result in %s\n" "${wildcardlist/*dnsmasq.d\/}" - fi - - if [[ -n "${blockpage}" ]]; then - echo "π ${wildcardlist/*\/}" - else - echo " *.${match}" - fi + if [[ -z "${wcMatch:-}" ]] && [[ -z "${blockpage}" ]]; then + wcMatch=true + echo " ${matchType^} found in ${COL_BOLD}Wildcards${COL_NC}:" fi + + case "${blockpage}" in + true ) echo "π ${wildcardlist##*/}"; exit 0;; + * ) echo " *.${match}";; + esac fi done - - [[ -n "${blResult}" ]] && [[ -n "${blockpage}" ]] && exit 0 fi - # Glob *.domains file names, remove file paths and sort by list number - lists_raw=(/etc/pihole/*.domains) - IFS_OLD=$IFS - IFS=$'\n' - lists=$(sort -t . -k 2 -g <<< "${lists_raw[*]//\/etc\/pihole\//}") + # Get version sorted *.domains filenames (without dir path) + lists=("$(cd "/etc/pihole" || exit 0; printf "%s\\n" -- *.domains | sort -V)") - # Scan Domains files - results=($(scanList "${query}" "${lists}" "${method}")) + # Query blocklists for occurences of domain + mapfile -t results <<< "$(scanList "${domainQuery}" "${lists[*]}" "${exact}")" # Handle notices - if [[ -z "${blResult}" ]] && [[ -z "${results[*]}" ]]; then - notice=true - str="No ${method/t/t }results found for ${query} found within block lists" - elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 16000 ]]; then - # 16000 chars is 15 chars X 1000 lines worth of results - notice=true - str="Hundreds of ${method/t/t }results found for ${query} - This can be overriden using the -all option" + if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then + echo -e " ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} found within block lists" + exit 0 + elif [[ -z "${results[*]}" ]]; then + # Result found in WL/BL/Wildcards + exit 0 + elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 100 ]]; then + echo -e " ${INFO} Over 100 ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} + This can be overridden using the -all option" + exit 0 fi - if [[ -n "${notice}" ]]; then - echo -e " ${INFO} ${str}" - exit + # Remove unwanted content from non-exact $results + if [[ -z "${exact}" ]]; then + # Delete lines starting with # + # Remove comments after domain + # Remove hosts format IP address + mapfile -t results <<< "$(IFS=$'\n'; sed \ + -e "/:#/d" \ + -e "s/[ \\t]#.*//g" \ + -e "s/:.*[ \\t]/:/g" \ + <<< "${results[*]}")" + + # Exit if result was in a comment + [[ -z "${results[*]}" ]] && exit 0 fi - # Remove unwanted content from results - if [[ -z "${method}" ]]; then - results=($(sed "/:#/d" <<< "${results[*]}")) # Lines starting with comments - results=($(sed "s/[ \t]#.*//g" <<< "${results[*]}")) # Comments after domain - results=($(sed "s/:.*[ \t]/:/g" <<< "${results[*]}")) # IP address - fi - IFS=$IFS_OLD - - # Get adlist content as array + # Get adlist file content as array if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then - if [[ -f "/etc/pihole/adlists.list" ]]; then - for url in $(< /etc/pihole/adlists.list); do - if [[ "${url:0:4}" == "http" ]] || [[ "${url:0:3}" == "www" ]]; then - adlists+=("$url") - fi - done - else - echo -e " ${COL_LIGHT_RED}The file '/etc/pihole/adlists.list' was not found${COL_NC}" - exit 1 - fi - fi - - if [[ -n "${results[*]}" ]]; then - if [[ -n "${exact}" ]]; then - echo " Exact result(s) for ${query} found in:" - fi - - for result in "${results[@]}"; do - filename="${result/:*/}" - - # Convert file name to URL name for -adlist or -bp options - if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then - filenum=("${filename/list./}") - filenum=("${filenum/.*/}") - filename="${adlists[$filenum]}" - - # If gravity has generated associated .domains files - # but adlists.list has been modified since - if [[ -z "${filename}" ]]; then - filename="${COL_LIGHT_RED}Error: no associated adlists URL found${COL_NC}" - fi - fi - - if [[ -n "${exact}" ]]; then - printf " %s\n" "${filename}" - elif [[ -n "${blockpage}" ]]; then - printf "%s %s\n" "${filenum}" "${filename}" - else # Standard query output - - # Print filename heading once per file, not for every match - if [[ ! "${filename}" == "${filename_prev:-}" ]]; then - unset count - printf " Result from %s\n" "${filename}" - else - let count++ - fi - - # Print matching domain if $max_count has not been reached - [[ -z "${all}" ]] && max_count="20" - if [[ -z "${all}" ]] && [[ "${count}" -eq "${max_count}" ]]; then - echo " Over $count results found, skipping rest of file" - elif [[ -z "${all}" ]] && [[ "${count}" -gt "${max_count}" ]]; then - continue - else - domain="${result/*:/}" - printf " %s\n" "${domain}" - fi - filename_prev="${filename}" + for adlistUrl in $(< "/etc/pihole/adlists.list"); do + if [[ "${adlistUrl:0:4}" =~ (http|www.) ]]; then + adlists+=("${adlistUrl}") fi done fi + # Print "Exact matches for" title + if [[ -n "${exact}" ]] && [[ -z "${blockpage}" ]]; then + plural=""; [[ "${#results[*]}" -gt 1 ]] && plural="es" + echo " ${matchType^}${plural} for ${COL_BOLD}${domainQuery}${COL_NC} found in:" + fi + + for result in "${results[@]}"; do + fileName="${result/:*/}" + + # Determine *.domains URL using filename's number + if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then + fileNum="${fileName/list./}"; fileNum="${fileNum%%.*}" + fileName="${adlists[$fileNum]}" + + # Discrepency occurs when adlists has been modified, but Gravity has not been run + if [[ -z "${fileName}" ]]; then + fileName="${COL_LIGHT_RED}(no associated adlists URL found)${COL_NC}" + fi + fi + + if [[ -n "${blockpage}" ]]; then + echo "${fileNum} ${fileName}" + elif [[ -n "${exact}" ]]; then + echo " ${fileName}" + else + if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then + count="" + echo " ${matchType^} found in ${COL_BOLD}${fileName}${COL_NC}:" + fileName_prev="${fileName}" + fi + : $((count++)) + + # Print matching domain if $max_count has not been reached + [[ -z "${all}" ]] && max_count="50" + if [[ -z "${all}" ]] && [[ "${count}" -ge "${max_count}" ]]; then + [[ "${count}" -gt "${max_count}" ]] && continue + echo " ${COL_GRAY}Over ${count} results found, skipping rest of file${COL_NC}" + else + echo " ${result#*:}" + fi + fi + done + exit 0 } From d56beedd7a4c8482e23881fb9152e2339058d062 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Mon, 18 Sep 2017 00:41:53 +1000 Subject: [PATCH 23/50] Add idn2 dependency --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 1786813d..61d40ada 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -164,7 +164,7 @@ if command -v apt-get &> /dev/null; then # These programs are stored in an array so they can be looped through later INSTALLER_DEPS=(apt-utils dialog debconf dhcpcd5 git ${iproute_pkg} whiptail) # Pi-hole itself has several dependencies that also need to be installed - PIHOLE_DEPS=(bc cron curl dnsmasq dnsutils iputils-ping lsof netcat sudo unzip wget) + PIHOLE_DEPS=(bc cron curl dnsmasq dnsutils iputils-ping lsof netcat sudo unzip wget idn2) # The Web dashboard has some that also need to be installed # It's useful to separate the two since our repos are also setup as "Core" code and "Web" code PIHOLE_WEB_DEPS=(lighttpd ${phpVer}-common ${phpVer}-cgi ${phpVer}-${phpSqlite}) @@ -208,7 +208,7 @@ elif command -v rpm &> /dev/null; then PKG_INSTALL=(${PKG_MANAGER} install -y) PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l" INSTALLER_DEPS=(dialog git iproute net-tools newt procps-ng) - PIHOLE_DEPS=(bc bind-utils cronie curl dnsmasq findutils nmap-ncat sudo unzip wget) + PIHOLE_DEPS=(bc bind-utils cronie curl dnsmasq findutils nmap-ncat sudo unzip wget idn2) PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php php-common php-cli php-pdo) if ! grep -q 'Fedora' /etc/redhat-release; then INSTALLER_DEPS=("${INSTALLER_DEPS[@]}" "epel-release"); From 54a85d3a6336b1c74cf084d75a6b67c806d53dd3 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Mon, 18 Sep 2017 00:58:56 +1000 Subject: [PATCH 24/50] Fix invalid option case * Ensure `pihole -q doubleclick.com asd` exits with error --- pihole | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pihole b/pihole index 63de5a7c..999ab7c1 100755 --- a/pihole +++ b/pihole @@ -161,7 +161,7 @@ Options: # If $options contain non ASCII characters, convert to punycode case "${options}" in "" ) str="No domain specified";; - *$'\n'* ) str="Unknown query option specified";; + *" "* ) str="Unknown query option specified";; *[![:ascii:]]* ) domainQuery=$(idn2 "${options}");; * ) domainQuery="${options}";; esac From 56990618e9af28ccb75cd99fd46302745f079870 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Mon, 18 Sep 2017 01:12:19 +1000 Subject: [PATCH 25/50] Prevent Block Page issue * Block Page expects to see a full stop, otherwise it will throw an unhandled error --- pihole | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pihole b/pihole index 999ab7c1..a41c9506 100755 --- a/pihole +++ b/pihole @@ -183,7 +183,7 @@ Options: fileName="${result%%.*}" if [[ -n "${blockpage}" ]]; then - echo "π ${fileName}" + echo "π ${result}" exit 0 elif [[ -n "${exact}" ]]; then echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" From d02bf258af2dcabd3de648dcf31d98c62d903083 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Mon, 18 Sep 2017 17:36:03 +1000 Subject: [PATCH 26/50] Improve non-standard list parsing * Add 504 status (Gateway connection timed out) * Add text for non-standard list parsing * Improve adblock parsing * Ensure adblock exception rules are removed from file * Ensure "www." is not treated as a URL-format list * Corrected typo * Ensure script does not fail if "-f" is used when there are no blocklists generated Signed off by WaLLy3K --- gravity.sh | 55 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/gravity.sh b/gravity.sh index c1295852..3bed8cb3 100755 --- a/gravity.sh +++ b/gravity.sh @@ -217,6 +217,7 @@ gravity_Pull() { "408") echo -e "${OVER} ${CROSS} ${str} Time-out";; "451") echo -e "${OVER} ${CROSS} ${str} Unavailable For Legal Reasons";; "500") echo -e "${OVER} ${CROSS} ${str} Internal Server Error";; + "504") echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Gateway)";; "521") echo -e "${OVER} ${CROSS} ${str} Web Server Is Down (Cloudflare)";; "522") echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Cloudflare)";; * ) echo -e "${OVER} ${CROSS} ${str} ${httpCode}";; @@ -286,6 +287,7 @@ gravity_ParseFileIntoDomains() { # Determine how to parse individual source file formats if [[ "${firstLine,,}" =~ (adblock|ublock|^!) ]]; then # Compare $firstLine against lower case words found in Adblock lists + echo -ne " ${INFO} Format: Adblock" # Define symbols used as comments: [! # "||.*^" includes the "Example 2" domains we can extract @@ -296,19 +298,42 @@ gravity_ParseFileIntoDomains() { # Logic: Ignore lines which do not include comments or domain name anchor awk ''"${abpFilter}"' { # Remove valid adblock type options - gsub(/~?(important|third-party|popup|subdocument|websocket),?/, "", $0) - # Remove starting domain name anchor "||" and ending seperator "^$" ($ optional) - gsub(/(\|\||\^\$?$)/, "", $0) - # Remove lines which are only IPv4 addresses or contain "^/*" - if($0 ~ /(^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$|[\\^\/\*])/) { $0="" } + gsub(/\$?~?(important|third-party|popup|subdocument|websocket),?/, "", $0) + # Remove starting domain name anchor "||" and ending seperator "^" + gsub(/^(\|\|)|(\^)/, "", $0) + # Remove invalid characters (*/,=$) + if($0 ~ /[*\/,=\$]/) { $0="" } + # Remove lines which are only IPv4 addresses + if($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" } if($0) { print $0 } - }' "${source}" 2> /dev/null > "${destination}" + }' "${source}" > "${destination}" + + # Determine if there are Adblock exception rules + # https://adblockplus.org/filters + if grep -q "^@@||" "${source}" &> /dev/null; then + # Parse Adblock lists by extracting exception rules + # Logic: Ignore lines which do not include exception format "@@||example.com^" + awk -F "[|^]" '/^@@\|\|.*\^/ { + # Remove valid adblock type options + gsub(/\$?~?(third-party)/, "", $0) + # Remove invalid characters (*/,=$) + if($0 ~ /[*\/,=\$]/) { $0="" } + if($3) { print $3 } + }' "${source}" > "${destination}.exceptionsFile.tmp" + + # Remove exceptions + grep -F -x -v -f "${destination}.exceptionsFile.tmp" "${destination}" > "${source}" + mv "${source}" "${destination}" + fi + + echo -e "${OVER} ${TICK} Format: Adblock" elif grep -q "^address=/" "${source}" &> /dev/null; then # Parse Dnsmasq format lists - echo -e " ${CROSS} ${COL_BOLD}dnsmasq${COL_NC} format lists are not supported" - elif grep -q -E "^(https?://|www\\.)" "${source}" &> /dev/null; then - # Parse URL list if source file contains "http://" or "www." + echo -e " ${CROSS} Format: Dnsmasq (list type not supported)" + elif grep -q -E "^https?://" "${source}" &> /dev/null; then + # Parse URL list if source file contains "http://" or "https://" # Scanning for "^IPv4$" is too slow with large (1M) lists on low-end hardware + echo -ne " ${INFO} Format: URL" awk '{ # Remove URL protocol, optional "username:password@", and ":?/;" @@ -317,6 +342,8 @@ gravity_ParseFileIntoDomains() { if ($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" } if ($0) { print $0 } }' "${source}" 2> /dev/null > "${destination}" + + echo -e "${OVER} ${TICK} Format: URL" else # Default: Keep hosts/domains file in same format as it was downloaded output=$( { mv "${source}" "${destination}"; } 2>&1 ) @@ -580,15 +607,11 @@ done gravity_Trap if [[ "${forceDelete:-}" == true ]]; then - str="Deleting exising list cache" + str="Deleting existing list cache" echo -ne "${INFO} ${str}..." - if rm /etc/pihole/list.* 2> /dev/null; then - echo -e "${OVER} ${TICK} ${str}" - else - echo -e "${OVER} ${CROSS} ${str}" - exit 1 - fi + rm /etc/pihole/list.* 2> /dev/null || true + echo -e "${OVER} ${TICK} ${str}" fi # Determine which functions to run From 500a75525098bda95a22ddc8bb2e7fc71fa36283 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Tue, 19 Sep 2017 01:03:24 +1000 Subject: [PATCH 27/50] Delete superseded file * Functionality is reproduced from within the Block Page --- advanced/index.js | 1 - 1 file changed, 1 deletion(-) delete mode 100644 advanced/index.js diff --git a/advanced/index.js b/advanced/index.js deleted file mode 100644 index 9e153e96..00000000 --- a/advanced/index.js +++ /dev/null @@ -1 +0,0 @@ -var x = "" From 26afe049265681cf06029cd9f1fec0dc8a779391 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Tue, 19 Sep 2017 01:04:04 +1000 Subject: [PATCH 28/50] Remove index.js test --- test/test_automated_install.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_automated_install.py b/test/test_automated_install.py index 3f0bd969..0e961c7f 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -186,7 +186,6 @@ def test_installPiholeWeb_fresh_install_no_errors(Pihole): assert tick_box + ' Installing sudoer file' in installWeb.stdout web_directory = Pihole.run('ls -r /var/www/html/pihole').stdout assert 'index.php' in web_directory - assert 'index.js' in web_directory assert 'blockingpage.css' in web_directory def test_update_package_cache_success_no_errors(Pihole): From 8d8482d60b73c97a513a3fab57cba1b2125d3590 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 19 Sep 2017 19:02:50 +0100 Subject: [PATCH 29/50] `<<<$()` back to `< <()` Signed-off-by: Adam Warner --- gravity.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index 3bed8cb3..bb69c918 100755 --- a/gravity.sh +++ b/gravity.sh @@ -128,7 +128,7 @@ gravity_Collapse() { echo -ne " ${INFO} ${str}..." # Retrieve source URLs from $adListFile - mapfile -t sources <<< $( + mapfile -t sources < <( # Logic: Remove comments (#@;![) awk '!/^[#@;!\[]/ { # Remove windows CR line endings @@ -139,7 +139,7 @@ gravity_Collapse() { ) # Parse source domains from $sources - mapfile -t sourceDomains <<< $( + mapfile -t sourceDomains < <( # Logic: Split by folder/port awk -F '[/:]' '{ # Remove URL protocol & optional username:password@ @@ -291,7 +291,7 @@ gravity_ParseFileIntoDomains() { # Define symbols used as comments: [! # "||.*^" includes the "Example 2" domains we can extract - # https://adblockplus.org/filter-cheatsheet + # https://adblockplus.org/filter-cheatsheet abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/" # Parse Adblock lists by extracting "Example 2" domains From 911596daf869a71966d7a13a93ee8d6ff9ac47ba Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Wed, 20 Sep 2017 17:29:11 +1000 Subject: [PATCH 30/50] Start and enable FTL prior to running Gravity --- automated install/basic-install.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 1786813d..3c82d921 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2064,13 +2064,13 @@ main() { fi fi - # Download and compile the aggregated block list - runGravity - # Enable FTL start_service pihole-FTL enable_service pihole-FTL + # Download and compile the aggregated block list + runGravity + # if [[ "${useUpdateVars}" == false ]]; then displayFinalMessage "${pw}" From a6f9272d4b021dbce152ebfc2ac8150a7a96f9b1 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Wed, 20 Sep 2017 22:25:33 +1000 Subject: [PATCH 31/50] Fix gravity from only parsing one adlist URL * Redirect `grep` correctly to $sources (instead of using `awk`) * Redirect $sourceDomains correctly * Replace use of ${COL_LIGHT_BLUE} * Add numeric count informing user of unique source domains being whitelisted --- gravity.sh | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/gravity.sh b/gravity.sh index bb69c918..cb717aba 100755 --- a/gravity.sh +++ b/gravity.sh @@ -128,25 +128,18 @@ gravity_Collapse() { echo -ne " ${INFO} ${str}..." # Retrieve source URLs from $adListFile - mapfile -t sources < <( - # Logic: Remove comments (#@;![) - awk '!/^[#@;!\[]/ { - # Remove windows CR line endings - gsub(/\r$/, "", $0) - # Print non-empty line - if ($1) { print $1 } - }' "${adListFile}" 2> /dev/null - ) + # Logic: Remove comments and empty lines + mapfile -t sources <<< "$(grep -v -E "^(#|$)" "${adListFile}" 2> /dev/null)" # Parse source domains from $sources - mapfile -t sourceDomains < <( + mapfile -t sourceDomains <<< "$( # Logic: Split by folder/port awk -F '[/:]' '{ # Remove URL protocol & optional username:password@ gsub(/(.*:\/\/|.*:.*@)/, "", $0) print $1 }' <<< "$(printf '%s\n' "${sources[@]}")" 2> /dev/null - ) + )" if [[ -n "${sources[*]}" ]] && [[ -n "${sourceDomains[*]}" ]]; then echo -e "${OVER} ${TICK} ${str}" @@ -397,7 +390,7 @@ gravity_Filter() { # Format $parsedMatter line total as currency num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${parsedMatter}")") echo -e "${OVER} ${TICK} ${str} - ${INFO} ${COL_LIGHT_BLUE}${num}${COL_NC} domains being pulled in by gravity" + ${INFO} ${COL_BLUE}${num}${COL_NC} domains being pulled in by gravity" str="Removing duplicate domains" echo -ne " ${INFO} ${str}..." @@ -406,20 +399,21 @@ gravity_Filter() { # Format $preEventHorizon line total as currency num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") - echo -e " ${INFO} ${COL_LIGHT_BLUE}${num}${COL_NC} unique domains trapped in the Event Horizon" + echo -e " ${INFO} ${COL_BLUE}${num}${COL_NC} unique domains trapped in the Event Horizon" } # Whitelist unique blocklist domain sources gravity_WhitelistBLD() { - local plural="" str uniqDomains + local uniqDomains plural="" str echo "" - [[ "${#sources[@]}" -ne 1 ]] && plural="s" - str="Adding blocklist domain source${plural} to the whitelist" - echo -ne " ${INFO} ${str}..." # Create array of unique $sourceDomains mapfile -t uniqDomains <<< "$(awk '{ if(!a[$1]++) { print $1 } }' <<< "$(printf '%s\n' "${sourceDomains[@]}")")" + [[ "${#uniqDomains[@]}" -ne 1 ]] && plural="s" + + str="Adding ${#uniqDomains[@]} blocklist source domain${plural} to the whitelist" + echo -ne " ${INFO} ${str}..." # Whitelist $uniqDomains "${PIHOLE_COMMAND}" -w -nr -q "${uniqDomains[*]}" &> /dev/null @@ -627,7 +621,7 @@ else # Gravity needs to modify Blacklist/Whitelist/Wildcards echo -e " ${INFO} Using cached Event Horizon list..." numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") - echo -e " ${INFO} ${COL_LIGHT_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon" + echo -e " ${INFO} ${COL_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon" fi # Perform when downloading blocklists, or modifying the whitelist From 909bc92c01646271cbb634adcaf5426784347b34 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Thu, 21 Sep 2017 00:22:47 +1000 Subject: [PATCH 32/50] Tweak wording to fit on 80 character screen * Also move colour after $INFO --- automated install/uninstall.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index cfc60bee..2f4f4f9f 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -208,7 +208,7 @@ else echo -e " ${INFO} Be sure to confirm if any dependencies should not be removed" fi while true; do - echo -e "${COL_YELLOW} ${INFO} The following dependencies may have been added to the system by Pi-hole install:" + echo -e " ${INFO} ${COL_YELLOW}The following dependencies may have been added by the Pi-hole install:" echo -n " " for i in "${DEPS[@]}"; do echo -n "${i} " From 11282aaca378e52861dfb7683cf4f63cc7d71602 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Thu, 21 Sep 2017 23:11:44 +1000 Subject: [PATCH 33/50] Decrease `local-ttl` to 2 * Decreasing value should benefit clients when whitelisting blocked sites, [as per this discussion](https://github.com/pi-hole/pi-hole/pull/1698#issuecomment-331134576). --- advanced/01-pihole.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/01-pihole.conf b/advanced/01-pihole.conf index 8b772ae8..f7b78ab0 100644 --- a/advanced/01-pihole.conf +++ b/advanced/01-pihole.conf @@ -42,6 +42,6 @@ cache-size=10000 log-queries log-facility=/var/log/pihole.log -local-ttl=300 +local-ttl=2 log-async From 2deb2bf03f452f824645abb55deb3252014cf0f1 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Fri, 22 Sep 2017 02:23:43 +1000 Subject: [PATCH 34/50] Fix broken whitelist functionality --- gravity.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index cb717aba..881fc6f2 100755 --- a/gravity.sh +++ b/gravity.sh @@ -631,8 +631,8 @@ fi gravity_ShowBlockCount -# Perform when downloading blocklists, or modifying the blacklist -if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "blacklist" ]]; then +# Perform when downloading blocklists, or modifying the white/blacklist (not wildcards) +if [[ "${skipDownload}" == false ]] || [[ "${listType}" == *"list" ]]; then str="Parsing domains into hosts format" echo -ne " ${INFO} ${str}..." @@ -645,10 +645,7 @@ if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "blacklist" ]]; then fi echo -e "${OVER} ${TICK} ${str}" -fi -# Perform when downloading blocklists -if [[ "${skipDownload}" == false ]]; then gravity_Cleanup fi From 47099e2855884feb782479172803cf3e4baf4297 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Fri, 22 Sep 2017 03:56:53 +1000 Subject: [PATCH 35/50] Ensure resolver test occurs each second * Ensure that gravity will run the second the resolver is available * Increase timeout to 120s --- gravity.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/gravity.sh b/gravity.sh index 881fc6f2..6c3e9ec6 100755 --- a/gravity.sh +++ b/gravity.sh @@ -77,14 +77,14 @@ gravity_DNSLookup() { fi # Determine if $lookupDomain is resolvable - if timeout 5 getent hosts "${lookupDomain}" &> /dev/null; then + if timeout 1 getent hosts "${lookupDomain}" &> /dev/null; then # Print confirmation of resolvability if it had previously failed if [[ -n "${secs:-}" ]]; then echo -e "${OVER} ${TICK} DNS resolution is now available\\n" fi return 0 elif [[ -n "${secs:-}" ]]; then - echo -e "${OVER} ${CROSS} DNS resolution is still unavailable, cancelling" + echo -e "${OVER} ${CROSS} DNS resolution is not available" exit 1 fi @@ -96,13 +96,15 @@ gravity_DNSLookup() { "${PIHOLE_COMMAND}" restartdns fi - # Give time for DNS server to be resolvable - secs="30" - while [[ "${secs}" -ne 0 ]]; do - [[ "${secs}" -ne 1 ]] && plural="s" - echo -ne "${OVER} ${INFO} Waiting $secs second${plural} before continuing..." - sleep 1 + # Ensure DNS server is given time to be resolvable + secs="120" + echo -ne " ${INFO} Waiting up to ${secs} seconds before continuing..." + until timeout 1 getent hosts "${lookupDomain}" &> /dev/null; do + [[ "${secs:-}" -eq 0 ]] && break + [[ "${secs:-}" -ne 1 ]] && plural="s" + echo -ne "${OVER} ${INFO} Waiting up to ${secs} second${plural} before continuing..." : $((secs--)) + sleep 1 done # Try again From e3e3b4da58d4b08f98d4e638864d12a6321a1ad0 Mon Sep 17 00:00:00 2001 From: Celly Date: Thu, 21 Sep 2017 17:47:37 -0400 Subject: [PATCH 36/50] Add in some exclusions form some leaky files in the admin --- advanced/lighttpd.conf.debian | 7 ++++++- advanced/lighttpd.conf.fedora | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/advanced/lighttpd.conf.debian b/advanced/lighttpd.conf.debian index 47f6af02..10085dd8 100644 --- a/advanced/lighttpd.conf.debian +++ b/advanced/lighttpd.conf.debian @@ -41,7 +41,7 @@ accesslog.format = "%{%s}t|%V|%r|%s|%b" index-file.names = ( "index.php", "index.html", "index.lighttpd.html" ) -url.access-deny = ( "~", ".inc" ) +url.access-deny = ( "~", ".inc", ".md", ".yml", ".ini" ) static-file.exclude-extensions = ( ".php", ".pl", ".fcgi" ) compress.cache-dir = "/var/cache/lighttpd/compress/" @@ -66,5 +66,10 @@ $HTTP["url"] =~ "^/admin/" { } } +# Block the github files from being accessible. +$HTTP["url"] =~ "^/admin/(.git|.gitignore|.github)" { + url.access-deny = ("") +} + # Add user chosen options held in external file include_shell "cat external.conf 2>/dev/null" diff --git a/advanced/lighttpd.conf.fedora b/advanced/lighttpd.conf.fedora index 773f0142..27a854af 100644 --- a/advanced/lighttpd.conf.fedora +++ b/advanced/lighttpd.conf.fedora @@ -42,7 +42,7 @@ accesslog.format = "%{%s}t|%V|%r|%s|%b" index-file.names = ( "index.php", "index.html", "index.lighttpd.html" ) -url.access-deny = ( "~", ".inc" ) +url.access-deny = ( "~", ".inc", ".md", ".yml", ".ini" ) static-file.exclude-extensions = ( ".php", ".pl", ".fcgi" ) compress.cache-dir = "/var/cache/lighttpd/compress/" @@ -85,5 +85,10 @@ $HTTP["url"] =~ "^/admin/" { } } +# Block the github files from being accessible. +$HTTP["url"] =~ "^/admin/(.git|.gitignore|.github)" { + url.access-deny = ("") +} + # Add user chosen options held in external file include_shell "cat external.conf 2>/dev/null" From 3aa525b0c0311f8d6775cdf7153625992510fa53 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Fri, 22 Sep 2017 14:17:56 +1000 Subject: [PATCH 37/50] Ensure domains files are not deleted upon w/blist --- gravity.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/gravity.sh b/gravity.sh index 6c3e9ec6..cba3c083 100755 --- a/gravity.sh +++ b/gravity.sh @@ -554,14 +554,17 @@ gravity_Cleanup() { rm ${piholeDir}/*.tmp 2> /dev/null rm /tmp/*.phgpb 2> /dev/null - # Remove any unused .domains files - for file in ${piholeDir}/*.${domainsExtension}; do - # If list is not in active array, then remove it - if [[ ! "${activeDomains[*]}" == *"${file}"* ]]; then - rm -f "${file}" 2> /dev/null || \ - echo -e " ${CROSS} Failed to remove ${file##*/}" - fi - done + # Ensure this function only runs alongside gravity_Supernova() + if [[ "${skipDownload}" == false ]]; then + # Remove any unused .domains files + for file in ${piholeDir}/*.${domainsExtension}; do + # If list is not in active array, then remove it + if [[ ! "${activeDomains[*]}" == *"${file}"* ]]; then + rm -f "${file}" 2> /dev/null || \ + echo -e " ${CROSS} Failed to remove ${file##*/}" + fi + done + fi echo -e "${OVER} ${TICK} ${str}" From feb2150d9b47e55655d3624f81ffe5c2f6cfcdd4 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Fri, 22 Sep 2017 19:14:11 +1000 Subject: [PATCH 38/50] Fix `LIGHTTPD_ENABLED` dupes in `setupVars.conf` * Fix issue merged [here](https://github.com/pi-hole/pi-hole/commit/fdf2649f2fef8981176735fb856997e4856fc13b#diff-689793e68cde6270d074695b71c969afL1488) --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f19481cf..6a602257 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1450,7 +1450,7 @@ finalExports() { # If the setup variable file exists, if [[ -e "${setupVars}" ]]; then # update the variables in the file - sed -i.update.bak '/PIHOLE_INTERFACE/d;/IPV4_ADDRESS/d;/IPV6_ADDRESS/d;/PIHOLE_DNS_1/d;/PIHOLE_DNS_2/d;/QUERY_LOGGING/d;/INSTALL_WEB/d;' "${setupVars}" + sed -i.update.bak '/PIHOLE_INTERFACE/d;/IPV4_ADDRESS/d;/IPV6_ADDRESS/d;/PIHOLE_DNS_1/d;/PIHOLE_DNS_2/d;/QUERY_LOGGING/d;/INSTALL_WEB/d;/LIGHTTPD_ENABLED/d;' "${setupVars}" fi # echo the information to the user { From f54a812ad5c68cf0b64d436bfc650f1a0262edfa Mon Sep 17 00:00:00 2001 From: Celly Date: Fri, 22 Sep 2017 09:27:43 -0400 Subject: [PATCH 39/50] Update access rules to block all root '.' files --- advanced/lighttpd.conf.debian | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/lighttpd.conf.debian b/advanced/lighttpd.conf.debian index 10085dd8..07f0e964 100644 --- a/advanced/lighttpd.conf.debian +++ b/advanced/lighttpd.conf.debian @@ -66,8 +66,8 @@ $HTTP["url"] =~ "^/admin/" { } } -# Block the github files from being accessible. -$HTTP["url"] =~ "^/admin/(.git|.gitignore|.github)" { +# Block . files from being served, such as .git, .github, .gitignore +$HTTP["url"] =~ "^/admin/\.(.*)" { url.access-deny = ("") } From 3f20981aab1ba1d48a9c14be1c0cbae924e80dff Mon Sep 17 00:00:00 2001 From: Celly Date: Fri, 22 Sep 2017 09:29:00 -0400 Subject: [PATCH 40/50] Update access rules to block all root '.' files --- advanced/lighttpd.conf.fedora | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/lighttpd.conf.fedora b/advanced/lighttpd.conf.fedora index 27a854af..7b2449c6 100644 --- a/advanced/lighttpd.conf.fedora +++ b/advanced/lighttpd.conf.fedora @@ -85,8 +85,8 @@ $HTTP["url"] =~ "^/admin/" { } } -# Block the github files from being accessible. -$HTTP["url"] =~ "^/admin/(.git|.gitignore|.github)" { +# Block . files from being served, such as .git, .github, .gitignore +$HTTP["url"] =~ "^/admin/\.(.*)" { url.access-deny = ("") } From cd7c00ef8e4dd3b64721c31045c3c616694c9d5f Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Sat, 23 Sep 2017 10:32:56 +1000 Subject: [PATCH 41/50] Fix Ctrl-C inadvertently deleting domains files --- gravity.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index cba3c083..a228644c 100755 --- a/gravity.sh +++ b/gravity.sh @@ -181,6 +181,7 @@ gravity_Supernova() { echo "" fi done + gravity_Blackbody=true } # Download specified URL and perform checks on HTTP status and file content @@ -554,8 +555,8 @@ gravity_Cleanup() { rm ${piholeDir}/*.tmp 2> /dev/null rm /tmp/*.phgpb 2> /dev/null - # Ensure this function only runs alongside gravity_Supernova() - if [[ "${skipDownload}" == false ]]; then + # Ensure this function only runs when gravity_Supernova() has completed + if [[ "${gravity_Blackbody:-}" == true ]]; then # Remove any unused .domains files for file in ${piholeDir}/*.${domainsExtension}; do # If list is not in active array, then remove it From 664d0ea0237db09a4c6bcbac900c596bd70af3cc Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Sat, 23 Sep 2017 11:27:40 +1000 Subject: [PATCH 42/50] Correctly retrieve IPv4/6 addresses for tailFunc() * Add comments for readability * Use `sed -E` for readability * Move `date` into `sed` * Use updated colour codes --- pihole | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pihole b/pihole index c2925d78..45f7ea92 100755 --- a/pihole +++ b/pihole @@ -508,13 +508,20 @@ statusFunc() { } tailFunc() { - date=$(date +'%b %d ') echo -e " ${INFO} Press Ctrl-C to exit" - tail -f /var/log/pihole.log | sed \ - -e "s,\(${date}\| dnsmasq\[.*[0-9]]\),,g" \ - -e "s,\(.*\(gravity.list\|black.list\| config \).* is \(${IPV4_ADDRESS%/*}\|${IPV6_ADDRESS:-NULL}\).*\),${COL_LIGHT_RED}&${COL_NC}," \ - -e "s,.*\(query\[A\|DHCP\).*,${COL_NC}&${COL_NC}," \ - -e "s,.*,${COL_DARK_GRAY}&${COL_NC}," + + # Retrieve IPv4/6 addresses + source /etc/pihole/setupVars.conf + + # Strip date from each line + # Colour blocklist/blacklist/wildcard entries as red + # Colour A/AAAA/DHCP strings as white + # Colour everything else as gray + tail -f /var/log/pihole.log | sed -E \ + -e "s,($(date +'%b %d ')| dnsmasq[.*[0-9]]),,g" \ + -e "s,(.*(gravity.list|black.list| config ).* is (${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \ + -e "s,.*(query\\[A|DHCP).*,${COL_NC}&${COL_NC}," \ + -e "s,.*,${COL_GRAY}&${COL_NC}," exit 0 } From b9bcfe36ff2d15d988bc62a6a0a8dbd619d3aada Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Sun, 24 Sep 2017 11:57:15 +1000 Subject: [PATCH 43/50] Place errors on newline --- gravity.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index a228644c..bec843df 100755 --- a/gravity.sh +++ b/gravity.sh @@ -345,7 +345,7 @@ gravity_ParseFileIntoDomains() { output=$( { mv "${source}" "${destination}"; } 2>&1 ) if [[ ! -e "${destination}" ]]; then - echo -e " ${CROSS} Unable to move tmp file to ${piholeDir} + echo -e "\\n ${CROSS} Unable to move tmp file to ${piholeDir} ${output}" gravity_Cleanup "error" fi @@ -521,7 +521,7 @@ gravity_ParseBlacklistDomains() { status="$?" if [[ "${status}" -ne 0 ]]; then - echo -e " ${CROSS} Unable to move ${accretionDisc} from ${piholeDir}\\n ${output}" + echo -e "\\n ${CROSS} Unable to move ${accretionDisc} from ${piholeDir}\\n ${output}" gravity_Cleanup "error" fi } @@ -535,7 +535,7 @@ gravity_ParseUserDomains() { gravity_ParseDomainsIntoHosts "${blacklistFile}" "${blackList}.tmp" # Copy the file over as /etc/pihole/black.list so dnsmasq can use it mv "${blackList}.tmp" "${blackList}" 2> /dev/null || \ - echo -e " ${CROSS} Unable to move ${blackList##*/}.tmp to ${piholeDir}" + echo -e "\\n ${CROSS} Unable to move ${blackList##*/}.tmp to ${piholeDir}" } # Trap Ctrl-C From 466e6d9b3024fcf536f9ad536e29bc88d033cfe3 Mon Sep 17 00:00:00 2001 From: spacedingo Date: Tue, 26 Sep 2017 05:53:07 +0400 Subject: [PATCH 44/50] Redo pull #1687 Fix for case sensitive duplicate domains The change (https://github.com/pi-hole/pi-hole/pull/1687) was recently undone. --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index bec843df..5954764c 100755 --- a/gravity.sh +++ b/gravity.sh @@ -367,7 +367,7 @@ gravity_Schwarzschild() { # Determine if file has read permissions, as download might have failed if [[ -r "${i}" ]]; then # Remove windows CRs from file, and append into $matterAndLight - tr -d '\r' < "${i}" >> "${piholeDir}/${matterAndLight}" + tr -d '\r' < "${i}" | tr '[:upper:]' '[:lower:]' >> "${piholeDir}/${matterAndLight}" # Ensure that the first line of a new list is on a new line lastLine=$(tail -1 "${piholeDir}/${matterAndLight}") From 60365ad36a84b3bc99cdca8f0d5c5350842608a7 Mon Sep 17 00:00:00 2001 From: spacedingo Date: Tue, 26 Sep 2017 09:25:47 +0400 Subject: [PATCH 45/50] Update gravity.sh --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 5954764c..8719885c 100755 --- a/gravity.sh +++ b/gravity.sh @@ -366,7 +366,7 @@ gravity_Schwarzschild() { for i in "${activeDomains[@]}"; do # Determine if file has read permissions, as download might have failed if [[ -r "${i}" ]]; then - # Remove windows CRs from file, and append into $matterAndLight + # Remove windows CRs from file, convert list to lower case, and append into $matterAndLight tr -d '\r' < "${i}" | tr '[:upper:]' '[:lower:]' >> "${piholeDir}/${matterAndLight}" # Ensure that the first line of a new list is on a new line From 39ab1e1ea7ceaea5993cd733cecc2d4a092032aa Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 30 Sep 2017 13:03:49 +0200 Subject: [PATCH 46/50] When using the Pi-hole DHCP server, local host names are called "something.local". Thir PR ensures that clients that are connected via VPN are augmented by a similar suffix like "someother.vpn". Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 8719885c..f4b5fc36 100755 --- a/gravity.sh +++ b/gravity.sh @@ -503,7 +503,7 @@ gravity_ParseLocalDomains() { # Add additional LAN hosts provided by OpenVPN (if available) if [[ -f "${VPNList}" ]]; then - awk -F, '{printf $2"\t"$1"\n"}' "${VPNList}" >> "${localList}" + awk -F, '{printf $2"\t"$1".vpn\n"}' "${VPNList}" >> "${localList}" fi } From 1615fa63e38a8500e9d0513ca8b2406c3cbe55fc Mon Sep 17 00:00:00 2001 From: Eric Wolf <19wolf@gmail.com> Date: Sat, 30 Sep 2017 16:34:38 -0400 Subject: [PATCH 47/50] Update basic-install.sh Fixed comment line 40 --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 2cf2c61d..1775a8d9 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -37,7 +37,7 @@ QUERY_LOGGING=true INSTALL_WEB=true -# Find the rows and columns will default to 80x24 is it can not be detected +# Find the rows and columns will default to 80x24 if it can not be detected screen_size=$(stty size 2>/dev/null || echo 24 80) rows=$(echo "${screen_size}" | awk '{print $1}') columns=$(echo "${screen_size}" | awk '{print $2}') From 97dd1b4cf30901559703e664b97a53c437336da8 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Wed, 4 Oct 2017 00:46:51 +1100 Subject: [PATCH 48/50] Improve FQDN Authorized Hosts functionality * Use inverse if statement, instead of IF/ELSE when checking setupVars.conf * Remove $svFQDN * Add or elaborate on more comments * Add $serverName to $authorizedHosts if admin has specified `setenv.add-environment = ("fqdn" => "true")` within lighttpd's external.conf * e.g: `$HTTP["host"] == "pihole.domain.com" { setenv.add-environment = ("fqdn" => "true") }` * Move "No exact results" check to top of exception handling * Remove unnecessary IF/ELSE when handling $queryAds error Signed off by WaLLy3K --- advanced/index.php | 99 +++++++++++++++++++++++++++------------------- 1 file changed, 59 insertions(+), 40 deletions(-) diff --git a/advanced/index.php b/advanced/index.php index 911f3cc8..0111c95f 100644 --- a/advanced/index.php +++ b/advanced/index.php @@ -9,32 +9,30 @@ // Sanitise HTTP_HOST output $serverName = htmlspecialchars($_SERVER["HTTP_HOST"]); +if (!is_file("/etc/pihole/setupVars.conf")) + die("[ERROR] File not found: /etc/pihole/setupVars.conf"); + // Get values from setupVars.conf -if (is_file("/etc/pihole/setupVars.conf")) { - $setupVars = parse_ini_file("/etc/pihole/setupVars.conf"); - $svFQDN = $setupVars["FQDN"]; - $svPasswd = !empty($setupVars["WEBPASSWORD"]); - $svEmail = (!empty($setupVars["ADMIN_EMAIL"]) && filter_var($setupVars["ADMIN_EMAIL"], FILTER_VALIDATE_EMAIL)) ? $setupVars["ADMIN_EMAIL"] : ""; - unset($setupVars); -} else { - die("[ERROR] File not found: /etc/pihole/setupVars.conf"); -} +$setupVars = parse_ini_file("/etc/pihole/setupVars.conf"); +$svPasswd = !empty($setupVars["WEBPASSWORD"]); +$svEmail = (!empty($setupVars["ADMIN_EMAIL"]) && filter_var($setupVars["ADMIN_EMAIL"], FILTER_VALIDATE_EMAIL)) ? $setupVars["ADMIN_EMAIL"] : ""; +unset($setupVars); // Set landing page location, found within /var/www/html/ $landPage = "../landing.php"; -// Set empty array for hostnames to be accepted as self address for splash page +// Define array for hostnames to be accepted as self address for splash page $authorizedHosts = []; - -// Append FQDN to $authorizedHosts -if (!empty($svFQDN)) array_push($authorizedHosts, $svFQDN); - -// Append virtual hostname to $authorizedHosts -if (!empty($_SERVER["VIRTUAL_HOST"])) { +if (!empty($_SERVER["FQDN"])) { + // If setenv.add-environment = ("fqdn" => "true") is configured in lighttpd, + // append $serverName to $authorizedHosts + array_push($authorizedHosts, $serverName); +} else if (!empty($_SERVER["VIRTUAL_HOST"])) { + // Append virtual hostname to $authorizedHosts array_push($authorizedHosts, $_SERVER["VIRTUAL_HOST"]); } -// Set which extension types render as Block Page (Including "" for index.wxyz) +// Set which extension types render as Block Page (Including "" for index.ext) $validExtTypes = array("asp", "htm", "html", "php", "rss", "xml", ""); // Get extension of current URL @@ -49,8 +47,9 @@ function setHeader($type = "x") { if (isset($type) && $type === "js") header("Content-Type: application/javascript"); } -// Determine block page redirect type +// Determine block page type if ($serverName === "pi.hole") { + // Redirect to Web Interface exit(header("Location: /admin")); } elseif (filter_var($serverName, FILTER_VALIDATE_IP) || in_array($serverName, $authorizedHosts)) { // Set Splash Page output @@ -60,22 +59,29 @@ if ($serverName === "pi.hole") {
Pi-hole: Your black hole for Internet advertisements "; - - // Render splash page or landing page when directly browsing via IP or auth'd hostname + + // Set splash/landing page based off presence of $landPage $renderPage = is_file(getcwd()."/$landPage") ? include $landPage : "$splashPage"; - unset($serverName, $svFQDN, $svPasswd, $svEmail, $authorizedHosts, $validExtTypes, $currentUrlExt, $viewPort); + + // Unset variables so as to not be included in $landPage + unset($serverName, $svPasswd, $svEmail, $authorizedHosts, $validExtTypes, $currentUrlExt, $viewPort); + + // Render splash/landing page when directly browsing via IP or authorised hostname exit($renderPage); } elseif ($currentUrlExt === "js") { - // Serve dummy Javascript for blocked domains + // Serve Pi-hole Javascript for blocked domains requesting JS exit(setHeader("js").'var x = "Pi-hole: A black hole for Internet advertisements."'); } elseif (strpos($_SERVER["REQUEST_URI"], "?") !== FALSE && isset($_SERVER["HTTP_REFERER"])) { - // Serve blank image upon receiving REQUEST_URI w/ query string & HTTP_REFERRER (e.g: an iframe of a blocked domain) + // Serve blank image upon receiving REQUEST_URI w/ query string & HTTP_REFERRER + // e.g: An iframe of a blocked domain exit(setHeader().' '); } elseif (!in_array($currentUrlExt, $validExtTypes) || substr_count($_SERVER["REQUEST_URI"], "?")) { - // Serve SVG upon receiving non $validExtTypes URL extension or query string (e.g: not an iframe of a blocked domain) + // Serve SVG upon receiving non $validExtTypes URL extension or query string + // e.g: Not an iframe of a blocked domain, such as when browsing to a file/query directly + // QoL addition: Allow the SVG to be clicked on in order to quickly show the full Block Page $blockImg = 'Blocked by Pi-hole'; exit(setHeader()." $viewPort @@ -88,7 +94,7 @@ if ($serverName === "pi.hole") { // Determine placeholder text based off $svPasswd presence $wlPlaceHolder = empty($svPasswd) ? "No admin password set" : "Javascript disabled"; -// Define admin email address text +// Define admin email address text based off $svEmail presence $bpAskAdmin = !empty($svEmail) ? '' : ""; // Determine if at least one block list has been generated @@ -113,8 +119,10 @@ if (empty($adlistsUrls)) // Get total number of blocklists (Including Whitelist, Blacklist & Wildcard lists) $adlistsCount = count($adlistsUrls) + 3; -// Get results of queryads.php exact search +// Set query timeout ini_set("default_socket_timeout", 3); + +// Logic for querying blocklists function queryAds($serverName) { // Determine the time it takes while querying adlists $preQueryTime = microtime(true)-$_SERVER["REQUEST_TIME_FLOAT"]; @@ -124,32 +132,39 @@ function queryAds($serverName) { // Exception Handling try { - if ($queryTime >= ini_get("default_socket_timeout")) { + // Define Exceptions + if (strpos($queryAds[0], "No exact results") !== FALSE) { + // Return "none" into $queryAds array + return array("0" => "none"); + } else if ($queryTime >= ini_get("default_socket_timeout")) { + // Connection Timeout throw new Exception ("Connection timeout (".ini_get("default_socket_timeout")."s)"); } elseif (!strpos($queryAds[0], ".") !== false) { - if (strpos($queryAds[0], "No exact results") !== FALSE) return array("0" => "none"); + // Unknown $queryAds output throw new Exception ("Unhandled error message ($queryAds[0])"); } return $queryAds; } catch (Exception $e) { + // Return exception as array return array("0" => "error", "1" => $e->getMessage()); } - } +// Get results of queryads.php exact search $queryAds = queryAds($serverName); -if ($queryAds[0] === "error") { +// Pass error through to Block Page +if ($queryAds[0] === "error") die("[ERROR]: Unable to parse results from queryads.php: ".$queryAds[1].""); -} else { - $featuredTotal = count($queryAds); - // Place results into key => value array - $queryResults = null; - foreach ($queryAds as $str) { - $value = explode(" ", $str); - @$queryResults[$value[0]] .= "$value[1]"; - } +// Count total number of matching blocklists +$featuredTotal = count($queryAds); + +// Place results into key => value array +$queryResults = null; +foreach ($queryAds as $str) { + $value = explode(" ", $str); + @$queryResults[$value[0]] .= "$value[1]"; } // Determine if domain has been blacklisted, whitelisted, wildcarded or CNAME blocked @@ -167,7 +182,8 @@ if (strpos($queryAds[0], "blacklist") !== FALSE) { $featuredTotal = "0"; $notableFlagClass = "noblock"; - // Determine appropriate info message if CNAME exists + // QoL addition: Determine appropriate info message if CNAME exists + // Suggests to the user that $serverName has a CNAME (alias) that may be blocked $dnsRecord = dns_get_record("$serverName")[0]; if (array_key_exists("target", $dnsRecord)) { $wlInfo = $dnsRecord['target']; @@ -184,9 +200,12 @@ $wlOutput = (isset($wlInfo) && $wlInfo !== "recentwl") ? "