diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 5539cec9..01be8b25 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.4.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 89b6323f..0fe850d1 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -5,13 +5,35 @@ on: branches: - master +# The section is needed to drop the default write-all permissions for all jobs +# that are granted on `push` event. By specifying any permission explicitly +# all others are set to none. By using the principle of least privilege the damage a compromised +# workflow can do (because of an injection or compromised third party tool or +# action) is restricted. Adding labels to issues, commenting +# on pull-requests, etc. may need additional permissions: +# +# Syntax for this section: +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions +# +# Reference for how to assign permissions on a job-by-job basis: +# https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs +# +# Reference for available permissions that we can enable if needed: +# https://docs.github.com/en/actions/security-guides/automatic-token-authentication#permissions-for-the-github_token +permissions: {} + jobs: sync-branches: + # The job needs to be able to pull the code and create a pull request. + permissions: + contents: read # for actions/checkout + pull-requests: write # to create pull request + runs-on: ubuntu-latest name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.4.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ce948e09..27867ef3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.4.0 - name: Check scripts in repository are executable run: | @@ -62,7 +62,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.4.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.5.0 diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index ad25e866..fa1cebbb 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -230,7 +230,7 @@ initialize_debug() { # This is a function for visually displaying the current test that is being run. # Accepts one variable: the name of what is being diagnosed -# Colors do not show in the dasboard, but the icons do: [i], [✓], and [✗] +# Colors do not show in the dashboard, but the icons do: [i], [✓], and [✗] echo_current_diagnostic() { # Colors are used for visually distinguishing each test in the output # These colors do not show in the GUI, but the formatting will diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index d48e9363..12295fbc 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -30,33 +30,6 @@ gravityDBfile="${GRAVITYDB}" colfile="/opt/pihole/COL_TABLE" source "${colfile}" -# Scan an array of files for matching strings -scanList(){ - # Escape full stops - local domain="${1}" esc_domain="${1//./\\.}" lists="${2}" list_type="${3:-}" - - # Prevent grep from printing file path - cd "$piholeDir" || exit 1 - - # Prevent grep -i matching slowly: https://bit.ly/2xFXtUX - export LC_CTYPE=C - - # /dev/null forces filename to be printed when only one list has been generated - case "${list_type}" in - "exact" ) grep -i -E -l "(^|(?/dev/null;; - # Iterate through each regexp and check whether it matches the domainQuery - # If it does, print the matching regexp and continue looping - # Input 1 - regexps | Input 2 - domainQuery - "regex" ) - for list in ${lists}; do - if [[ "${domain}" =~ ${list} ]]; then - printf "%b\n" "${list}"; - fi - done;; - * ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;; - esac -} - if [[ "${options}" == "-h" ]] || [[ "${options}" == "--help" ]]; then echo "Usage: pihole -q [option] Example: 'pihole -q -exact domain.com' @@ -84,17 +57,47 @@ options=$(sed -E 's/ ?-(all|exact) ?//g' <<< "${options}") case "${options}" in "" ) str="No domain specified";; *" "* ) str="Unknown query option specified";; - *[![:ascii:]]* ) domainQuery=$(idn2 "${options}");; - * ) domainQuery="${options}";; + *[![:ascii:]]* ) rawDomainQuery=$(idn2 "${options}");; + * ) rawDomainQuery="${options}";; esac +# convert the domain to lowercase +domainQuery=$(echo "${rawDomainQuery}" | tr '[:upper:]' '[:lower:]') + if [[ -n "${str:-}" ]]; then echo -e "${str}${COL_NC}\\nTry 'pihole -q --help' for more information." exit 1 fi +# Scan an array of files for matching strings +scanList(){ + # Escape full stops + local domain="${1}" esc_domain="${1//./\\.}" lists="${2}" list_type="${3:-}" + + # Prevent grep from printing file path + cd "$piholeDir" || exit 1 + + # Prevent grep -i matching slowly: https://bit.ly/2xFXtUX + export LC_CTYPE=C + + # /dev/null forces filename to be printed when only one list has been generated + case "${list_type}" in + "exact" ) grep -i -E -l "(^|(?/dev/null;; + # Iterate through each regexp and check whether it matches the domainQuery + # If it does, print the matching regexp and continue looping + # Input 1 - regexps | Input 2 - domainQuery + "regex" ) + for list in ${lists}; do + if [[ "${domain}" =~ ${list} ]]; then + printf "%b\n" "${list}"; + fi + done;; + * ) grep -i "${esc_domain}" "${lists}" /dev/null 2>/dev/null;; + esac +} + scanDatabaseTable() { - local domain table list_type querystr result extra + local domain table list_type querystr result extra abpquerystr abpfound abpentry searchstr domain="$(printf "%q" "${1}")" table="${2}" list_type="${3:-}" @@ -104,9 +107,34 @@ scanDatabaseTable() { # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores. if [[ "${table}" == "gravity" ]]; then + + # Are there ABP entries on gravity? + # Return 1 if abp_domain=1 or Zero if abp_domain=0 or not set + abpquerystr="SELECT EXISTS (SELECT 1 FROM info WHERE property='abp_domains' and value='1')" + abpfound="$(pihole-FTL sqlite3 "${gravityDBfile}" "${abpquerystr}")" 2> /dev/null + + # Create search string for ABP entries only if needed + if [ "${abpfound}" -eq 1 ]; then + abpentry="${domain}" + + searchstr="'||${abpentry}^'" + + # While a dot is found ... + while [ "${abpentry}" != "${abpentry/./}" ] + do + # ... remove text before the dot (including the dot) and append the result to $searchstr + abpentry=$(echo "${abpentry}" | cut -f 2- -d '.') + searchstr="$searchstr, '||${abpentry}^'" + done + + # The final search string will look like: + # "domain IN ('||sub2.sub1.domain.com^', '||sub1.domain.com^', '||domain.com^', '||com^') OR" + searchstr="domain IN (${searchstr}) OR " + fi + case "${exact}" in "exact" ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain = '${domain}'";; - * ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; + * ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE ${searchstr} domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; esac else case "${exact}" in @@ -116,7 +144,7 @@ scanDatabaseTable() { fi # Send prepared query to gravity database - result="$(pihole-FTL sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null + result="$(pihole-FTL sqlite3 -separator ',' "${gravityDBfile}" "${querystr}")" 2> /dev/null if [[ -z "${result}" ]]; then # Return early when there are no matches in this table return @@ -136,8 +164,8 @@ scanDatabaseTable() { # Loop over results and print them mapfile -t results <<< "${result}" for result in "${results[@]}"; do - domain="${result/|*}" - if [[ "${result#*|}" == "0" ]]; then + domain="${result/,*}" + if [[ "${result#*,}" == "0" ]]; then extra=" (disabled)" else extra="" @@ -212,10 +240,10 @@ if [[ -n "${exact}" ]]; then fi for result in "${results[@]}"; do - match="${result/|*/}" - extra="${result#*|}" - adlistAddress="${extra/|*/}" - extra="${extra#*|}" + match="${result/,*/}" + extra="${result#*,}" + adlistAddress="${extra/,*/}" + extra="${extra#*,}" if [[ "${extra}" == "0" ]]; then extra=" (disabled)" else diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 37516472..2d3c7fb1 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -44,7 +44,7 @@ addOrEditKeyValPair() { } ####################### -# Takes two arguments: file, and key. +# Takes two arguments: file and key. # Adds a key to target file # # Example usage: @@ -57,14 +57,18 @@ addKey(){ # touch file to prevent grep error if file does not exist yet touch "${file}" - if ! grep -q "^${key}" "${file}"; then + # Match key against entire line, using both anchors. We assume + # that the file's keys never have bounding whitespace. Anchors + # are necessary to ensure the key is considered absent when it + # is a substring of another key present in the file. + if ! grep -q "^${key}$" "${file}"; then # Key does not exist, add it. echo "${key}" >> "${file}" fi } ####################### -# Takes two arguments: file, and key. +# Takes two arguments: file and key. # Deletes a key or key/value pair from target file # # Example usage: @@ -76,6 +80,24 @@ removeKey() { sed -i "/^${key}/d" "${file}" } +####################### +# Takes two arguments: file and key. +# Returns the value of a given key from target file +# - ignores all commented lines +# - only returns the first value if multiple identical keys exist +# +# +# Example usage: +# getVal "/etc/pihole/setupVars.conf" "PIHOLE_DNS_1" +####################### +getVal() { + local file="${1}" + local key="${2}" + local value + value=$(sed -e '/^[[:blank:]]*#/d' "${file}" | grep "${key}" | awk -F "=" 'NR==1{printf$2}') + printf "%s" "$value" +} + ####################### # returns FTL's current telnet API port based on the setting in /etc/pihole-FTL.conf diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ccb5eac7..a781f8c6 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2612,7 +2612,8 @@ main() { # Get the privacy level if it exists (default is 0) if [[ -f "${FTL_CONFIG_FILE}" ]]; then - PRIVACY_LEVEL=$(sed -ne 's/PRIVACYLEVEL=\(.*\)/\1/p' "${FTL_CONFIG_FILE}") + # use getVal from utils.sh to get PRIVACYLEVEL + PRIVACY_LEVEL=$(getVal "${FTL_CONFIG_FILE}" "PRIVACYLEVEL") # If no setting was found, default to 0 PRIVACY_LEVEL="${PRIVACY_LEVEL:-0}" diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index c36027fc..7a1a290d 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -193,6 +193,18 @@ removeNoPurge() { else service pihole-FTL stop fi + ${SUDO} rm -f /etc/systemd/system/pihole-FTL.service + if [[ -d '/etc/systemd/system/pihole-FTL.service.d' ]]; then + read -rp " ${QST} FTL service override directory /etc/systemd/system/pihole-FTL.service.d detected. Do you wish to remove this from your system? [y/N] " answer + case $answer in + [yY]*) + echo -ne " ${INFO} Removing /etc/systemd/system/pihole-FTL.service.d..." + ${SUDO} rm -R /etc/systemd/system/pihole-FTL.service.d + echo -e "${OVER} ${INFO} Removed /etc/systemd/system/pihole-FTL.service.d" + ;; + *) echo -e " ${INFO} Leaving /etc/systemd/system/pihole-FTL.service.d in place.";; + esac + fi ${SUDO} rm -f /etc/init.d/pihole-FTL ${SUDO} rm -f /usr/bin/pihole-FTL echo -e "${OVER} ${TICK} Removed pihole-FTL" diff --git a/gravity.sh b/gravity.sh index c2795442..ca859e38 100755 --- a/gravity.sh +++ b/gravity.sh @@ -52,6 +52,14 @@ else exit 1 fi +# Set up tmp dir variable in case it's not configured +: "${GRAVITY_TMPDIR:=/tmp}" + +if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then + echo -e " ${COL_LIGHT_RED}Gravity temporary directory does not exist or is not a writeable directory, falling back to /tmp. ${COL_NC}" + GRAVITY_TMPDIR="/tmp" +fi + # Source pihole-FTL from install script pihole_FTL="${piholeDir}/pihole-FTL.conf" if [[ -f "${pihole_FTL}" ]]; then @@ -137,6 +145,18 @@ update_gravity_timestamp() { return 0 } +# Update timestamp when the gravity table was last updated successfully +set_abp_info() { + pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('abp_domains',${abp_domains});" + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to update ABP domain status in database ${gravityDBfile}\\n ${output}" + return 1 + fi + return 0 +} + # Import domains from file and store them in the specified database table database_table_from_file() { # Define locals @@ -145,7 +165,7 @@ database_table_from_file() { src="${2}" backup_path="${piholeDir}/migration_backup" backup_file="${backup_path}/$(basename "${2}")" - tmpFile="$(mktemp -p "/tmp" --suffix=".gravity")" + tmpFile="$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".gravity")" local timestamp timestamp="$(date --utc +'%s')" @@ -418,7 +438,7 @@ gravity_DownloadBlocklists() { echo -e "${OVER} ${TICK} ${str}" fi - target="$(mktemp -p "/tmp" --suffix=".gravity")" + target="$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".gravity")" # Use compression to reduce the amount of data that is transferred # between the Pi-hole and the ad list provider. Use this feature @@ -519,64 +539,69 @@ gravity_DownloadBlocklists() { gravity_Blackbody=true } -# num_total_imported_domains increases for each list processed -num_total_imported_domains=0 -num_domains=0 -num_non_domains=0 + +# global variable to indicate if we found ABP style domains during the gravity run +# is saved in gravtiy's info table to signal FTL if such domains are available +abp_domains=0 parseList() { - local adlistID="${1}" src="${2}" target="${3}" non_domains sample_non_domains tmp_non_domains_str false_positive - # This sed does the following things: - # 1. Remove all lines containing no domains - # 2. Remove all domains containing invalid characters. Valid are: a-z, A-Z, 0-9, dot (.), minus (-), underscore (_) - # 3. Append ,adlistID to every line - # 4. Remove trailing period (see https://github.com/pi-hole/pi-hole/issues/4701) - # 5. Ensures there is a newline on the last line - sed -r "/([^\.]+\.)+[^\.]{2,}/!d;/[^a-zA-Z0-9.\_-]/d;s/\.$//;s/$/,${adlistID}/;/.$/a\\" "${src}" >> "${target}" + local adlistID="${1}" src="${2}" target="${3}" temp_file temp_file_base non_domains sample_non_domains valid_domain_pattern abp_domain_pattern + + # Create a temporary file for the sed magic instead of using "${target}" directly + # this allows to split the sed commands to improve readability + # we use a file handle here and remove the temporary file immediately so the content will be deleted in any case + # when the script stops + temp_file_base="$(mktemp -p "/tmp" --suffix=".gravity")" + exec 3>"$temp_file_base" + rm "${temp_file_base}" + temp_file="/proc/$$/fd/3" + + # define valid domain patterns + # no need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already + # adapted from https://stackoverflow.com/a/30007882 + # supported ABP style: ||subdomain.domain.tlp^ + + valid_domain_pattern="([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" + abp_domain_pattern="\|\|${valid_domain_pattern}\^" + + + # 1. Add all valid domains + sed -r "/^${valid_domain_pattern}$/!d" "${src}" > "${temp_file}" + + # 2. Add valid ABP style domains if there is at least one such domain + if grep -E "^${abp_domain_pattern}$" -m 1 -q "${src}"; then + echo " ${INFO} List contained AdBlock Plus style domains" + abp_domains=1 + sed -r "/^${abp_domain_pattern}$/!d" "${src}" >> "${temp_file}" + fi + - # Find lines containing no domains or with invalid characters (see above) + # Find lines containing no domains or with invalid characters (not matching regex above) + # This is simply everything that is not in $temp_file compared to $src # Remove duplicates from the list - mapfile -t non_domains <<< "$(sed -r "/([^\.]+\.)+[^\.]{2,}/d" < "${src}")" - mapfile -t -O "${#non_domains[@]}" non_domains <<< "$(sed -r "/[^a-zA-Z0-9.\_-]/!d" < "${src}")" - IFS=" " read -r -a non_domains <<< "$(tr ' ' '\n' <<< "${non_domains[@]}" | sort -u | tr '\n' ' ')" + mapfile -t non_domains < <(grep -Fvf "${temp_file}" "${src}" | sort -u ) + + # 3. Remove trailing period (see https://github.com/pi-hole/pi-hole/issues/4701) + # 4. Append ,adlistID to every line + # 5. Ensures there is a newline on the last line + # and write everything to the target file + sed "s/\.$//;s/$/,${adlistID}/;/.$/a\\" "${temp_file}" >> "${target}" # A list of items of common local hostnames not to report as unusable # Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files # but flagging them as unusable causes more confusion than it's worth - so we suppress them from the output - false_positives=( - "localhost" - "localhost.localdomain" - "local" - "broadcasthost" - "localhost" - "ip6-localhost" - "ip6-loopback" - "lo0 localhost" - "ip6-localnet" - "ip6-mcastprefix" - "ip6-allnodes" - "ip6-allrouters" - "ip6-allhosts" - ) - - # Read the unusable lines into a string - tmp_non_domains_str=" ${non_domains[*]} " - for false_positive in "${false_positives[@]}"; do - # Remove false positives from tmp_non_domains_str - tmp_non_domains_str="${tmp_non_domains_str/ ${false_positive} / }" - done - # Read the string back into an array - IFS=" " read -r -a non_domains <<< "${tmp_non_domains_str}" + false_positives="localhost|localhost.localdomain|local|broadcasthost|localhost|ip6-localhost|ip6-loopback|lo0 localhost|ip6-localnet|ip6-mcastprefix|ip6-allnodes|ip6-allrouters|ip6-allhosts" + + # if there are any non-domains, filter the array for false-positives + # Credit: https://stackoverflow.com/a/40264051 + if [[ "${#non_domains[@]}" -gt 0 ]]; then + mapfile -d $'\0' -t non_domains < <(printf '%s\0' "${non_domains[@]}" | grep -Ezv "^${false_positives}") + fi # Get a sample of non-domain entries, limited to 5 (the list should already have been de-duplicated) IFS=" " read -r -a sample_non_domains <<< "$(tr ' ' '\n' <<< "${non_domains[@]}" | head -n 5 | tr '\n' ' ')" - local tmp_new_imported_total - # Get the new number of domains in destination file - tmp_new_imported_total="$(grep -c "^" "${target}")" - # Number of imported lines for this file is the difference between the new total and the old total. (Or, the number of domains we just added.) - num_domains="$(( tmp_new_imported_total-num_total_imported_domains ))" - # Replace the running total with the new total. - num_total_imported_domains="$tmp_new_imported_total" + # Get the number of domains added + num_domains="$(grep -c "^" "${temp_file}")" # Get the number of non_domains (this is the number of entries left after stripping the source of comments/duplicates/false positives/domains) num_non_domains="${#non_domains[@]}" @@ -591,6 +616,9 @@ parseList() { else echo " ${INFO} Imported ${num_domains} domains" fi + + # close file handle + exec 3<&- } compareLists() { @@ -623,7 +651,7 @@ gravity_DownloadBlocklistFromUrl() { local heisenbergCompensator="" patternBuffer str httpCode success="" ip # Create temp file to store content on disk instead of RAM - patternBuffer=$(mktemp -p "/tmp" --suffix=".phgpb") + patternBuffer=$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".phgpb") # Determine if $saveLocation has read permission if [[ -r "${saveLocation}" && $url != "file"* ]]; then @@ -761,18 +789,30 @@ gravity_ParseFileIntoDomains() { # Most of the lists downloaded are already in hosts file format but the spacing/formatting is not contiguous # This helps with that and makes it easier to read # It also helps with debugging so each stage of the script can be researched more in depth - # 1) Remove carriage returns - # 2) Convert all characters to lowercase - # 3) Remove comments (text starting with "#", include possible spaces before the hash sign) + # 1) Convert all characters to lowercase + tr '[:upper:]' '[:lower:]' < "${src}" > "${destination}" + + # 2) Remove carriage returns + sed -i 's/\r$//' "${destination}" + + # 3a) Remove comments (text starting with "#", include possible spaces before the hash sign) + sed -i 's/\s*#.*//g' "${destination}" + + # 3b) Remove lines starting with ! (ABP Comments) + sed -i 's/\s*!.*//g' "${destination}" + + # 3c) Remove lines starting with [ (ABP Header) + sed -i 's/\s*\[.*//g' "${destination}" + # 4) Remove lines containing "/" - # 5) Remove leading tabs, spaces, etc. + sed -i -r '/(\/).*$/d' "${destination}" + + # 5) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) + sed -i -r 's/^.*\s+//g' "${destination}" + # 6) Remove empty lines - < "${src}" tr -d '\r' | \ - tr '[:upper:]' '[:lower:]' | \ - sed 's/\s*#.*//g' | \ - sed -r '/(\/).*$/d' | \ - sed -r 's/^.*\s+//g' | \ - sed '/^$/d'> "${destination}" + sed -i '/^$/d' "${destination}" + chmod 644 "${destination}" } @@ -828,7 +868,7 @@ gravity_Cleanup() { # Delete tmp content generated by Gravity rm ${piholeDir}/pihole.*.txt 2> /dev/null rm ${piholeDir}/*.tmp 2> /dev/null - rm /tmp/*.phgpb 2> /dev/null + rm "${GRAVITY_TMPDIR}"/*.phgpb 2> /dev/null # Ensure this function only runs when gravity_SetDownloadOptions() has completed if [[ "${gravity_Blackbody:-}" == true ]]; then @@ -1005,6 +1045,9 @@ fi # Update gravity timestamp update_gravity_timestamp +# Set abp_domain info field +set_abp_info + # Ensure proper permissions are set for the database chown pihole:pihole "${gravityDBfile}" chmod g+w "${piholeDir}" "${gravityDBfile}" diff --git a/test/requirements.txt b/test/requirements.txt index 08e7027a..8ba6156f 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ docker-compose == 1.29.2 -pytest == 7.2.1 -pytest-xdist == 3.1.0 +pytest == 7.2.2 +pytest-xdist == 3.2.1 pytest-testinfra == 7.0.0 -tox == 4.4.4 +tox == 4.4.7 diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 5b4075d9..cb5ddeaa 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -40,6 +40,26 @@ def test_key_addition_works(host): assert expected_stdout == output.stdout +def test_key_addition_substr(host): + """Confirms addKey adds substring keys (no value) to a file""" + host.run( + """ + source /opt/pihole/utils.sh + addKey "./testoutput" "KEY_ONE" + addKey "./testoutput" "KEY_O" + addKey "./testoutput" "KEY_TWO" + addKey "./testoutput" "Y_TWO" + """ + ) + output = host.run( + """ + cat ./testoutput + """ + ) + expected_stdout = "KEY_ONE\nKEY_O\nKEY_TWO\nY_TWO\n" + assert expected_stdout == output.stdout + + def test_key_removal_works(host): """Confirms removeKey removes a key or key/value pair""" host.run( @@ -62,6 +82,22 @@ def test_key_removal_works(host): assert expected_stdout == output.stdout +def test_get_value_works(host): + """Confirms getVal returns the correct value for a given key""" + output = host.run( + """ + source /opt/pihole/utils.sh + echo "Somekey=xxx" >> /tmp/testfile + echo "#Testkey=1234" >> /tmp/testfile + echo "Testkey=5678" >> /tmp/testfile + echo "Testkey=abcd" >> /tmp/testfile + getVal "/tmp/testfile" "Testkey" + """ + ) + expected_stdout = "5678" + assert expected_stdout == output.stdout + + def test_getFTLAPIPort_default(host): """Confirms getFTLAPIPort returns the default API port""" output = host.run( diff --git a/test/tox.centos_8.ini b/test/tox.centos_8.ini index dac10e97..dca77c93 100644 --- a/test/tox.centos_8.ini +++ b/test/tox.centos_8.ini @@ -4,5 +4,5 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt -commands = docker build -f _centos_8.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _centos_8.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py diff --git a/test/tox.centos_9.ini b/test/tox.centos_9.ini index aa7009e1..a69c336a 100644 --- a/test/tox.centos_9.ini +++ b/test/tox.centos_9.ini @@ -4,5 +4,5 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt -commands = docker build -f _centos_9.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _centos_9.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py diff --git a/test/tox.debian_10.ini b/test/tox.debian_10.ini index a012bda4..f107300f 100644 --- a/test/tox.debian_10.ini +++ b/test/tox.debian_10.ini @@ -4,5 +4,5 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt -commands = docker build -f _debian_10.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _debian_10.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.debian_11.ini b/test/tox.debian_11.ini index 48dc9df1..c38a15fb 100644 --- a/test/tox.debian_11.ini +++ b/test/tox.debian_11.ini @@ -4,5 +4,5 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt -commands = docker build -f _debian_11.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _debian_11.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.fedora_36.ini b/test/tox.fedora_36.ini index 0cc6f29c..515487ed 100644 --- a/test/tox.fedora_36.ini +++ b/test/tox.fedora_36.ini @@ -4,5 +4,5 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt -commands = docker build -f _fedora_36.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _fedora_36.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py diff --git a/test/tox.fedora_37.ini b/test/tox.fedora_37.ini index d6f44533..2a8ef398 100644 --- a/test/tox.fedora_37.ini +++ b/test/tox.fedora_37.ini @@ -4,5 +4,5 @@ envlist = py3 [testenv] allowlist_externals = docker deps = -rrequirements.txt -commands = docker build -f _fedora_37.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _fedora_37.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py diff --git a/test/tox.ubuntu_20.ini b/test/tox.ubuntu_20.ini index 88ee0b54..49a6153e 100644 --- a/test/tox.ubuntu_20.ini +++ b/test/tox.ubuntu_20.ini @@ -4,5 +4,5 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt -commands = docker build -f _ubuntu_20.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _ubuntu_20.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.ubuntu_22.ini b/test/tox.ubuntu_22.ini index cb5527ab..8014d6d6 100644 --- a/test/tox.ubuntu_22.ini +++ b/test/tox.ubuntu_22.ini @@ -4,5 +4,5 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt -commands = docker build -f _ubuntu_22.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _ubuntu_22.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py