From 839fe32042982b1d7109b2c8e3b818da0f9b4118 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 21 Feb 2020 19:56:48 +0100 Subject: [PATCH] Fix issue with missing newline at the end of adlists (#3144) * Also display non-fatal warnings during the database importing. Previously, we have only show warnings when there were also errors (errors are always fatal). Signed-off-by: DL6ER * Ensure there is always a newline on the last line. Signed-off-by: DL6ER * Stickler linting Signed-off-by: Dan Schaper * Move sed command into subroutine to avoid code duplication. Signed-off-by: DL6ER * Also unify comments. Signed-off-by: DL6ER * Also unify comments. Signed-off-by: DL6ER Co-authored-by: Dan Schaper --- gravity.sh | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/gravity.sh b/gravity.sh index bc2cace3..b9abd83d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -390,12 +390,23 @@ gravity_DownloadBlocklists() { echo -e "${OVER} ${TICK} ${str}" fi + if [[ "${status}" -eq 0 && -n "${output}" ]]; then + echo -e " Encountered non-critical SQL warnings. Please check the suitability of the list you're using!\\nSQL warnings:\\n${output}\\n" + fi + rm "${target}" > /dev/null 2>&1 || \ echo -e " ${CROSS} Unable to remove ${target}" gravity_Blackbody=true } +parseList() { + local adlistID="${1}" src="${2}" target="${3}" + #Append ,${arg} to every line and then remove blank lines before import + # /.$/a\\ ensures there is a newline on the last line + sed -e "s/$/,${adlistID}/;/^$/d;/.$/a\\" "${src}" >> "${target}" +} + # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { local url="${1}" cmd_ext="${2}" agent="${3}" adlistID="${4}" saveLocation="${5}" target="${6}" @@ -481,14 +492,13 @@ gravity_DownloadBlocklistFromUrl() { if [[ "${success}" == true ]]; then if [[ "${httpCode}" == "304" ]]; then # Add domains to database table file - #Append ,${arg} to every line and then remove blank lines before import - sed -e "s/$/,${adlistID}/;/^$/d" "${saveLocation}" >> "${target}" + parseList "${adlistID}" "${saveLocation}" "${target}" # Check if $patternbuffer is a non-zero length file elif [[ -s "${patternBuffer}" ]]; then # Determine if blocklist is non-standard and parse as appropriate gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}" - #Append ,${arg} to every line and then remove blank lines before import - sed -e "s/$/,${adlistID}/;/^$/d" "${saveLocation}" >> "${target}" + # Add domains to database table file + parseList "${adlistID}" "${saveLocation}" "${target}" else # Fall back to previously cached list if $patternBuffer is empty echo -e " ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" @@ -497,8 +507,8 @@ gravity_DownloadBlocklistFromUrl() { # Determine if cached list has read permission if [[ -r "${saveLocation}" ]]; then echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" - #Append ,${arg} to every line and then remove blank lines before import - sed -e "s/$/,${adlistID}/;/^$/d" "${saveLocation}" >> "${target}" + # Add domains to database table file + parseList "${adlistID}" "${saveLocation}" "${target}" else echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" fi