From 164a81776eb055f98418f6eafcfe17f9333be2b2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Nov 2017 18:30:40 +0100 Subject: [PATCH 1/6] Allow to add local lists to gravity using e.g. file:///path/to/my.list in adlists.list Signed-off-by: DL6ER --- gravity.sh | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/gravity.sh b/gravity.sh index f4b5fc36..453985e6 100755 --- a/gravity.sh +++ b/gravity.sh @@ -138,8 +138,9 @@ gravity_Collapse() { # Logic: Split by folder/port awk -F '[/:]' '{ # Remove URL protocol & optional username:password@ - gsub(/(.*:\/\/|.*:.*@)/, "", $0) - print $1 + gsub(/(.*:\/\/|)/, "", $0) + if(length($1)>0){print $1} + else {print "local"} }' <<< "$(printf '%s\n' "${sources[@]}")" 2> /dev/null )" @@ -203,20 +204,27 @@ gravity_Pull() { # shellcheck disable=SC2086 httpCode=$(curl -s -L ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${patternBuffer}" 2> /dev/null) - # Determine "Status:" output based on HTTP response - case "${httpCode}" in - "200") echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true;; - "304") echo -e "${OVER} ${TICK} ${str} No changes detected"; success=true;; - "000") echo -e "${OVER} ${CROSS} ${str} Connection Refused";; - "403") echo -e "${OVER} ${CROSS} ${str} Forbidden";; - "404") echo -e "${OVER} ${CROSS} ${str} Not found";; - "408") echo -e "${OVER} ${CROSS} ${str} Time-out";; - "451") echo -e "${OVER} ${CROSS} ${str} Unavailable For Legal Reasons";; - "500") echo -e "${OVER} ${CROSS} ${str} Internal Server Error";; - "504") echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Gateway)";; - "521") echo -e "${OVER} ${CROSS} ${str} Web Server Is Down (Cloudflare)";; - "522") echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Cloudflare)";; - * ) echo -e "${OVER} ${CROSS} ${str} ${httpCode}";; + case $url in + # Did we "download" a remote file? + "http"*) + # Determine "Status:" output based on HTTP response + case "${httpCode}" in + "200") echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true;; + "304") echo -e "${OVER} ${TICK} ${str} No changes detected"; success=true;; + "000") echo -e "${OVER} ${CROSS} ${str} Connection Refused";; + "403") echo -e "${OVER} ${CROSS} ${str} Forbidden";; + "404") echo -e "${OVER} ${CROSS} ${str} Not found";; + "408") echo -e "${OVER} ${CROSS} ${str} Time-out";; + "451") echo -e "${OVER} ${CROSS} ${str} Unavailable For Legal Reasons";; + "500") echo -e "${OVER} ${CROSS} ${str} Internal Server Error";; + "504") echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Gateway)";; + "521") echo -e "${OVER} ${CROSS} ${str} Web Server Is Down (Cloudflare)";; + "522") echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Cloudflare)";; + * ) echo -e "${OVER} ${CROSS} ${str} ${httpCode}";; + esac;; + # Did we "download" a local file? + "file"*) echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true;; + * ) echo -e "${OVER} ${CROSS} ${str} ${url} ${httpCode}";; esac # Determine if the blocklist was downloaded and saved correctly @@ -229,7 +237,7 @@ gravity_Pull() { gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}" else # Fall back to previously cached list if $patternBuffer is empty - echo -e " ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" + echo -e " ${INFO} ${COL_LIGHT_GREEN}Using previously cached list${COL_NC}" fi else # Determine if cached list has read permission @@ -407,7 +415,7 @@ gravity_Filter() { # Whitelist unique blocklist domain sources gravity_WhitelistBLD() { - local uniqDomains plural="" str + local uniqDomains plural="" str echo "" From 8976930e20fcabe529c544498db442ef04f824fb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Nov 2017 18:35:58 +0100 Subject: [PATCH 2/6] Properly detect if local file was successfully downloaded Signed-off-by: DL6ER --- gravity.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index 453985e6..f563a442 100755 --- a/gravity.sh +++ b/gravity.sh @@ -193,9 +193,10 @@ gravity_Pull() { patternBuffer=$(mktemp -p "/tmp" --suffix=".phgpb") # Determine if $saveLocation has read permission - if [[ -r "${saveLocation}" ]]; then + if [[ -r "${saveLocation}" && $url != "file"* ]]; then # Have curl determine if a remote file has been modified since last retrieval # Uses "Last-Modified" header, which certain web servers do not provide (e.g: raw github urls) + # Note: Don't do this for local files, always download them heisenbergCompensator="-z ${saveLocation}" fi @@ -223,7 +224,12 @@ gravity_Pull() { * ) echo -e "${OVER} ${CROSS} ${str} ${httpCode}";; esac;; # Did we "download" a local file? - "file"*) echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true;; + "file"*) + if [[ -s "${patternBuffer}" ]]; then + echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true + else + echo -e "${OVER} ${CROSS} ${str} Not found" + fi;; * ) echo -e "${OVER} ${CROSS} ${str} ${url} ${httpCode}";; esac @@ -237,7 +243,7 @@ gravity_Pull() { gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}" else # Fall back to previously cached list if $patternBuffer is empty - echo -e " ${INFO} ${COL_LIGHT_GREEN}Using previously cached list${COL_NC}" + echo -e " ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" fi else # Determine if cached list has read permission From eb83081a5c4f8b06cbbbd893ef4d95032c8d7e90 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Nov 2017 18:36:34 +0100 Subject: [PATCH 3/6] Clarify error message: File could also have been of zero size Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index f563a442..946dd820 100755 --- a/gravity.sh +++ b/gravity.sh @@ -228,7 +228,7 @@ gravity_Pull() { if [[ -s "${patternBuffer}" ]]; then echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true else - echo -e "${OVER} ${CROSS} ${str} Not found" + echo -e "${OVER} ${CROSS} ${str} Not found / empty list" fi;; * ) echo -e "${OVER} ${CROSS} ${str} ${url} ${httpCode}";; esac From c54f04ef4cbb3b850d4f96bc0ca6576d6893c2cc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Nov 2017 20:55:47 +0100 Subject: [PATCH 4/6] Repair username:password filtering Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 946dd820..430b38f8 100755 --- a/gravity.sh +++ b/gravity.sh @@ -138,7 +138,7 @@ gravity_Collapse() { # Logic: Split by folder/port awk -F '[/:]' '{ # Remove URL protocol & optional username:password@ - gsub(/(.*:\/\/|)/, "", $0) + gsub(/(.*:\/\/|.*:.*@)/, "", $0) if(length($1)>0){print $1} else {print "local"} }' <<< "$(printf '%s\n' "${sources[@]}")" 2> /dev/null From 6f0bb30def2456c82a2b77ff97164b466c59ca1f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Nov 2017 20:58:27 +0100 Subject: [PATCH 5/6] Improve indentation Signed-off-by: DL6ER --- gravity.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 430b38f8..8e666e35 100755 --- a/gravity.sh +++ b/gravity.sh @@ -230,7 +230,8 @@ gravity_Pull() { else echo -e "${OVER} ${CROSS} ${str} Not found / empty list" fi;; - * ) echo -e "${OVER} ${CROSS} ${str} ${url} ${httpCode}";; + * ) + echo -e "${OVER} ${CROSS} ${str} ${url} ${httpCode}";; esac # Determine if the blocklist was downloaded and saved correctly From 8d721d086cbe4b49665c9e0b1d81499b284776a9 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Tue, 21 Nov 2017 18:37:38 -0500 Subject: [PATCH 6/6] Modify indentation --- gravity.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 8e666e35..eb69b49f 100755 --- a/gravity.sh +++ b/gravity.sh @@ -230,8 +230,7 @@ gravity_Pull() { else echo -e "${OVER} ${CROSS} ${str} Not found / empty list" fi;; - * ) - echo -e "${OVER} ${CROSS} ${str} ${url} ${httpCode}";; + *) echo -e "${OVER} ${CROSS} ${str} ${url} ${httpCode}";; esac # Determine if the blocklist was downloaded and saved correctly