From bd1b004d9417b6578c7673fd5fc4f391534230a4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 12 Dec 2019 10:13:51 +0000 Subject: [PATCH 1/2] Remove possible duplicates found in lower-quality adlists Signed-off-by: DL6ER --- gravity.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 1d9919a1..58d60ff4 100755 --- a/gravity.sh +++ b/gravity.sh @@ -160,10 +160,11 @@ database_table_from_file() { fi done fi - - inputfile="${tmpFile}" + # Remove possible duplicates found in lower-quality adlists + uniq -u "${inputfile}" "${inputfile}" + # Store domains in database table specified by ${table} # Use printf as .mode and .import need to be on separate lines # see https://unix.stackexchange.com/a/445615/83260 From 570a7a5c11ab6dead9b91d62beb79ab924cb626d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 12 Dec 2019 10:17:54 +0000 Subject: [PATCH 2/2] Use sort -u instead of uniq as it is guaranteed to be safe when doing inline file operations. Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 58d60ff4..49a4fc1c 100755 --- a/gravity.sh +++ b/gravity.sh @@ -163,7 +163,7 @@ database_table_from_file() { inputfile="${tmpFile}" # Remove possible duplicates found in lower-quality adlists - uniq -u "${inputfile}" "${inputfile}" + sort -u -o "${inputfile}" "${inputfile}" # Store domains in database table specified by ${table} # Use printf as .mode and .import need to be on separate lines