Remove possible duplicates found in lower-quality adlists

Signed-off-by: DL6ER <dl6er@dl6er.de>
5 years ago · bd1b004d94
parent 5457b2c6ea
commit bd1b004d94
1 changed files with 3 additions and 2 deletions
--- a/gravity.sh
+++ b/gravity.sh
@ -160,10 +160,11 @@ database_table_from_file() {
      fi
    done
  fi
-
-
  inputfile="${tmpFile}"

+  # Remove possible duplicates found in lower-quality adlists
+  uniq -u "${inputfile}" "${inputfile}"
+
  # Store domains in database table specified by ${table}
  # Use printf as .mode and .import need to be on separate lines
  # see https://unix.stackexchange.com/a/445615/83260