From 3aa838bbe47aee2f4ec804873fc2b267422d8f3c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 May 2019 12:47:25 +0200 Subject: [PATCH 1/7] Implement black- and whitelist searching with SQL statements. We use the ESCAPE clause in the LIKE query as the underscore "_" wildcard matches any single character but we want to suppress this behavior (underscores can be legitimate part of domains) Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 70 +++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 21 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index a4ac895b..9b7a0fab 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -102,29 +102,57 @@ if [[ -n "${str:-}" ]]; then exit 1 fi -# Scan Whitelist and Blacklist -lists="whitelist.txt blacklist.txt" -mapfile -t results <<< "$(scanList "${domainQuery}" "${lists}" "${exact}")" -if [[ -n "${results[*]}" ]]; then - wbMatch=true - # Loop through each result in order to print unique file title once - for result in "${results[@]}"; do - fileName="${result%%.*}" - if [[ -n "${blockpage}" ]]; then - echo "π ${result}" - exit 0 - elif [[ -n "${exact}" ]]; then - echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" - else - # Only print filename title once per file - if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then +scanDatabaseTable() { + local domain table type querystr result + domain="${1}" + table="${2}" + type="${3:-}" + + # As underscores are legitimate parts of domains, we escape possible them when using the LIKE operator. + # Underscores are a SQLite wildcard matching exactly one character. We obviously want to suppress this + # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched + # as a literal underscore character. + case "${type}" in + "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";; + * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\'";; + esac + + # Send prepared query to gravity database + result="$(sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null + if [[ -n "${result}" ]]; then + # Prepend listname (separated by a colon) if we found at least one result + # and output result + results="$(sed "s/^/${table}:/g;" <<< "${result}")" + else + # Output empty string as the database query didn't return any result + return + fi + mapfile -t results <<< "${results}" + if [[ -n "${results[*]}" ]]; then + wbMatch=true + # Loop through each result in order to print unique file title once + for result in "${results[@]}"; do + fileName="${result%%:*}" + if [[ -n "${blockpage}" ]]; then + echo "π ${result}" + exit 0 + elif [[ -n "${exact}" ]]; then echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" - fileName_prev="${fileName}" + else + # Only print filename title once per file + if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then + echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" + fileName_prev="${fileName}" + fi + echo " ${result#*:}" fi - echo " ${result#*:}" - fi - done -fi + done + fi +} + +# Scan Whitelist and Blacklist +scanDatabaseTable "${domainQuery}" "whitelist" "${exact}" +scanDatabaseTable "${domainQuery}" "blacklist" "${exact}" # Scan Wildcards if [[ -e "${wildcardlist}" ]]; then From 6ba58896d23c5ac7bf497413192a1fdd225e03b1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 May 2019 13:15:30 +0200 Subject: [PATCH 2/7] Simplify code Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 48 +++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 9b7a0fab..66b55e87 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -103,7 +103,7 @@ if [[ -n "${str:-}" ]]; then fi scanDatabaseTable() { - local domain table type querystr result + local domain table type querystr result table_prev domain="${1}" table="${2}" type="${3:-}" @@ -119,35 +119,29 @@ scanDatabaseTable() { # Send prepared query to gravity database result="$(sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null - if [[ -n "${result}" ]]; then - # Prepend listname (separated by a colon) if we found at least one result - # and output result - results="$(sed "s/^/${table}:/g;" <<< "${result}")" - else - # Output empty string as the database query didn't return any result + if [[ -z "${result}" ]]; then + # Return early when we have no results return fi - mapfile -t results <<< "${results}" - if [[ -n "${results[*]}" ]]; then - wbMatch=true - # Loop through each result in order to print unique file title once - for result in "${results[@]}"; do - fileName="${result%%:*}" - if [[ -n "${blockpage}" ]]; then - echo "π ${result}" - exit 0 - elif [[ -n "${exact}" ]]; then - echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" - else - # Only print filename title once per file - if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then - echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" - fileName_prev="${fileName}" - fi - echo " ${result#*:}" + + wbMatch=true + mapfile -t results <<< "${result}" + # Loop through each result + for result in "${results[@]}"; do + if [[ -n "${blockpage}" ]]; then + echo "π ${result}" + exit 0 + elif [[ -n "${exact}" ]]; then + echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}" + else + # Only print table name once + if [[ ! "${table}" == "${table_prev:-}" ]]; then + echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}" + table_prev="${table}" fi - done - fi + echo " ${result}" + fi + done } # Scan Whitelist and Blacklist From f80fdd7e83b4bb23edb1671316f986b9ed791b68 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 May 2019 13:19:50 +0200 Subject: [PATCH 3/7] Improve comments Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 66b55e87..a7cc9dfb 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -108,10 +108,10 @@ scanDatabaseTable() { table="${2}" type="${3:-}" - # As underscores are legitimate parts of domains, we escape possible them when using the LIKE operator. - # Underscores are a SQLite wildcard matching exactly one character. We obviously want to suppress this + # As underscores are legitimate parts of domains, we escape them when using the LIKE operator. + # Underscores are SQLite wildcards matching exactly one character. We obviously want to suppress this # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched - # as a literal underscore character. + # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores. case "${type}" in "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";; * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\'";; @@ -120,13 +120,14 @@ scanDatabaseTable() { # Send prepared query to gravity database result="$(sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null if [[ -z "${result}" ]]; then - # Return early when we have no results + # Return early when there are no matches in this table return fi + # Mark domain as having been white-/blacklist matched (global variable) wbMatch=true - mapfile -t results <<< "${result}" # Loop through each result + mapfile -t results <<< "${result}" for result in "${results[@]}"; do if [[ -n "${blockpage}" ]]; then echo "π ${result}" From 5246b3e49672edb7e397418160ca2c327cef0ccc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 May 2019 13:24:36 +0200 Subject: [PATCH 4/7] Explicitly escape backslash in ESCAPE clause. This has been suggested by Stickler bot. Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index a7cc9dfb..d4de380c 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -114,7 +114,7 @@ scanDatabaseTable() { # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores. case "${type}" in "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";; - * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\'";; + * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; esac # Send prepared query to gravity database From a904c183dfc2de7a503466f5ff39d1af3a4d1aca Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 May 2019 18:25:11 +0200 Subject: [PATCH 5/7] Use printf to escape domain content. This prevents possible SQL injection issues Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index d4de380c..d2e16e79 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -104,7 +104,7 @@ fi scanDatabaseTable() { local domain table type querystr result table_prev - domain="${1}" + domain="$(printf "%q" "${1}")" table="${2}" type="${3:-}" From 5060605626950007aa0fc35153356ea49c96a187 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 30 May 2019 21:44:47 +0200 Subject: [PATCH 6/7] Print table name before entering the loop for the sake of simplicity Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index d2e16e79..88ec2173 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -103,7 +103,7 @@ if [[ -n "${str:-}" ]]; then fi scanDatabaseTable() { - local domain table type querystr result table_prev + local domain table type querystr result domain="$(printf "%q" "${1}")" table="${2}" type="${3:-}" @@ -126,22 +126,20 @@ scanDatabaseTable() { # Mark domain as having been white-/blacklist matched (global variable) wbMatch=true - # Loop through each result + + # Print table name + if [[ ! -z "${result}" ]]; then + echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}" + fi + + # Loop over results and print them mapfile -t results <<< "${result}" for result in "${results[@]}"; do if [[ -n "${blockpage}" ]]; then echo "π ${result}" exit 0 - elif [[ -n "${exact}" ]]; then - echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}" - else - # Only print table name once - if [[ ! "${table}" == "${table_prev:-}" ]]; then - echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}" - table_prev="${table}" - fi - echo " ${result}" fi + echo " ${result}" done } From 44f8fcb600cc78f7b4dd8c6c9c8bba73c64c718c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 31 May 2019 08:39:18 +0200 Subject: [PATCH 7/7] We can print the table name without any extra check as we already returned early in case there are no results. Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 88ec2173..9cb298df 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -128,9 +128,7 @@ scanDatabaseTable() { wbMatch=true # Print table name - if [[ ! -z "${result}" ]]; then - echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}" - fi + echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}" # Loop over results and print them mapfile -t results <<< "${result}"