Rewrite list.sh to operate on database-based lists

Signed-off-by: DL6ER <dl6er@dl6er.de>

advanced/Scripts/list.sh

@@ -11,10 +11,8 @@
 # Globals
 basename=pihole
 piholeDir=/etc/"${basename}"
-whitelist="${piholeDir}"/whitelist.txt
+gravityDBfile="${piholeDir}/gravity.db"
-blacklist="${piholeDir}"/blacklist.txt
 
-readonly regexlist="/etc/pihole/regex.list"
 reload=false
 addmode=true
 verbose=true
@@ -22,35 +20,34 @@ wildcard=false
 
 domList=()
 
-listMain=""
+listType=""
-listAlt=""
 
 colfile="/opt/pihole/COL_TABLE"
 source ${colfile}
 
 
 helpFunc() {
-    if [[ "${listMain}" == "${whitelist}" ]]; then
+    if [[ "${listType}" == "whitelist" ]]; then
         param="w"
-        type="white"
+        type="whitelist"
-    elif [[ "${listMain}" == "${regexlist}" && "${wildcard}" == true ]]; then
+    elif [[ "${listType}" == "regex" && "${wildcard}" == true ]]; then
         param="-wild"
-        type="wildcard black"
+        type="wildcard blacklist"
-    elif [[ "${listMain}" == "${regexlist}" ]]; then
+    elif [[ "${listType}" == "regex" ]]; then
         param="-regex"
-        type="regex black"
+        type="regex filter"
     else
         param="b"
-        type="black"
+        type="blacklist"
     fi
 
     echo "Usage: pihole -${param} [options] <domain> <domain2 ...>
 Example: 'pihole -${param} site.com', or 'pihole -${param} site1.com site2.com'
-${type^}list one or more domains
+${type^} one or more domains
 
 Options:
-  -d, --delmode       Remove domain(s) from the ${type}list
+  -d, --delmode       Remove domain(s) from the ${type}
-  -nr, --noreload     Update ${type}list without refreshing dnsmasq
+  -nr, --noreload     Update ${type} without reloading the DNS server
   -q, --quiet         Make output less verbose
   -h, --help          Show this help dialog
   -l, --list          Display all your ${type}listed domains
@@ -73,7 +70,7 @@ HandleOther() {
 
     # Check validity of domain (don't check for regex entries)
     if [[ "${#domain}" -le 253 ]]; then
-        if [[ "${listMain}" == "${regexlist}" && "${wildcard}" == false ]]; then
+        if [[ "${listType}" == "regex" && "${wildcard}" == false ]]; then
             validDomain="${domain}"
         else
             validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check
@@ -88,169 +85,134 @@ HandleOther() {
     fi
 }
 
-PoplistFile() {
+ProcessDomainList() {
-    # Check whitelist file exists, and if not, create it
-    if [[ ! -f "${whitelist}" ]]; then
-        touch "${whitelist}"
-    fi
-
-    # Check blacklist file exists, and if not, create it
-    if [[ ! -f "${blacklist}" ]]; then
-        touch "${blacklist}"
-    fi
-
     for dom in "${domList[@]}"; do
         # Logic: If addmode then add to desired list and remove from the other; if delmode then remove from desired list but do not add to the other
         if ${addmode}; then
-            AddDomain "${dom}" "${listMain}"
+            AddDomain "${dom}" "${listType}"
             RemoveDomain "${dom}" "${listAlt}"
         else
-            RemoveDomain "${dom}" "${listMain}"
+            RemoveDomain "${dom}" "${listType}"
         fi
   done
 }
 
 AddDomain() {
+    local domain list listname sqlitekey num
+    domain="$1"
     list="$2"
-    domain=$(EscapeRegexp "$1")
 
-    [[ "${list}" == "${whitelist}" ]] && listname="whitelist"
+    if [[ "${list}" == "regex" ]]; then
-    [[ "${list}" == "${blacklist}" ]] && listname="blacklist"
+        listname="regex filters"
-
+        sqlitekey="filter"
-    if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then
+        if [[ "${wildcard}" == true ]]; then
-        [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only"
+            domain="(^|\\.)${domain//\./\\.}$"
-        [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only"
-        bool=true
-        # Is the domain in the list we want to add it to?
-        grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false
-
-        if [[ "${bool}" == false ]]; then
-            # Domain not found in the whitelist file, add it!
-            if [[ "${verbose}" == true ]]; then
-                echo -e "  ${INFO} Adding ${1} to ${listname}..."
-            fi
-            reload=true
-            # Add it to the list we want to add it to
-            echo "$1" >> "${list}"
-        else
-            if [[ "${verbose}" == true ]]; then
-                echo -e "  ${INFO} ${1} already exists in ${listname}, no need to add!"
-            fi
         fi
-    elif [[ "${list}" == "${regexlist}" ]]; then
+    else
-        [[ -z "${type}" ]] && type="--wildcard-only"
+        # Whitelist / Blacklist
-        bool=true
+        listname="${list}list"
-        domain="${1}"
+        sqlitekey="domain"
+    fi
 
-        [[ "${wildcard}" == true ]] && domain="(^|\\.)${domain//\./\\.}$"
+    # Is the domain in the list we want to add it to?
+    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")"
 
-        # Is the domain in the list?
+    if [[ "${num}" -eq 0 ]]; then
-        # Search only for exactly matching lines
+        # Domain not found in the file, add it!
-        grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false
+        if [[ "${verbose}" == true ]]; then
-
+            echo -e "  ${INFO} Adding ${1} to ${listname}..."
-        if [[ "${bool}" == false ]]; then
+        fi
-            if [[ "${verbose}" == true ]]; then
+        reload=true
-                echo -e "  ${INFO} Adding ${domain} to regex list..."
+        # Add it to the list we want to add it to
-            fi
+        local timestamp
-            reload="restart"
+        timestamp="$(date --utc +'%s')"
-            echo "$domain" >> "${regexlist}"
+        sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (${sqlitekey},enabled,date_added) VALUES (\"${domain}\",1,${timestamp});"
-        else
+    else
-            if [[ "${verbose}" == true ]]; then
+        if [[ "${verbose}" == true ]]; then
-                echo -e "  ${INFO} ${domain} already exists in regex list, no need to add!"
+            echo -e "  ${INFO} ${1} already exists in ${listname}, no need to add!"
-            fi
         fi
     fi
 }
 
 RemoveDomain() {
+    local domain list listname sqlitekey num
+    domain="$1"
     list="$2"
-    domain=$(EscapeRegexp "$1")
 
-    [[ "${list}" == "${whitelist}" ]] && listname="whitelist"
+    if [[ "${list}" == "regex" ]]; then
-    [[ "${list}" == "${blacklist}" ]] && listname="blacklist"
+        listname="regex filters"
-
+        sqlitekey="filter"
-    if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then
+        if [[ "${wildcard}" == true ]]; then
-        bool=true
+            domain="(^|\\.)${domain//\./\\.}$"
-        [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only"
-        [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only"
-        # Is it in the list? Logic follows that if its whitelisted it should not be blacklisted and vice versa
-        grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false
-        if [[ "${bool}" == true ]]; then
-            # Remove it from the other one
-            echo -e "  ${INFO} Removing $1 from ${listname}..."
-            # /I flag: search case-insensitive
-            sed -i "/${domain}/Id" "${list}"
-            reload=true
-        else
-            if [[ "${verbose}" == true ]]; then
-                echo -e "  ${INFO} ${1} does not exist in ${listname}, no need to remove!"
-            fi
         fi
-    elif [[ "${list}" == "${regexlist}" ]]; then
+    else
-        [[ -z "${type}" ]] && type="--wildcard-only"
+        # Whitelist / Blacklist
-        domain="${1}"
+        listname="${list}list"
+        sqlitekey="domain"
+    fi
 
-        [[ "${wildcard}" == true ]] && domain="(^|\\.)${domain//\./\\.}$"
+    # Is the domain in the list we want to remove it from?
+    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")"
 
-        bool=true
+    if [[ "${num}" -ne 0 ]]; then
-        # Is it in the list?
+        # Domain found in the file, remove it!
-        grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false
+        if [[ "${verbose}" == true ]]; then
-        if [[ "${bool}" == true ]]; then
+            echo -e "  ${INFO} Removing ${1} from ${listname}..."
-            # Remove it from the other one
+        fi
-            echo -e "  ${INFO} Removing $domain from regex list..."
+        reload=true
-            local lineNumber
+        # Remove it from the current list
-            lineNumber=$(grep -Fnx "$domain" "${list}" | cut -f1 -d:)
+        local timestamp
-            sed -i "${lineNumber}d" "${list}"
+        timestamp="$(date --utc +'%s')"
-            reload=true
+        sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE ${sqlitekey} = \"${domain}\";"
-        else
+    else
-            if [[ "${verbose}" == true ]]; then
+        if [[ "${verbose}" == true ]]; then
-                echo -e "  ${INFO} ${domain} does not exist in regex list, no need to remove!"
+            echo -e "  ${INFO} ${1} does not exist in ${listname}, no need to remove!"
-            fi
         fi
     fi
 }
 
-# Update Gravity
-Reload() {
-    echo ""
-    pihole -g --skip-download "${type:-}"
-}
-
 Displaylist() {
-    if [[ -f ${listMain} ]]; then
+    local domain list listname count status
-        if [[ "${listMain}" == "${whitelist}" ]]; then
+
-            string="gravity resistant domains"
+    if [[ "${listType}" == "regex" ]]; then
-        else
+        listname="regex filters list"
-            string="domains caught in the sinkhole"
-        fi
-        verbose=false
-        echo -e "Displaying $string:\n"
-        count=1
-        while IFS= read -r RD || [ -n "${RD}" ]; do
-            echo "  ${count}: ${RD}"
-            count=$((count+1))
-        done < "${listMain}"
     else
-        echo -e "  ${COL_LIGHT_RED}${listMain} does not exist!${COL_NC}"
+        # Whitelist / Blacklist
+        listname="${listType}"
+    fi
+    data="$(sqlite3 "${gravityDBfile}" "SELECT * FROM ${listType};" 2> /dev/null)"
+
+    if [[ -z $data ]]; then
+        echo -e "Not showing empty ${listname}"
+    else
+        echo -e "Displaying ${listname}:"
+        count=1
+        while IFS= read -r line
+        do
+            domain="$(cut -d'|' -f1 <<< "${line}")"
+            enabled="$(cut -d'|' -f2 <<< "${line}")"
+            if [[ "${enabled}" -eq 1 ]]; then
+                status="enabled"
+            else
+                status="disabled"
+            fi
+            echo "  ${count}: ${domain} (${status})"
+            count=$((count+1))
+        done <<< "${data}"
+        exit 0;
     fi
-    exit 0;
 }
 
 NukeList() {
-    if [[ -f "${listMain}" ]]; then
+    sqlite3 "${gravityDBfile}" "DELETE FROM ${listType};"
-        # Back up original list
-        cp "${listMain}" "${listMain}.bck~"
-        # Empty out file
-        echo "" > "${listMain}"
-    fi
 }
 
 for var in "$@"; do
     case "${var}" in
-        "-w" | "whitelist"   ) listMain="${whitelist}"; listAlt="${blacklist}";;
+        "-w" | "whitelist"   ) listType="whitelist"; listAlt="blacklist";;
-        "-b" | "blacklist"   ) listMain="${blacklist}"; listAlt="${whitelist}";;
+        "-b" | "blacklist"   ) listType="blacklist"; listAlt="whitelist";;
-        "--wild" | "wildcard" ) listMain="${regexlist}"; wildcard=true;;
+        "--wild" | "wildcard" ) listType="regex"; wildcard=true;;
-        "--regex" | "regex"   ) listMain="${regexlist}";;
+        "--regex" | "regex"   ) listType="regex";;
         "-nr"| "--noreload"  ) reload=false;;
         "-d" | "--delmode"   ) addmode=false;;
         "-q" | "--quiet"     ) verbose=false;;
@@ -267,9 +229,8 @@ if [[ $# = 0 ]]; then
     helpFunc
 fi
 
-PoplistFile
+ProcessDomainList
 
 if [[ "${reload}" != false ]]; then
-    # Ensure that "restart" is used for Wildcard updates
+    pihole restartdns reload
-    Reload "${reload}"
 fi