From e84134705720eacc57d7b31129b7ea34a79cd2b3 Mon Sep 17 00:00:00 2001 From: Reto Gantenbein Date: Sat, 5 Dec 2020 02:55:25 +0100 Subject: [PATCH 1/8] Fix chronometer output if web interface is not installed Signed-off-by: Reto Gantenbein --- advanced/Scripts/chronometer.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh index 37fd5177..83697c1c 100755 --- a/advanced/Scripts/chronometer.sh +++ b/advanced/Scripts/chronometer.sh @@ -228,8 +228,14 @@ get_sys_stats() { mapfile -t ph_ver_raw < <(pihole -v -c 2> /dev/null | sed -n 's/^.* v/v/p') if [[ -n "${ph_ver_raw[0]}" ]]; then ph_core_ver="${ph_ver_raw[0]}" - ph_lte_ver="${ph_ver_raw[1]}" - ph_ftl_ver="${ph_ver_raw[2]}" + if [[ ${#ph_ver_raw[@]} -eq 2 ]]; then + # AdminLTE not installed + ph_lte_ver="n.a." + ph_ftl_ver="${ph_ver_raw[1]}" + else + ph_lte_ver="${ph_ver_raw[1]}" + ph_ftl_ver="${ph_ver_raw[2]}" + fi else ph_core_ver="-1" fi From 18e4eb460c1cdcf396736b403bbd83ece7e8e637 Mon Sep 17 00:00:00 2001 From: Reto Gantenbein Date: Sat, 5 Dec 2020 13:40:44 +0100 Subject: [PATCH 2/8] piholeDebug: Fix version check if lighttpd/php are not installed Signed-off-by: Reto Gantenbein --- advanced/Scripts/piholeDebug.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index bc83d70a..9b923eb6 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -371,9 +371,9 @@ get_program_version() { echo_current_diagnostic "${program_name} version" # Evalutate the program we are checking, if it is any of the ones below, show the version case "${program_name}" in - "lighttpd") program_version="$(${program_name} -v |& head -n1 | cut -d '/' -f2 | cut -d ' ' -f1)" + "lighttpd") program_version="$(${program_name} -v 2> /dev/null | head -n1 | cut -d '/' -f2 | cut -d ' ' -f1)" ;; - "php") program_version="$(${program_name} -v |& head -n1 | cut -d '-' -f1 | cut -d ' ' -f2)" + "php") program_version="$(${program_name} -v 2> /dev/null | head -n1 | cut -d '-' -f1 | cut -d ' ' -f2)" ;; # If a match is not found, show an error *) echo "Unrecognized program"; From 9dc539c98bdf9ff3b3746f8ddeebb67af6e0af3c Mon Sep 17 00:00:00 2001 From: Reto Gantenbein Date: Sat, 5 Dec 2020 15:16:45 +0100 Subject: [PATCH 3/8] piholeDebug: More detailed diagnostics if Web interface is not found Signed-off-by: Reto Gantenbein --- advanced/Scripts/piholeDebug.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index bc83d70a..16042353 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -334,7 +334,17 @@ compare_local_version_to_git_version() { return 1 fi else - : + # There is no git directory so check if the web interface was disabled + local setup_vars_web_interface + setup_vars_web_interface=$(< ${PIHOLE_SETUP_VARS_FILE} grep ^INSTALL_WEB_INTERFACE | cut -d '=' -f2) + if [[ "${pihole_component}" == "Web" ]] && [[ "${setup_vars_web_interface}" == "false" ]]; then + log_write "${INFO} ${pihole_component}: Disabled in setupVars.conf via INSTALL_WEB_INTERFACE=false" + else + # Return an error message + log_write "${COL_RED}Directory ${git_dir} doesn't exist${COL_NC}" + # and exit with a non zero code + return 1 + fi fi } From 922f795fe6f85b026b0b3c6af52571aad64c2c3e Mon Sep 17 00:00:00 2001 From: user Date: Sun, 6 Dec 2020 06:03:30 -0500 Subject: [PATCH 4/8] Adding comma to the adlist table format To account for the Date Updated column in the adlist table adding a comma to the adlist to be imported into the database. This was causing intermittent failures on updates and therefore no domains would be blocked. Signed-off-by: user Date: Sun Dec 6 06:03:30 2020 -0500 Committer: user On branch fix/gravity_date_updated Your branch and 'origin/fix/gravity_date_updated' have diverged, and have 1 and 1 different commits each, respectively. (use "git pull" to merge the remote branch into yours) Changes to be committed: modified: gravity.sh Signed-off-by: user --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 9ac38512..5a831dae 100755 --- a/gravity.sh +++ b/gravity.sh @@ -176,7 +176,7 @@ database_table_from_file() { echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" elif [[ "${table}" == "adlist" ]]; then # Adlist table format - echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"," >> "${tmpFile}" else # White-, black-, and regexlist table format echo "${rowid},${type},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" From 20faa1bd1c0ccb237c541548265bb8bcd2bf2a23 Mon Sep 17 00:00:00 2001 From: Reto Gantenbein <1031031+ganto@users.noreply.github.com> Date: Thu, 10 Dec 2020 22:58:07 +0100 Subject: [PATCH 5/8] Update pihole-FTL man-pages (#3883) Extend latest options and adjust ordering to Web documentation Ensure that all configuration options currently recognized by FTL 5.3.1 are properly documented in the man page of the configuration file. Signed-off-by: Reto Gantenbein --- manpages/pihole-FTL.8 | 44 +++++- manpages/pihole-FTL.conf.5 | 265 +++++++++++++++++++++++++++++++++---- 2 files changed, 280 insertions(+), 29 deletions(-) diff --git a/manpages/pihole-FTL.8 b/manpages/pihole-FTL.8 index 2928f2d8..c7b69d83 100644 --- a/manpages/pihole-FTL.8 +++ b/manpages/pihole-FTL.8 @@ -1,4 +1,4 @@ -.TH "Pihole-FTL" "8" "pihole-FTL" "Pi-hole" "June 2018" +.TH "Pihole-FTL" "8" "pihole-FTL" "Pi-hole" "November 2020" .SH "NAME" pihole-FTL - Pi-hole : The Faster-Than-Light (FTL) Engine .br @@ -10,7 +10,7 @@ pihole-FTL - Pi-hole : The Faster-Than-Light (FTL) Engine .br \fBpihole-FTL test\fR .br -\fBpihole-FTL -v\fR +\fBpihole-FTL -v|-vv\fR .br \fBpihole-FTL -t\fR .br @@ -22,6 +22,16 @@ pihole-FTL - Pi-hole : The Faster-Than-Light (FTL) Engine .br \fBpihole-FTL dnsmasq-test\fR .br +\fBpihole-FTL regex-test str\fR +.br +\fBpihole-FTL regex-test str rgx\fR +.br +\fBpihole-FTL lua\fR +.br +\fBpihole-FTL luac\fR +.br +\fBpihole-FTL dhcp-discover\fR +.br \fBpihole-FTL --\fR (\fBoptions\fR) .br @@ -65,6 +75,11 @@ Command line arguments Don't start FTL, show only version .br +\fB-vv\fR +.br + Don't start FTL, show verbose version information of embedded applications +.br + \fB-t, tag\fR .br Don't start FTL, show only git tag @@ -90,6 +105,31 @@ Command line arguments Test resolver config file syntax .br +\fBregex-test str\fR +.br + Test str against all regular expressions in the database +.br + +\fBregex-test str rgx\fR +.br + Test str against regular expression given by rgx +.br + +\fBlua\fR +.br + Start the embedded Lua interpreter +.br + +\fBluac\fR +.br + Execute the embedded Lua compiler +.br + +\fBdhcp-discover\fR +.br + Discover DHCP servers in the local network +.br + \fB--\fR (options) .br Pass options to internal dnsmasq resolver diff --git a/manpages/pihole-FTL.conf.5 b/manpages/pihole-FTL.conf.5 index 5cb0481e..42405436 100644 --- a/manpages/pihole-FTL.conf.5 +++ b/manpages/pihole-FTL.conf.5 @@ -1,4 +1,4 @@ -.TH "pihole-FTL.conf" "5" "pihole-FTL.conf" "pihole-FTL.conf" "June 2018" +.TH "pihole-FTL.conf" "5" "pihole-FTL.conf" "pihole-FTL.conf" "November 2020" .SH "NAME" pihole-FTL.conf - FTL's config file @@ -7,43 +7,154 @@ pihole-FTL.conf - FTL's config file /etc/pihole/pihole-FTL.conf will be read by \fBpihole-FTL(8)\fR on startup. .br +For each setting the option shown first is the default. +.br -\fBSOCKET_LISTENING=localonly|all\fR +\fBBLOCKINGMODE=IP|IP-AAAA-NODATA|NODATA|NXDOMAIN|NULL\fR +.br + How should FTL reply to blocked queries? + + IP - Pi-hole's IPs for blocked domains + + IP-AAAA-NODATA - Pi-hole's IP + NODATA-IPv6 for blocked domains + + NODATA - Using NODATA for blocked domains + + NXDOMAIN - NXDOMAIN for blocked domains + + NULL - Null IPs for blocked domains +.br + +\fBCNAME_DEEP_INSPECT=true|false\fR +.br + Use this option to disable deep CNAME inspection. This might be beneficial for very low-end devices. +.br + +\fBBLOCK_ESNI=true|false\fR +.br + Block requests to _esni.* sub-domains. +.br + +\fBMAXLOGAGE=24.0\fR +.br + Up to how many hours of queries should be imported from the database and logs? .br - Listen only for local socket connections or permit all connections + Maximum is 744 (31 days) .br -\fBQUERY_DISPLAY=yes|no\fR +\fBPRIVACYLEVEL=0|1|2|3|4\fR +.br + Privacy level used to collect Pi-hole statistics. +.br + 0 - show everything .br - Display all queries? Set to no to hide query display + 1 - hide domains +.br + 2 - hide domains and clients +.br + 3 - anonymous mode (hide everything) +.br + 4 - disable all statistics +.br + +\fBIGNORE_LOCALHOST=no|yes\fR +.br + Should FTL ignore queries coming from the local machine? .br \fBAAAA_QUERY_ANALYSIS=yes|no\fR .br - Allow FTL to analyze AAAA queries from pihole.log? + Should FTL analyze AAAA queries? +.br + +\fBANALYZE_ONLY_A_AND_AAAA=false|true\fR +.br + Should FTL only analyze A and AAAA queries? +.br + +\fBSOCKET_LISTENING=localonly|all\fR +.br + Listen only for local socket connections on the API port or permit all connections. +.br + +\fBFTLPORT=4711\fR +.br + On which port should FTL be listening? .br \fBRESOLVE_IPV6=yes|no\fR .br - Should FTL try to resolve IPv6 addresses to host names? + Should FTL try to resolve IPv6 addresses to hostnames? .br \fBRESOLVE_IPV4=yes|no\fR .br - Should FTL try to resolve IPv4 addresses to host names? + Should FTL try to resolve IPv4 addresses to hostnames? .br -\fBMAXDBDAYS=365\fR +\fBDELAY_STARTUP=0\fR +.br + Time in seconds (between 0 and 300) to delay FTL startup. +.br + +\fBNICE=-10\fR +.br + Set the niceness of the Pi-hole FTL process. +.br + Can be disabled altogether by setting a value of -999. +.br + +\fBNAMES_FROM_NETDB=true|false\fR +.br + Control whether FTL should use a fallback option and try to obtain client names from checking the network table. +.br + E.g. IPv6 clients without a hostname will be compared via MAC address to known clients. +.br + +\fB\fBREFRESH_HOSTNAMES=IPV4|ALL|NONE\fR +.br + Change how (and if) hourly PTR requests are made to check for changes in client and upstream server hostnames: +.br + IPV4 - Do the hourly PTR lookups only for IPv4 addresses resolving issues in networks with many short-lived PE IPv6 addresses. +.br + ALL - Do the hourly PTR lookups for all addresses. This can create a lot of PTR queries in networks with many IPv6 addresses. +.br + NONE - Don't do hourly PTR lookups. Look up hostnames once (when first seeing a client) and never again. Future hostname changes may be missed. +.br + +\fBMAXNETAGE=365\fR +.br + IP addresses (and associated host names) older than the specified number of days are removed. +.br + This avoids dead entries in the network overview table. +.br + +\fBEDNS0_ECS=true|false\fR +.br + Should we overwrite the query source when client information is provided through EDNS0 client subnet (ECS) information? +.br + +\fBPARSE_ARP_CACHE=true|false\fR +.br + Parse ARP cache to fill network overview table. +.br + +\fBDBIMPORT=yes|no\fR .br - How long should queries be stored in the database? + Should FTL load information from the database on startup to be aware of the most recent history? .br - Setting this to 0 disables the database + +\fBMAXDBDAYS=365\fR +.br + How long should queries be stored in the database? Setting this to 0 disables the database .br \fBDBINTERVAL=1.0\fR .br How often do we store queries in FTL's database [minutes]? .br + Accepts value between 0.1 (6 sec) and 1440 (1 day) +.br \fBDBFILE=/etc/pihole/pihole-FTL.db\fR .br @@ -52,43 +163,143 @@ pihole-FTL.conf - FTL's config file Setting this to DBFILE= disables the database altogether .br -\fBMAXLOGAGE=24.0\fR +\fBLOGFILE=/var/log/pihole-FTL.log\fR .br - Up to how many hours of queries should be imported from the database and logs? + The location of FTL's log file. .br - Maximum is 744 (31 days) + +\fBPIDFILE=/run/pihole-FTL.pid\fR +.br + The file which contains the PID of FTL's main process. .br -\fBFTLPORT=4711\fR +\fBPORTFILE=/run/pihole-FTL.port\fR .br - On which port should FTL be listening? + Specify path and filename where the FTL process will write its API port number. .br -\fBPRIVACYLEVEL=0|1|2|3\fR +\fBSOCKETFILE=/run/pihole/FTL.sock\fR .br - Which privacy level is used? + The file containing the socket FTL's API is listening on. .br - 0 - show everything + +\fBSETUPVARSFILE=/etc/pihole/setupVars.conf\fR .br - 1 - hide domains + The config file of Pi-hole containing, e.g., the current blocking status (do not change). .br - 2 - hide domains and clients + +\fBMACVENDORDB=/etc/pihole/macvendor.db\fR .br - 3 - anonymous mode (hide everything) + The database containing MAC -> Vendor information for the network table. .br -\fBIGNORE_LOCALHOST=no|yes\fR +\fBGRAVITYDB=/etc/pihole/gravity.db\fR .br - Should FTL ignore queries coming from the local machine? + Specify path and filename of FTL's SQLite3 gravity database. This database contains all domains relevant for Pi-hole's DNS blocking. .br -\fBBLOCKINGMODE=IP|IP-AAAA-NODATA|NXDOMAIN|NULL\fR +\fBDEBUG_ALL=false|true\fR .br - How should FTL reply to blocked queries? + Enable all debug flags. If this is set to true, all other debug config options are ignored. +.br + +\fBDEBUG_DATABASE=false|true\fR +.br + Print debugging information about database actions such as SQL statements and performance. +.br + +\fBDEBUG_NETWORKING=false|true\fR +.br + Prints a list of the detected network interfaces on the startup of FTL. .br -For each setting, the option shown first is the default. +\fBDEBUG_LOCKS=false|true\fR +.br + Print information about shared memory locks. +.br + Messages will be generated when waiting, obtaining, and releasing a lock. +.br + +\fBDEBUG_QUERIES=false|true\fR +.br + Print extensive DNS query information (domains, types, replies, etc.). +.br + +\fBDEBUG_FLAGS=false|true\fR +.br + Print flags of queries received by the DNS hooks. +.br + Only effective when \fBDEBUG_QUERIES\fR is enabled as well. + +\fBDEBUG_SHMEM=false|true\fR +.br + Print information about shared memory buffers. +.br + Messages are either about creating or enlarging shmem objects or string injections. .br + +\fBDEBUG_GC=false|true\fR +.br + Print information about garbage collection (GC): +.br + What is to be removed, how many have been removed and how long did GC take. +.br + +\fBDEBUG_ARP=false|true\fR +.br + Print information about ARP table processing: +.br + How long did parsing take, whether read MAC addresses are valid, and if the macvendor.db file exists. +.br + +\fBDEBUG_REGEX=false|true\fR +.br + Controls if FTL should print extended details about regex matching. +.br + +\fBDEBUG_API=false|true\fR +.br + Print extra debugging information during telnet API calls. +.br + Currently only used to send extra information when getting all queries. +.br + +\fBDEBUG_OVERTIME=false|true\fR +.br + Print information about overTime memory operations, such as initializing or moving overTime slots. +.br + +\fBDEBUG_EXTBLOCKED=false|true\fR +.br + Print information about why FTL decided that certain queries were recognized as being externally blocked. +.br + +\fBDEBUG_CAPS=false|true\fR +.br + Print information about POSIX capabilities granted to the FTL process. +.br + The current capabilities are printed on receipt of SIGHUP i.e. after executing `killall -HUP pihole-FTL`. +.br + +\fBDEBUG_DNSMASQ_LINES=false|true\fR +.br + Print file and line causing a dnsmasq event into FTL's log files. +.br + This is handy to implement additional hooks missing from FTL. +.br + +\fBDEBUG_VECTORS=false|true\fR +.br + FTL uses dynamically allocated vectors for various tasks. +.br + This config option enables extensive debugging information such as information about allocation, referencing, deletion, and appending. +.br + +\fBDEBUG_RESOLVER=false|true\fR +.br + Extensive information about hostname resolution like which DNS servers are used in the first and second hostname resolving tries. +.br + .SH "SEE ALSO" \fBpihole\fR(8), \fBpihole-FTL\fR(8) From 357b6702ec5bfdfa0a276c4c0a30b15be7386e1b Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 11 Dec 2020 12:30:52 +0000 Subject: [PATCH 6/8] Update advanced/Scripts/chronometer.sh Co-authored-by: DL6ER --- advanced/Scripts/chronometer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh index 83697c1c..3e77f578 100755 --- a/advanced/Scripts/chronometer.sh +++ b/advanced/Scripts/chronometer.sh @@ -230,7 +230,7 @@ get_sys_stats() { ph_core_ver="${ph_ver_raw[0]}" if [[ ${#ph_ver_raw[@]} -eq 2 ]]; then # AdminLTE not installed - ph_lte_ver="n.a." + ph_lte_ver="(not installed)" ph_ftl_ver="${ph_ver_raw[1]}" else ph_lte_ver="${ph_ver_raw[1]}" From 3a05ac27a29e18ec41b8884355d5751815b3eeaa Mon Sep 17 00:00:00 2001 From: Michael Woolweaver Date: Sat, 19 Dec 2020 17:37:17 -0600 Subject: [PATCH 7/8] Fix typo found by #3933 Signed-off-by: Michael Woolweaver --- advanced/Templates/gravity.db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 2aa7e8f8..26b030c8 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -73,7 +73,7 @@ CREATE TABLE domainlist_by_group CREATE TABLE client ( id INTEGER PRIMARY KEY AUTOINCREMENT, - ip TEXT NOL NULL UNIQUE, + ip TEXT NOT NULL UNIQUE, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT From 60fa93ab47e96fb99a864b3805572f209003383b Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Wed, 23 Dec 2020 14:59:28 -0800 Subject: [PATCH 8/8] Add fallback for arch detection on i386/x86_64 Signed-off-by: Dan Schaper --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 45e96354..353899c8 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2451,7 +2451,7 @@ get_binary_name() { elif [[ "${machine}" == "x86_64" ]]; then # This gives the processor of packages dpkg installs (for example, "i386") local dpkgarch - dpkgarch=$(dpkg --print-processor 2> /dev/null || true) + dpkgarch=$(dpkg --print-processor 2> /dev/null || dpkg --print-architecture 2> /dev/null) # Special case: This is a 32 bit OS, installed on a 64 bit machine # -> change machine processor to download the 32 bit executable