Merge pull request pi-hole#82 from pi-hole/development

FTL v2.9
GarrettSocling · Jun 16, 2017 · 0d6134d · 0d6134d
2 parents 7ddef5e + 63dd395
commit 0d6134d
Show file tree

Hide file tree

Showing 23 changed files with 214,388 additions and 114 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,13 @@
-# Generated binary file
+# Generated binary files
+/obj/
 pihole-FTL
 
 # Versioning files
 version*
+
+# CMake files
+/CMakeLists.txt
+/cmake-build-debug/
+
+# IDE files
+.idea/
diff --git a/.travis.yml b/.travis.yml
@@ -11,6 +11,10 @@ matrix:
   - env:
     - BUILD_CC="cc"
     - BIN_NAME="pihole-FTL-linux-x86_64"
+    addons:
+      apt:
+        packages:
+        - sqlite3
   - env:
     - BUILD_CC="arm-linux-gnueabihf-gcc"
     - BIN_NAME="pihole-FTL-arm-linux-gnueabihf"
@@ -59,6 +63,7 @@ before_script:
 script:
 - make pihole-FTL CC="${BUILD_CC}" CFLAGS="${CFLAGS}"
 - file pihole-FTL
+- if [[ "${BIN_NAME}" == "pihole-FTL-linux-x86_64" ]]; then test/run.sh; fi
 - mv pihole-FTL "${DEST_DIR}/${BIN_NAME}"
 - cd "${DEST_DIR}"
 - sha1sum pihole-FTL-* > ${BIN_NAME}.sha1

diff --git a/FTL.h b/FTL.h
@@ -8,6 +8,8 @@
 *  This file is copyright under the latest version of the EUPL.
 *  Please see LICENSE file for your rights under this license. */
 
+#define __USE_XOPEN
+#define _GNU_SOURCE
 #include <stdio.h>
 // variable argument lists
 #include <stdarg.h>
@@ -18,8 +20,6 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <string.h>
-#define __USE_XOPEN
-#define _GNU_SOURCE
 #include <time.h>
 #include <sys/time.h>
 #include <sys/socket.h>
@@ -36,6 +36,8 @@
 #include <pwd.h>
 // syslog
 #include <syslog.h>
+// SQLite
+#include "sqlite3.h"
 
 #include "routines.h"
 
@@ -62,12 +64,17 @@
 // Default -60 (one minute before a full hour)
 #define GCdelay (-60)
 
+// How often do we dump into FTL's database?
+// Default: 60 (once per minute)
+#define DBinterval 60
+
 // Static structs
 typedef struct {
 	const char* conf;
 	const char* log;
 	const char* pid;
 	const char* port;
+	const char* db;
 } FTLFileNamesStruct;
 
 typedef struct {
@@ -112,6 +119,7 @@ typedef struct {
 	bool rolling_24h;
 	bool query_display;
 	bool analyze_AAAA;
+	int maxDBfilesize;
 } ConfigStruct;
 
 // Dynamic structs
@@ -126,6 +134,7 @@ typedef struct {
 	int clientID;
 	int forwardID;
 	bool valid;
+	bool db;
 } queriesDataStruct;
 
 typedef struct {
@@ -173,6 +182,7 @@ typedef struct {
 } memoryStruct;
 
 enum { QUERIES, FORWARDED, CLIENTS, DOMAINS, OVERTIME, WILDCARD };
+enum { SOCKET };
 
 logFileNamesStruct files;
 FTLFileNamesStruct FTLfiles;
@@ -196,6 +206,7 @@ bool debug;
 bool debugthreads;
 bool debugclients;
 bool debugGC;
+bool debugDB;
 bool threadwritelock;
 bool threadreadlock;
 
@@ -209,3 +220,7 @@ char timestamp[16];
 bool flush;
 bool needGC;
 bool daemonmode;
+bool database;
+long int lastdbindex;
+bool travis;
+bool needDBGC;
diff --git a/Makefile b/Makefile
@@ -9,7 +9,7 @@
 # Please see LICENSE file for your rights under this license.
 
 DEPS = FTL.h routines.h version.h
-OBJ = main.o structs.o log.o daemon.o parser.o signals.o socket.o request.o grep.o setupVars.o args.o flush.o threads.o gc.o config.o
+OBJ = main.o structs.o log.o daemon.o parser.o signals.o socket.o request.o grep.o setupVars.o args.o flush.o threads.o gc.o config.o database.o
 
 # Get git commit version and date
 GIT_BRANCH := $(shell git branch | sed -n 's/^\* //p')
@@ -18,7 +18,7 @@ GIT_DATE := $(shell git --no-pager show --date=short --format="%ai" --name-only
 GIT_TAG := $(shell git describe --tags --abbrev=0)
 
 # -fstack-protector: The program will be resistant to having its stack overflowed
-# -D_FORTIFY_SOURCE=2 and -O1 or higher: This causes certain unsafe glibc functions zo be replaced with their safer counterparts
+# -D_FORTIFY_SOURCE=2 and -O1 or higher: This causes certain unsafe glibc functions to be replaced with their safer counterparts
 # -Wl,-z,relro: reduces the possible areas of memory in a program that can be used by an attacker that performs a successful memory corruption exploit
 # -Wl,-z,now: When combined with RELRO above, this further reduces the regions of memory available to memory corruption attacks
 # -pie -fPIE: For ASLR
@@ -28,8 +28,14 @@ GIT_TAG := $(shell git describe --tags --abbrev=0)
 # -fno-omit-frame-pointer: get nicer stacktraces
 CC=gcc
 HARDENING_FLAGS=-fstack-protector -D_FORTIFY_SOURCE=2 -O3 -Wl,-z,relro,-z,now -pie -fPIE
-DEBUG_FLAGS=-g3 -rdynamic -fno-omit-frame-pointer #-fsanitize=address
-CCFLAGS=-I$(IDIR) -Wall -Wextra -Wno-unused-parameter -D_FILE_OFFSET_BITS=64 $(HARDENING_FLAGS) $(DEBUG_FLAGS) $(CFLAGS)
+DEBUG_FLAGS=-rdynamic -fno-omit-frame-pointer #-fsanitize=address
+# -DSQLITE_OMIT_LOAD_EXTENSION: This option omits the entire extension loading mechanism from SQLite, including sqlite3_enable_load_extension() and sqlite3_load_extension() interfaces. (needs -ldl linking option, otherwise)
+# -DSQLITE_THREADSAFE=0: causes all of the mutex and thread-safety logic in SQLite to be omitted. This is the single compile-time option that makes the most difference in optimizing the performance of SQLite.
+# -DSQLITE_DEFAULT_MEMSTATUS=0: This setting causes the sqlite3_status() interfaces that track memory usage to be disabled. This helps the sqlite3_malloc() routines run much faster, and since SQLite uses sqlite3_malloc() internally, this helps to make the entire library faster.
+# -DSQLITE_OMIT_DEPRECATED: Omitting deprecated interfaces and features will not help SQLite to run any faster. It will reduce the library footprint, however. And it is the right thing to do.
+# -DSQLITE_OMIT_PROGRESS_CALLBACK: The progress handler callback counter must be checked in the inner loop of the bytecode engine. By omitting this interface, a single conditional is removed from the inner loop of the bytecode engine, helping SQL statements to run slightly faster.
+SQLITEFLAGS=-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_THREADSAFE=0 -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_MEMORYDB
+CCFLAGS=-I$(IDIR) -Wall -Wextra -Wno-unused-parameter -D_FILE_OFFSET_BITS=64 $(HARDENING_FLAGS) $(DEBUG_FLAGS) $(CFLAGS) $(SQLITEFLAGS)
 LIBS=-pthread
 
 ODIR =obj
@@ -40,10 +46,18 @@ _DEPS = $(patsubst %,$(IDIR)/%,$(DEPS))
 
 _OBJ = $(patsubst %,$(ODIR)/%,$(OBJ))
 
-$(ODIR)/%.o: %.c $(_DEPS)
+all: pihole-FTL
+
+$(ODIR)/%.o: %.c $(_DEPS) | $(ODIR)
+	$(CC) -c -o $@ $< -g3 $(CCFLAGS)
+
+$(ODIR):
+	mkdir -p $(ODIR)
+
+$(ODIR)/sqlite3.o: sqlite3.c
 	$(CC) -c -o $@ $< $(CCFLAGS)
 
-pihole-FTL: $(_OBJ)
+pihole-FTL: $(_OBJ) $(ODIR)/sqlite3.o
 	$(CC) -v $(CCFLAGS) -o $@ $^ $(LIBS)
 
 .PHONY: clean force install

diff --git a/README.md b/README.md
@@ -97,12 +97,13 @@ port may be automatically incremented if `4711` isn't available
 
 You can create a file `/etc/pihole/pihole-FTL.conf` that will be read by `FTL` on startup.
 
-Possible settings (the first one is the default setting):
+Possible settings (**the option shown first is the default**):
 
-- `SOCKET_LISTENING=localonly|all` (listen only for local connections or permit all connections)
-- `TIMEFRAME=rolling24h|yesterday|today` (rolling data window, up to 48h (today + yesterday), or up to 24h (only today, as in Pi-hole `v2.x` ))
-- `QUERY_DISPLAY=yes|no` (hide queries altogether)
-- `analyze_AAAA=yes|no` (do we want `FTL` to analyze AAAA queries from pihole.log?)
+- `SOCKET_LISTENING=localonly|all` (Listen only for local socket connections or permit all connections)
+- `TIMEFRAME=rolling24h|yesterday|today` (Rolling data window, up to 48h (today + yesterday), or up to 24h (only today, as in Pi-hole `v2.x` ))
+- `QUERY_DISPLAY=yes|no` (Display all queries? Set to `no` to hide query display)
+- `AAAA_QUERY_ANALYSIS=yes|no` (Allow `FTL` to analyze AAAA queries from pihole.log?)
+- `MAXDBFILESIZE=100` (How large do we want the FTL database to grow at max (given in MB), setting this to `0` disables the database altogether)
 
 ### Implemented keywords (starting with `>`, subject to change):
 
@@ -242,3 +243,9 @@ tag v1.6
 branch master
 date 2017-03-26 13:10:43 +0200
 ```
+
+- `>dbstats` : Get some statistics about the FTL long-term storage database
+ ```
+ queries in database: 88387
+SQLite version: 3.19.3
+```
diff --git a/args.c b/args.c
@@ -17,6 +17,8 @@ bool debugthreads = false;
 bool debugclients = false;
 bool debugGC = false;
 bool runtest = false;
+bool debugDB = false;
+bool travis = false;
 void parse_args(int argc, char* argv[])
 {
 	int i;
@@ -42,6 +44,12 @@ void parse_args(int argc, char* argv[])
 			debugGC = true;
 		}
 
+		if(strcmp(argv[i], "debugDB") == 0)
+		{
+			debug = true;
+			debugDB = true;
+		}
+
 		if(strcmp(argv[i], "test") == 0)
 			killed = 1;
 
@@ -70,6 +78,15 @@ void parse_args(int argc, char* argv[])
 			daemonmode = false;
 		}
 
+		// Use files in local places for Travis-CI tests
+		if(strcmp(argv[i], "travis-ci") == 0)
+		{
+			travis = true;
+			FTLfiles.log = "pihole-FTL.log";
+			FTLfiles.db = "pihole-FTL.db";
+			files.log = "pihole.log";
+		}
+
 		// Other arguments are ignored
 	}
 }
diff --git a/config.c b/config.c
@@ -3,7 +3,7 @@
 *  Network-wide ad blocking via your own hardware.
 *
 *  FTL Engine
-*  Log parsing routine
+*  Config routines
 *
 *  This file is copyright under the latest version of the EUPL.
 *  Please see LICENSE file for your rights under this license. */
@@ -95,6 +95,22 @@ void read_FTLconf(void)
 	else
 		logg("   AAAA_QUERY_ANALYSIS: Hide AAAA queries");
 
+	// MAXDBFILESIZE
+	// defaults to: 100 MB
+	config.maxDBfilesize = 100;
+	buffer = parse_FTLconf(fp, "MAXDBFILESIZE");
+	if(buffer != NULL)
+	{
+		int value = 0;
+		if(sscanf(buffer, "%i", &value))
+			if(value >= 0)
+				config.maxDBfilesize = value;
+	}
+	if(config.maxDBfilesize == 0)
+		logg("   MAXDBFILESIZE: --- (DB disabled)", config.maxDBfilesize);
+	else
+		logg("   MAXDBFILESIZE: %i MB", config.maxDBfilesize);
+
 	logg("Finished config file parsing");
 
 	if(conflinebuffer != NULL)

diff --git a/daemon.c b/daemon.c
@@ -167,11 +167,14 @@ void go_daemon(void)
 	savepid();
 
 	// Change the current working directory
-	if(chdir("/etc/pihole") != 0)
+	if(!travis)
 	{
-		logg("FATAL: Cannot change directory to /etc/pihole. Error code: %i",errno);
-		// Return failure
-		exit(EXIT_FAILURE);
+		if(chdir("/etc/pihole") != 0)
+		{
+			logg("FATAL: Cannot change directory to /etc/pihole. Error code: %i",errno);
+			// Return failure
+			exit(EXIT_FAILURE);
+		}
 	}
 
 	// Close stdin, stdout and stderr