Performance optimizations and bug fixes

build.sh

@@ -41,6 +41,7 @@ do
         "debug"          ) debug=1;;
         "dev"            ) dev=1;;
         "test"           ) test=1;;
+        "test-fast"      ) test=1; export SKIP_PERF_TEST=1;;
         "clean-logs"     ) clean_logs=1;;
         "clang"          ) clang=1;;
         "ci"             ) builddir="cmake_ci/";;

src/CMakeLists.txt

@@ -264,8 +264,6 @@ set(sources
         signals.h
         timers.c
         timers.h
-        vector.c
-        vector.h
         version.c
         version.h
         )

src/api/auth.c

@@ -378,10 +378,9 @@ static int send_api_auth_status(struct ftl_conn *api, const int user_id, const t
 	{
 		log_debug(DEBUG_API, "API Auth status: OK");
 
-		// Ten minutes validity
 		if(snprintf(pi_hole_extra_headers, sizeof(pi_hole_extra_headers),
 		            FTL_SET_COOKIE,
-		            auth_data[user_id].sid, config.webserver.session.timeout.d.ui) < 0)
+		            auth_data[user_id].sid, config.webserver.session.timeout.v.ui) < 0)
 		{
 			return send_json_error(api, 500, "internal_error", "Internal server error", NULL);
 		}

src/api/docs/content/specs/config.yaml

@@ -637,6 +637,8 @@ components:
                   type: boolean
                 timing:
                   type: boolean
+                performance:
+                   type: boolean
                 all:
                   type: boolean
     topics:
@@ -892,6 +894,7 @@ components:
             ntp: false
             netlink: false
             timing: false
+            performance: false
             all: false
           took: 0.003
     config_one:

src/api/docs/content/specs/info.yaml

@@ -708,8 +708,12 @@ components:
               properties:
                 gravity:
                   type: integer
-                  description: Number of collected exact domains on lists
+                  description: Number of collected exact domains on blocking lists
                   example: 67906
+                antigravity:
+                  type: integer
+                  description: Number of collected exact domains on allowing lists
+                  example: 0
                 groups:
                   type: integer
                   description: Number of groups

src/api/info.c

@@ -132,7 +132,7 @@ int api_info_database(struct ftl_conn *api)
 
 	cJSON *group = JSON_NEW_OBJECT();
 	JSON_ADD_NUMBER_TO_OBJECT(group, "gid", st.st_gid); // GID
-	const struct group *gr = getgrgid(st.st_uid);
+	const struct group *gr = getgrgid(st.st_gid);
 	if(gr != NULL)
 	{
 		JSON_COPY_STR_TO_OBJECT(group, "name", gr->gr_name); // Group name
@@ -560,6 +560,7 @@ static int get_ftl_obj(struct ftl_conn *api, cJSON *ftl)
 	// Source from shared objects within lock
 	lock_shm();
 	const int db_gravity = counters->database.gravity;
+	const int db_antigravity = counters->database.antigravity;
 	const int db_groups = counters->database.groups;
 	const int db_lists = counters->database.lists;
 	const int db_clients = counters->database.clients;
@@ -590,6 +591,7 @@ static int get_ftl_obj(struct ftl_conn *api, cJSON *ftl)
 	unlock_shm();
 
 	JSON_ADD_NUMBER_TO_OBJECT(database, "gravity", db_gravity);
+	JSON_ADD_NUMBER_TO_OBJECT(database, "antigravity", db_antigravity);
 	JSON_ADD_NUMBER_TO_OBJECT(database, "groups", db_groups);
 	JSON_ADD_NUMBER_TO_OBJECT(database, "lists", db_lists);
 	JSON_ADD_NUMBER_TO_OBJECT(database, "clients", db_clients);

src/api/list.c

@@ -431,6 +431,7 @@ static int api_list_write(struct ftl_conn *api,
 				// which convert to xn--{{{-pla4gpb.com
 				if(!valid_domain(punycode, strlen(punycode), false))
 				{
+					free(punycode);
 					if(allocated_json)
 						cJSON_Delete(row.items);
 					return send_json_error(api, 400, // 400 Bad Request

src/api/network.c

@@ -302,6 +302,7 @@ static int api_network_devices_DELETE(struct ftl_conn *api)
 	int deleted = 0;
 	if(!networkTable_deleteDevice(db, device_id, &deleted, &sql_msg))
 	{
+		dbclose(&db);
 		// Add SQL message (may be NULL = not available)
 		return send_json_error(api, 500,
 		                       "database_error",
@@ -412,7 +413,7 @@ int api_client_suggestions(struct ftl_conn *api)
 		cJSON *client = JSON_NEW_OBJECT();
 		JSON_COPY_STR_TO_OBJECT(client, "hwaddr", sqlite3_column_text(stmt, 0));
 		JSON_COPY_STR_TO_OBJECT(client, "macVendor", sqlite3_column_text(stmt, 1));
-		JSON_ADD_NUMBER_TO_OBJECT(client, "lastQuery", sqlite3_column_int(stmt, 2));
+		JSON_ADD_NUMBER_TO_OBJECT(client, "lastQuery", sqlite3_column_int64(stmt, 2));
 		JSON_COPY_STR_TO_OBJECT(client, "addresses", sqlite3_column_text(stmt, 3));
 		JSON_COPY_STR_TO_OBJECT(client, "names", sqlite3_column_text(stmt, 4));
 		JSON_ADD_ITEM_TO_ARRAY(clients, client);

src/api/padd.c

@@ -48,7 +48,7 @@ int api_padd(struct ftl_conn *api)
 	if(config.misc.privacylevel.v.privacy_level < PRIVACY_HIDE_DOMAINS)
 	{
 		// Find most recently blocked query
-		for(int queryID = counters->queries - 1; queryID > 0 ; queryID--)
+		for(int queryID = counters->queries - 1; queryID >= 0 ; queryID--)
 		{
 			const queriesData *query = getQuery(queryID, true);
 			if(query == NULL)
@@ -98,6 +98,7 @@ int api_padd(struct ftl_conn *api)
 		const char *domain = cJSON_GetStringValue(top_block);
 		JSON_COPY_STR_TO_OBJECT(json, "top_blocked", domain);
 	}
+	cJSON_Delete(top_blocked);
 	cJSON *top_clients = get_top_clients(api, 1, false, true, false, true);
 	if(cJSON_GetArraySize(top_clients) == 0)
 	{
@@ -109,6 +110,7 @@ int api_padd(struct ftl_conn *api)
 		const char *client = cJSON_GetStringValue(top_client);
 		JSON_COPY_STR_TO_OBJECT(json, "top_client", client);
 	}
+	cJSON_Delete(top_clients);
 
 	// Add a null entry if the domain is hidden or there is no recent
 	// blocked domain (e.g. when blocking is disabled)

src/api/stats.c

@@ -78,6 +78,27 @@ static int __attribute__((pure)) cmpdesc_te(const void *a, const void *b)
 		return 0;
 }
 
+// Min-heap sift-down for top-K selection: maintain the smallest element at the
+// root so it can be replaced when a larger one is found
+static void heap_sift_down(struct top_entries *heap, const unsigned int size, unsigned int i)
+{
+	for(;;)
+	{
+		unsigned int min = i;
+		const unsigned int l = 2*i + 1, r = 2*i + 2;
+		if(l < size && heap[l].count < heap[min].count)
+			min = l;
+		if(r < size && heap[r].count < heap[min].count)
+			min = r;
+		if(min == i)
+			break;
+		const struct top_entries tmp = heap[i];
+		heap[i] = heap[min];
+		heap[min] = tmp;
+		i = min;
+	}
+}
+
 static int get_query_types_obj(struct ftl_conn *api, cJSON *types)
 {
 	for(unsigned int i = TYPE_A; i < TYPE_MAX; i++)
@@ -210,14 +231,21 @@ cJSON *get_top_domains(struct ftl_conn *api, const int count,
 	const unsigned int domains = counters->domains;
 	const unsigned int total_queries = counters->queries;
 	const unsigned int blocked_count = get_blocked_count();
-	struct top_entries *top_domains = calloc(domains, sizeof(struct top_entries));
-	if(top_domains == NULL)
+
+	// Heap-based top-K selection: allocate only for the top entries
+	// rather than for all domains, reducing memory from O(N) to O(K).
+	const unsigned int k = count > 0 ? (unsigned int)count : 0u;
+	const unsigned int heap_cap = (k <= domains / 4) ? k * 4 : domains;
+	struct top_entries *top_domains = heap_cap > 0 ? calloc(heap_cap, sizeof(struct top_entries)) : NULL;
+	if(heap_cap > 0 && top_domains == NULL)
 	{
 		log_err("Memory allocation failed in %s()", __FUNCTION__);
+		unlock_shm();
 		return NULL;
 	}
 
-	unsigned int added_domains = 0u;
+	unsigned int heap_size = 0;
+	bool heap_ready = false;
 	for(unsigned int domainID = 0; domainID < domains; domainID++)
 	{
 		// Get domain pointer
@@ -232,28 +260,52 @@ cJSON *get_top_domains(struct ftl_conn *api, const int count,
 			continue;
 
 		// Use either blocked or total count based on request string
-		top_domains[added_domains].count = blocked ? domain->blockedcount : domain->count - domain->blockedcount;
+		const int entry_count = blocked ? domain->blockedcount : domain->count - domain->blockedcount;
 
-		// Get domain name
-		top_domains[added_domains].namepos = domain->domainpos;
+		// Skip zero-count entries early
+		if(entry_count < 1)
+			continue;
 
-		// Increment counter
-		added_domains++;
+		if(heap_size < heap_cap)
+		{
+			// Heap not full yet, append directly
+			top_domains[heap_size].count = entry_count;
+			top_domains[heap_size].namepos = domain->domainpos;
+			heap_size++;
+		}
+		else
+		{
+			// Build min-heap once on first overflow
+			if(!heap_ready)
+			{
+				for(int j = (int)(heap_size / 2) - 1; j >= 0; j--)
+					heap_sift_down(top_domains, heap_size, (unsigned int)j);
+				heap_ready = true;
+			}
+			// Replace root (minimum) if this entry is larger
+			if(entry_count > top_domains[0].count)
+			{
+				top_domains[0].count = entry_count;
+				top_domains[0].namepos = domain->domainpos;
+				heap_sift_down(top_domains, heap_size, 0);
+			}
+		}
 	}
 
 	// Unlock shared memory
 	unlock_shm();
 
-	// Sort temporary array
-	qsort(top_domains, added_domains, sizeof(*top_domains), cmpdesc_te);
+	// Sort the small heap array descending
+	if(heap_size > 1)
+		qsort(top_domains, heap_size, sizeof(*top_domains), cmpdesc_te);
 
 	int n = 0;
 	cJSON *jtop_domains = cJSON_CreateArray();
 
 	// Lock shared memory
 	lock_shm();
 
-	for(unsigned int i = 0; i < added_domains; i++)
+	for(unsigned int i = 0; i < heap_size; i++)
 	{
 		// Skip e.g. recycled domains
 		if(top_domains[i].namepos == 0)
@@ -376,14 +428,21 @@ cJSON *get_top_clients(struct ftl_conn *api, const int count,
 	const unsigned int clients = counters->clients;
 	const int total_queries = counters->queries;
 	const int blocked_count = get_blocked_count();
-	struct top_entries *top_clients = calloc(clients, sizeof(struct top_entries));
-	if(top_clients == NULL)
+
+	// Heap-based top-K selection: allocate only for the top entries
+	// rather than for all clients, reducing memory from O(N) to O(K).
+	const unsigned int k = count > 0 ? (unsigned int)count : 0u;
+	const unsigned int heap_cap = (k <= clients / 4) ? k * 4 : clients;
+	struct top_entries *top_clients = heap_cap > 0 ? calloc(heap_cap, sizeof(struct top_entries)) : NULL;
+	if(heap_cap > 0 && top_clients == NULL)
 	{
 		log_err("Memory allocation failed in %s()", __FUNCTION__);
+		unlock_shm();
 		return 0;
 	}
 
-	unsigned int added_clients = 0;
+	unsigned int heap_size = 0;
+	bool heap_ready = false;
 	for(unsigned int clientID = 0; clientID < clients; clientID++)
 	{
 		// Get client pointer
@@ -413,22 +472,48 @@ cJSON *get_top_clients(struct ftl_conn *api, const int count,
 		}
 
 		// Use either blocked or total count based on request string
-		top_clients[added_clients].count = blocked ? client->blockedcount : client->count;
+		const int entry_count = blocked ? client->blockedcount : client->count;
 
-		// Get client name and IP
-		top_clients[added_clients].ippos = client->ippos;
-		top_clients[added_clients].namepos = client->namepos;
+		// Skip zero-count entries early
+		if(entry_count < 1)
+			continue;
 
-		added_clients++;
+		if(heap_size < heap_cap)
+		{
+			// Heap not full yet, append directly
+			top_clients[heap_size].count = entry_count;
+			top_clients[heap_size].ippos = client->ippos;
+			top_clients[heap_size].namepos = client->namepos;
+			heap_size++;
+		}
+		else
+		{
+			// Build min-heap once on first overflow
+			if(!heap_ready)
+			{
+				for(int j = (int)(heap_size / 2) - 1; j >= 0; j--)
+					heap_sift_down(top_clients, heap_size, (unsigned int)j);
+				heap_ready = true;
+			}
+			// Replace root (minimum) if this entry is larger
+			if(entry_count > top_clients[0].count)
+			{
+				top_clients[0].count = entry_count;
+				top_clients[0].ippos = client->ippos;
+				top_clients[0].namepos = client->namepos;
+				heap_sift_down(top_clients, heap_size, 0);
+			}
+		}
 	}
 
-	log_debug(DEBUG_API, "Found %u clients", added_clients);
+	log_debug(DEBUG_API, "Found %u clients (heap selected from %u)", heap_size, clients);
 
 	// Unlock shared memory
 	unlock_shm();
 
-	// Sort temporary array
-	qsort(top_clients, added_clients, sizeof(*top_clients), cmpdesc_te);
+	// Sort the small heap array descending
+	if(heap_size > 1)
+		qsort(top_clients, heap_size, sizeof(*top_clients), cmpdesc_te);
 
 	// Get clients which the user doesn't want to see
 	regex_t *regex_clients = NULL;
@@ -443,7 +528,7 @@ cJSON *get_top_clients(struct ftl_conn *api, const int count,
 	// Lock shared memory
 	lock_shm();
 
-	for(unsigned int i = 0; i < added_clients; i++)
+	for(unsigned int i = 0; i < heap_size; i++)
 	{
 		const char *client_ip = getstr(top_clients[i].ippos);
 		const char *client_name = getstr(top_clients[i].namepos);
@@ -596,7 +681,7 @@ cJSON *get_top_upstreams(struct ftl_conn *api, const bool upstreams_only)
 	unlock_shm();
 
 	// Sort temporary array in descending order
-	qsort(top_upstreams, added_upstreams, sizeof(*top_upstreams), cmpdesc);
+	qsort(top_upstreams, added_upstreams, sizeof(*top_upstreams), cmpdesc_te);
 
 	// Loop over available forward destinations
 	cJSON *jtop_upstreams = JSON_NEW_ARRAY();
@@ -770,7 +855,7 @@ int api_stats_recentblocked(struct ftl_conn *api)
 	// Find most recently blocked query
 	unsigned int found = 0;
 	cJSON *blocked = JSON_NEW_ARRAY();
-	for(int queryID = counters->queries - 1; queryID > 0 ; queryID--)
+	for(int queryID = counters->queries - 1; queryID >= 0 ; queryID--)
 	{
 		const queriesData *query = getQuery(queryID, true);
 		if(query == NULL)