refactor(core): improve backend storage robustness, error handling, and unit test coverage

.github/workflows/build-and-test.yml

@@ -228,12 +228,8 @@ jobs:
   FreeBSD:
     env:
       CFLAGS: --coverage
-      CTEST_CLIENT_HOST: ::1
-      CTEST_SERVER_HOST: localhost
       CXXFLAGS: --coverage
       INSTALL_PREFIX: /usr/local
-      LIBRARY_PATH: /usr/local/lib
-      TMPDIR: /tmp
     name: >-
       FreeBSD
       ${{ matrix.freebsd_release }}

CMakeLists.txt

@@ -106,6 +106,11 @@ if(NOT MALLOC_LIB STREQUAL "libc")
   endif()
 endif()
 
+# Use /usr/local/lib for FreeBSD
+if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
+  link_directories(/usr/local/lib)
+endif()
+
 #-----------------------------------------------------------------------------
 #
 #  Set variables

Makefile.am

@@ -132,7 +132,7 @@ render_speedtest_test_LDADD = $(GLIB_LIBS) catch_main.o catch_test_common.o
 unit_tests_SOURCES = \
 	tests/unit_tests.cpp
 unit_tests_CXXFLAGS = -Isrc
-unit_tests_LDADD = $(GLIB_LIBS) $(INIPARSER_LDFLAGS) $(MAPNIK_LDFLAGS) $(PTHREAD_CFLAGS) catch_main.o catch_test_common.o
+unit_tests_LDADD = $(STORE_LDFLAGS) $(INIPARSER_LDFLAGS) $(MAPNIK_LDFLAGS) $(PTHREAD_CFLAGS) catch_main.o catch_test_common.o
 
 CLEANFILES=*.slo mod_tile.la stderr.out src/*.slo src/*.lo src/.libs/* src/*.la
 

docs/build/building_on_freebsd.md

@@ -34,8 +34,6 @@ sudo pkg install --yes \
 
 # Download, Build, Test & Install `mod_tile`
 export CMAKE_BUILD_PARALLEL_LEVEL=$(sysctl -n hw.ncpu)
-export CTEST_CLIENT_HOST="::1"
-export CTEST_SERVER_HOST="localhost"
 export LIBRARY_PATH="/usr/local/lib"
 rm -rf /tmp/mod_tile_src /tmp/mod_tile_build
 mkdir /tmp/mod_tile_src /tmp/mod_tile_build

includes/render_submit_queue.h

@@ -17,16 +17,41 @@
 
 #ifndef RENDERSUBQUEUE_H
 #define RENDERSUBQUEUE_H
+
+#include <sys/time.h>
+
+#include "render_config.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+struct qItem {
+	char *mapname;
+	int x, y, z;
+	struct qItem *next;
+};
+
+struct speed_stat {
+	time_t time_min;
+	time_t time_max;
+	time_t time_total;
+	int noRendered;
+};
+
+struct speed_stats {
+	struct speed_stat stat[MAX_ZOOM + 1];
+};
+
 int make_connection(const char *spath);
+void check_load(void);
+void display_rate(struct timeval start, struct timeval end, int num);
 void enqueue(const char *xmlname, int x, int y, int z);
-void spawn_workers(int num, const char *socketpath, int maxLoad);
-void wait_for_empty_queue(void);
 void finish_workers(void);
 void print_statistics(void);
+void spawn_workers(int num, const char *socketpath, int maxLoad);
+void *thread_main(void *arg);
+void wait_for_empty_queue(void);
 
 #ifdef __cplusplus
 }

src/CMakeLists.txt

@@ -239,6 +239,7 @@ set(unit_tests_LIBS
   ${COMMON_LIBRARIES}
   ${INIPARSER_LIBRARIES}
   ${LIBMAPNIK_LIBRARIES}
+  ${STORE_LIBRARIES}
 )
 add_executable(unit_tests ${unit_tests_SRCS})
 set_target_properties(unit_tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/tests)

src/parameterize_style.cpp

@@ -83,7 +83,8 @@ static void parameterize_map_language(mapnik::Map &m, char *parameter)
 
 parameterize_function_ptr init_parameterization_function(const char *function_name)
 {
-	if (strcmp(function_name, "") == 0) {
+
+	if (!function_name || strcmp(function_name, "") == 0) {
 		g_logger(G_LOG_LEVEL_DEBUG, "Parameterize_style not specified (or empty string specified)");
 		return NULL;
 	} else if (strcmp(function_name, "language") == 0) {

src/protocol_helper.c

@@ -27,13 +27,26 @@
 int send_cmd(struct protocol * cmd, int fd)
 {
 	int ret;
+	int socket_type;
+	socklen_t optlen = sizeof(socket_type);
+
+	if (fd < 0) {
+		g_logger(G_LOG_LEVEL_WARNING, "Attempted to send cmd on invalid fd %d", fd);
+		return -1;
+	}
+
 	g_logger(G_LOG_LEVEL_DEBUG, "Sending render cmd(%i %s %i/%i/%i) with protocol version %i to fd %i", cmd->cmd, cmd->xmlname, cmd->z, cmd->x, cmd->y, cmd->ver, fd);
 
 	if ((cmd->ver > 3) || (cmd->ver < 1)) {
 		g_logger(G_LOG_LEVEL_ERROR, "Failed to send render cmd with unknown protocol version %i on fd %d", cmd->ver, fd);
 		return -1;
 	}
 
+	if (getsockopt(fd, SOL_SOCKET, SO_TYPE, &socket_type, &optlen) < 0) {
+		g_logger(G_LOG_LEVEL_ERROR, "fd %d is not a valid socket: %s", fd, strerror(errno));
+		return -1;
+	}
+
 	switch (cmd->ver) {
 		case 1:
 			ret = send(fd, cmd, sizeof(struct protocol_v1), 0);

src/render_expired.c

@@ -54,19 +54,6 @@ static void set_tile_requested(int z, int x, int y)
 	g_hash_table_add(tile_requested, key);
 }
 
-void display_rate(struct timeval start, struct timeval end, int num)
-{
-	int d_s, d_us;
-	float sec;
-
-	d_s = end.tv_sec - start.tv_sec;
-	d_us = end.tv_usec - start.tv_usec;
-
-	sec = d_s + d_us / 1000000.0;
-
-	g_logger(G_LOG_LEVEL_MESSAGE, "\t%d in %.2f seconds (%.2f/s)", num, sec, num / sec);
-}
-
 int main(int argc, char **argv)
 {
 	const char *config_file_name_default = RENDERD_CONFIG;

src/render_list.c

@@ -45,19 +45,6 @@ int lat2tiley(double lat, int z)
 	return (int)(floor((1.0 - log(tan(latrad) + (1.0 / cos(latrad))) / M_PI) / 2.0 * pow(2.0, z)));
 }
 
-void display_rate(struct timeval start, struct timeval end, int num)
-{
-	int d_s, d_us;
-	float sec;
-
-	d_s = end.tv_sec - start.tv_sec;
-	d_us = end.tv_usec - start.tv_usec;
-
-	sec = d_s + d_us / 1000000.0;
-
-	g_logger(G_LOG_LEVEL_MESSAGE, "\t%d in %.2f seconds (%.2f/s)", num, sec, num / sec);
-}
-
 int main(int argc, char **argv)
 {
 	const char *config_file_name_default = RENDERD_CONFIG;

src/render_old.c

@@ -34,26 +34,11 @@
 #include "render_submit_queue.h"
 #include "renderd_config.h"
 #include "store_file_utils.h"
-#include "sys_utils.h"
 
 static int num_render = 0, num_all = 0;
-static int max_load;
 static time_t planet_timestamp = 0;
 static struct timeval start, end;
 
-void display_rate(struct timeval start, struct timeval end, int num)
-{
-	int d_s, d_us;
-	float sec;
-
-	d_s = end.tv_sec - start.tv_sec;
-	d_us = end.tv_usec - start.tv_usec;
-
-	sec = d_s + d_us / 1000000.0;
-
-	g_logger(G_LOG_LEVEL_MESSAGE, "\t%d in %.2f seconds (%.2f/s)", num, sec, num / sec);
-}
-
 static time_t get_planet_time(const char *tile_dir)
 {
 	static time_t last_check;
@@ -85,17 +70,6 @@ static time_t get_planet_time(const char *tile_dir)
 	return planet_time;
 }
 
-static void check_load(void)
-{
-	double avg = get_load_avg();
-
-	while (avg >= max_load) {
-		g_logger(G_LOG_LEVEL_MESSAGE, "Load average %f, sleeping", avg);
-		sleep(5);
-		avg = get_load_avg();
-	}
-}
-
 static void descend(const char *tile_dir, const char *search, int verbose)
 {
 	DIR *tiles = opendir(search);
@@ -178,7 +152,7 @@ int main(int argc, char **argv)
 	const char *mapname = mapname_default;
 	const char *socketname = socketname_default;
 	const char *tile_dir = tile_dir_default;
-	max_load = max_load_default;
+	int max_load = max_load_default;
 	int max_zoom = max_zoom_default;
 	int min_zoom = min_zoom_default;
 	int num_threads = num_threads_default;

src/render_speedtest.cpp

@@ -102,19 +102,6 @@ class GoogleProjection
 	}
 };
 
-void display_rate(struct timeval start, struct timeval end, int num)
-{
-	int d_s, d_us;
-	float sec;
-
-	d_s = end.tv_sec - start.tv_sec;
-	d_us = end.tv_usec - start.tv_usec;
-
-	sec = d_s + d_us / 1000000.0;
-
-	g_logger(G_LOG_LEVEL_MESSAGE, "\t%d in %.2f seconds (%.2f/s)", num, sec, num / sec);
-}
-
 int main(int argc, char **argv)
 {
 	const char *config_file_name_default = RENDERD_CONFIG;