Add benchmarks for json string serialize/deserialize (#12258)

Co-authored-by: sfan5 <sfan5@live.de>

CMakeLists.txt

@@ -52,6 +52,7 @@ set(RUN_IN_PLACE ${DEFAULT_RUN_IN_PLACE} CACHE BOOL
 set(BUILD_CLIENT TRUE CACHE BOOL "Build client")
 set(BUILD_SERVER FALSE CACHE BOOL "Build server")
 set(BUILD_UNITTESTS TRUE CACHE BOOL "Build unittests")
+set(BUILD_BENCHMARKS FALSE CACHE BOOL "Build benchmarks")
 
 set(WARN_ALL TRUE CACHE BOOL "Enable -Wall for Release build")
 
@@ -280,6 +281,10 @@ elseif(CMAKE_CXX_COMPILER_ID MATCHES "(Apple)?Clang")
 	endif()
 endif()
 
+if(BUILD_BENCHMARKS)
+	add_subdirectory(lib/catch2)
+endif()
+
 # Subdirectories
 # Be sure to add all relevant definitions above this
 add_subdirectory(src)

README.md

@@ -240,6 +240,7 @@ General options and their default values:
     BUILD_CLIENT=TRUE          - Build Minetest client
     BUILD_SERVER=FALSE         - Build Minetest server
     BUILD_UNITTESTS=TRUE       - Build unittest sources
+    BUILD_BENCHMARKS=FALSE     - Build benchmark sources
     CMAKE_BUILD_TYPE=Release   - Type of build (Release vs. Debug)
         Release                - Release build
         Debug                  - Debug build

lib/catch2/CMakeLists.txt

@@ -0,0 +1,16 @@
+# catch2 is distributed as a standalone header.
+#
+# Downloaded from:
+#
+#      https://github.com/catchorg/Catch2/releases/download/v2.13.9/catch.hpp
+#
+# The following changes were made to always print in microseconds, fixed format:
+#
+# -    explicit Duration(double inNanoseconds, Unit units = Unit::Auto)
+# +    explicit Duration(double inNanoseconds, Unit units = Unit::Microseconds)
+#
+# -        return os << duration.value() << ' ' << duration.unitsAsString();
+# +        return os << std::fixed << duration.value() << ' ' << duration.unitsAsString();
+
+add_library(catch2 INTERFACE)
+target_include_directories(catch2 INTERFACE ${CMAKE_CURRENT_SOURCE_DIR})

src/CMakeLists.txt

@@ -330,6 +330,7 @@ add_subdirectory(mapgen)
 add_subdirectory(network)
 add_subdirectory(script)
 add_subdirectory(unittest)
+add_subdirectory(benchmark)
 add_subdirectory(util)
 add_subdirectory(irrlicht_changes)
 add_subdirectory(server)
@@ -412,6 +413,9 @@ if(BUILD_UNITTESTS)
 	set(common_SRCS ${common_SRCS} ${UNITTEST_SRCS})
 endif()
 
+if(BUILD_BENCHMARKS)
+	set(common_SRCS ${common_SRCS} ${BENCHMARK_SRCS})
+endif()
 
 # This gives us the icon and file version information
 if(WIN32)
@@ -452,6 +456,10 @@ if(BUILD_UNITTESTS)
 	set(client_SRCS ${client_SRCS} ${UNITTEST_CLIENT_SRCS})
 endif()
 
+if(BUILD_BENCHMARKS)
+	set(client_SRCS ${client_SRCS} ${BENCHMARK_CLIENT_SRCS})
+endif()
+
 list(SORT client_SRCS)
 
 # Server sources
@@ -567,6 +575,9 @@ if(BUILD_CLIENT)
 	if (USE_SPATIAL)
 		target_link_libraries(${PROJECT_NAME} ${SPATIAL_LIBRARY})
 	endif()
+	if(BUILD_BENCHMARKS)
+		target_link_libraries(${PROJECT_NAME} catch2)
+	endif()
 endif(BUILD_CLIENT)
 
 
@@ -626,6 +637,9 @@ if(BUILD_SERVER)
 			${CURL_LIBRARY}
 		)
 	endif()
+	if(BUILD_BENCHMARKS)
+		target_link_libraries(${PROJECT_NAME}server catch2)
+	endif()
 endif(BUILD_SERVER)
 
 # Blacklisted locales that don't work.

src/benchmark/CMakeLists.txt

@@ -0,0 +1,7 @@
+set (BENCHMARK_SRCS
+	${CMAKE_CURRENT_SOURCE_DIR}/benchmark.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/benchmark_serialize.cpp
+	PARENT_SCOPE)
+
+set (BENCHMARK_CLIENT_SRCS
+	PARENT_SCOPE)

src/benchmark/benchmark.cpp

@@ -0,0 +1,32 @@
+/*
+Minetest
+Copyright (C) 2022 Minetest Authors
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#include "benchmark/benchmark.h"
+
+// This must be set in just this file
+#define CATCH_CONFIG_RUNNER
+#include "benchmark_setup.h"
+
+int run_benchmarks()
+{
+	int argc = 1;
+	const char *argv[] = { "MinetestBenchmark", NULL };
+	int errCount = Catch::Session().run(argc, argv);
+	return errCount ? 1 : 0;
+}

src/benchmark/benchmark.h

@@ -0,0 +1,26 @@
+/*
+Minetest
+Copyright (C) 2022 Minetest Authors
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#pragma once
+
+#include "config.h"
+
+#if BUILD_BENCHMARKS
+extern int run_benchmarks();
+#endif

src/benchmark/benchmark_serialize.cpp

@@ -0,0 +1,71 @@
+/*
+Minetest
+Copyright (C) 2022 Minetest Authors
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#include "benchmark_setup.h"
+#include "util/serialize.h"
+#include <sstream>
+#include <ios>
+
+// Builds a string of exactly `length` characters by repeating `s` (rest cut off)
+static std::string makeRepeatTo(const std::string &s, size_t length)
+{
+	std::string v;
+	v.reserve(length + s.size());
+	for (size_t i = 0; i < length; i += s.size()) {
+		v += s;
+	}
+	v.resize(length);
+	return v;
+}
+
+#define BENCH3(_label, _chars, _length, _lengthlabel) \
+	BENCHMARK_ADVANCED("serializeJsonStringIfNeeded_" _lengthlabel "_" _label)(Catch::Benchmark::Chronometer meter) { \
+		std::string s = makeRepeatTo(_chars, _length); \
+		meter.measure([&] { return serializeJsonStringIfNeeded(s); }); \
+	}; \
+	BENCHMARK_ADVANCED("deSerializeJsonStringIfNeeded_" _lengthlabel "_" _label)(Catch::Benchmark::Chronometer meter) { \
+		std::string s = makeRepeatTo(_chars, _length); \
+		std::string serialized = serializeJsonStringIfNeeded(s); \
+		std::istringstream is(serialized, std::ios::binary); \
+		meter.measure([&] { \
+			is.clear(); \
+			is.seekg(0, std::ios::beg); \
+			return deSerializeJsonStringIfNeeded(is); \
+		}); \
+	};
+
+/* Both with and without a space character (' ') */
+#define BENCH2(_label, _chars, _length, _lengthlabel) \
+	BENCH3(_label, _chars, _length, _lengthlabel) \
+	BENCH3(_label "_with_space", " " _chars, _length, _lengthlabel) \
+
+/* Iterate over input lengths */
+#define BENCH1(_label, _chars) \
+	BENCH2(_label, _chars, 10, "small") \
+	BENCH2(_label, _chars, 10000, "large")
+
+/* Iterate over character sets */
+#define BENCH_ALL() \
+	BENCH1("alpha", "abcdefghijklmnopqrstuvwxyz") \
+	BENCH1("escaped", "\"\\/\b\f\n\r\t") \
+	BENCH1("nonascii", "\xf0\xff")
+
+TEST_CASE("benchmark_serialize") {
+	BENCH_ALL()
+}

src/benchmark/benchmark_setup.h

@@ -0,0 +1,22 @@
+/*
+Minetest
+Copyright (C) 2022 Minetest Authors
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#define CATCH_CONFIG_ENABLE_BENCHMARKING
+#define CATCH_CONFIG_CONSOLE_WIDTH 160
+#include <catch.hpp>

src/cmake_config.h.in

@@ -35,3 +35,4 @@
 #cmakedefine01 CURSES_HAVE_NCURSESW_NCURSES_H
 #cmakedefine01 CURSES_HAVE_NCURSESW_CURSES_H
 #cmakedefine01 BUILD_UNITTESTS
+#cmakedefine01 BUILD_BENCHMARKS

src/main.cpp

@@ -20,6 +20,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "irrlichttypes.h" // must be included before anything irrlicht, see comment in the file
 #include "irrlicht.h" // createDevice
 #include "irrlichttypes_extrabloated.h"
+#include "benchmark/benchmark.h"
 #include "chat_interface.h"
 #include "debug.h"
 #include "unittest/test.h"
@@ -212,7 +213,19 @@ int main(int argc, char *argv[])
 		return 1;
 #endif
 	}
+
+	// Run benchmarks
+	if (cmd_args.getFlag("run-benchmarks")) {
+#if BUILD_BENCHMARKS
+		return run_benchmarks();
+#else
+		errorstream << "Benchmark support is not enabled in this binary. "
+			<< "If you want to enable it, compile project with BUILD_BENCHMARKS=1 flag."
+			<< std::endl;
+		return 1;
 #endif
+	}
+#endif // __ANDROID__
 
 	GameStartData game_params;
 #ifdef SERVER
@@ -277,6 +290,8 @@ static void set_allowed_options(OptionList *allowed_options)
 			_("Set network port (UDP)"))));
 	allowed_options->insert(std::make_pair("run-unittests", ValueSpec(VALUETYPE_FLAG,
 			_("Run the unit tests and exit"))));
+	allowed_options->insert(std::make_pair("run-benchmarks", ValueSpec(VALUETYPE_FLAG,
+			_("Run the benchmarks and exit"))));
 	allowed_options->insert(std::make_pair("map-dir", ValueSpec(VALUETYPE_STRING,
 			_("Same as --world (deprecated)"))));
 	allowed_options->insert(std::make_pair("world", ValueSpec(VALUETYPE_STRING,

util/ci/build.sh

@@ -1,6 +1,10 @@
 #! /bin/bash -e
 
-cmake -B build -DCMAKE_BUILD_TYPE=Debug \
-	-DRUN_IN_PLACE=TRUE -DENABLE_GETTEXT=TRUE \
-	-DBUILD_SERVER=TRUE ${CMAKE_FLAGS}
+cmake -B build \
+	-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Debug} \
+	-DRUN_IN_PLACE=TRUE \
+	-DENABLE_GETTEXT=${CMAKE_ENABLE_GETTEXT:-TRUE} \
+	-DBUILD_SERVER=${CMAKE_BUILD_SERVER:-TRUE} \
+	${CMAKE_FLAGS}
+
 cmake --build build --parallel $(($(nproc) + 1))

util/ci/common.sh

@@ -7,7 +7,9 @@ install_linux_deps() {
 		libhiredis-dev libogg-dev libgmp-dev libvorbis-dev libopenal-dev \
 		gettext libpq-dev libleveldb-dev libcurl4-openssl-dev libzstd-dev)
 
-	if [[ "$1" == "--old-irr" ]]; then
+	if [[ "$1" == "--no-irr" ]]; then
+		shift
+	elif [[ "$1" == "--old-irr" ]]; then
 		shift
 		pkgs+=(libirrlicht-dev)
 	else