Use openssl's sha1 and sha256, optionally (#15472)

.github/workflows/linux.yml

@@ -49,6 +49,8 @@ jobs:
         env:
           CC: gcc-7
           CXX: g++-7
+          # Test fallback SHA implementations
+          CMAKE_FLAGS: '-DENABLE_OPENSSL=0'
 
       - name: Test
         run: |

doc/compiling/README.md

@@ -30,6 +30,7 @@ General options and their default values:
     ENABLE_POSTGRESQL=ON       - Build with libpq; Enables use of PostgreSQL map backend (PostgreSQL 9.5 or greater recommended)
     ENABLE_REDIS=ON            - Build with libhiredis; Enables use of Redis map backend
     ENABLE_SPATIAL=ON          - Build with LibSpatial; Speeds up AreaStores
+    ENABLE_OPENSSL=ON          - Build with OpenSSL; Speeds up SHA1 and SHA2 hashing
     ENABLE_SOUND=ON            - Build with OpenAL, libogg & libvorbis; in-game sounds
     ENABLE_LTO=<varies>        - Build with IPO/LTO optimizations (smaller and more efficient than regular build)
     ENABLE_LUAJIT=ON           - Build with LuaJIT (much faster than non-JIT Lua)

doc/compiling/linux.md

@@ -18,6 +18,7 @@
 | JsonCPP    | 1.0.0+  | Bundled JsonCPP is used if not present |
 | Curl       | 7.56.0+ | Optional   |
 | gettext    | -       | Optional   |
+| OpenSSL    | 3.0+    | Optional (only libcrypto used) |
 
 For Debian/Ubuntu users:
 

src/CMakeLists.txt

@@ -249,6 +249,19 @@ if(ENABLE_SPATIAL)
 	endif(SPATIAL_LIBRARY AND SPATIAL_INCLUDE_DIR)
 endif(ENABLE_SPATIAL)
 
+option(ENABLE_OPENSSL "Use OpenSSL's libcrypto for faster SHA implementations" TRUE)
+set(USE_OPENSSL FALSE)
+
+if(ENABLE_OPENSSL)
+	find_package(OpenSSL 3.0)
+	if(OPENSSL_FOUND)
+		set(USE_OPENSSL TRUE)
+		message(STATUS "OpenSSL's libcrypto SHA enabled.")
+	else()
+		message(STATUS "OpenSSL not found!")
+	endif()
+endif(ENABLE_OPENSSL)
+
 
 find_package(ZLIB REQUIRED)
 find_package(Zstd REQUIRED)
@@ -593,6 +606,9 @@ add_dependencies(EngineCommon GenerateVersion)
 target_link_libraries(EngineCommon
 	sha256
 )
+if(USE_OPENSSL)
+	target_link_libraries(EngineCommon OpenSSL::Crypto)
+endif()
 get_target_property(
 	IRRLICHT_INCLUDES IrrlichtMt::IrrlichtMt INTERFACE_INCLUDE_DIRECTORIES)
 target_include_directories(EngineCommon PRIVATE ${IRRLICHT_INCLUDES})
@@ -726,6 +742,9 @@ if(BUILD_CLIENT)
 	if (USE_SPATIAL)
 		target_link_libraries(${PROJECT_NAME} ${SPATIAL_LIBRARY})
 	endif()
+	if (USE_OPENSSL)
+		target_link_libraries(${PROJECT_NAME} OpenSSL::Crypto)
+	endif()
 	if(BUILD_UNITTESTS OR BUILD_BENCHMARKS)
 		target_link_libraries(${PROJECT_NAME} Catch2::Catch2)
 	endif()
@@ -795,6 +814,9 @@ if(BUILD_SERVER)
 	if (USE_SPATIAL)
 		target_link_libraries(${PROJECT_NAME}server ${SPATIAL_LIBRARY})
 	endif()
+	if (USE_OPENSSL)
+		target_link_libraries(${PROJECT_NAME}server OpenSSL::Crypto)
+	endif()
 	if(USE_CURL)
 		target_link_libraries(
 			${PROJECT_NAME}server

src/benchmark/CMakeLists.txt

@@ -5,6 +5,7 @@ set (BENCHMARK_SRCS
 	${CMAKE_CURRENT_SOURCE_DIR}/benchmark_serialize.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/benchmark_mapblock.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/benchmark_mapmodify.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/benchmark_sha.cpp
 	PARENT_SCOPE)
 
 set (BENCHMARK_CLIENT_SRCS

src/benchmark/benchmark_sha.cpp

@@ -0,0 +1,23 @@
+// SPDX-FileCopyrightText: 2024 Luanti Contributors
+//
+// SPDX-License-Identifier: LGPL-2.1-or-later
+
+#include "catch.h"
+#include "util/hashing.h"
+#include <string>
+
+TEST_CASE("benchmark_sha")
+{
+	std::string input;
+	input.resize(100000);
+
+	BENCHMARK("sha1_input100000", i) {
+		input[0] = (char)i;
+		return hashing::sha1(input);
+	};
+
+	BENCHMARK("sha256_input100000", i) {
+		input[0] = (char)i;
+		return hashing::sha256(input);
+	};
+}

src/client/clientmedia.cpp

@@ -13,7 +13,7 @@
 #include "settings.h"
 #include "util/hex.h"
 #include "util/serialize.h"
-#include "util/sha1.h"
+#include "util/hashing.h"
 #include "util/string.h"
 #include <sstream>
 
@@ -537,12 +537,7 @@ bool IClientMediaDownloader::checkAndLoad(
 	std::string sha1_hex = hex_encode(sha1);
 
 	// Compute actual checksum of data
-	std::string data_sha1;
+	std::string data_sha1 = hashing::sha1(data);
-	{
-		SHA1 ctx;
-		ctx.addBytes(data);
-		data_sha1 = ctx.getDigest();
-	}
 
 	// Check that received file matches announced checksum
 	if (data_sha1 != sha1) {

src/cmake_config.h.in

@@ -29,6 +29,7 @@
 #cmakedefine01 USE_SYSTEM_GMP
 #cmakedefine01 USE_SYSTEM_JSONCPP
 #cmakedefine01 USE_REDIS
+#cmakedefine01 USE_OPENSSL
 #cmakedefine01 HAVE_ENDIAN_H
 #cmakedefine01 HAVE_STRLCPY
 #cmakedefine01 HAVE_MALLOC_TRIM

src/network/clientpackethandler.cpp

@@ -28,7 +28,7 @@
 #include "script/scripting_client.h"
 #include "util/serialize.h"
 #include "util/srp.h"
-#include "util/sha1.h"
+#include "util/hashing.h"
 #include "tileanimation.h"
 #include "gettext.h"
 #include "skyparams.h"
@@ -1645,12 +1645,7 @@ void Client::handleCommand_MediaPush(NetworkPacket *pkt)
 	if (!filedata.empty()) {
 		// LEGACY CODEPATH
 		// Compute and check checksum of data
-		std::string computed_hash;
+		std::string computed_hash = hashing::sha1(filedata);
-		{
-			SHA1 ctx;
-			ctx.addBytes(filedata);
-			computed_hash = ctx.getDigest();
-		}
 		if (raw_hash != computed_hash) {
 			verbosestream << "Hash of file data mismatches, ignoring." << std::endl;
 			return;

src/script/lua_api/l_util.cpp

@@ -27,8 +27,7 @@
 #include "config.h"
 #include "version.h"
 #include "util/hex.h"
-#include "util/sha1.h"
+#include "util/hashing.h"
-#include "my_sha256.h"
 #include "util/png.h"
 #include "player.h"
 #include "daynightratio.h"
@@ -540,12 +539,7 @@ int ModApiUtil::l_sha1(lua_State *L)
 	bool hex = !lua_isboolean(L, 2) || !readParam<bool>(L, 2);
 
 	// Compute actual checksum of data
-	std::string data_sha1;
+	std::string data_sha1 = hashing::sha1(data);
-	{
-		SHA1 ctx;
-		ctx.addBytes(data);
-		data_sha1 = ctx.getDigest();
-	}
 
 	if (hex) {
 		std::string sha1_hex = hex_encode(data_sha1);
@@ -564,10 +558,7 @@ int ModApiUtil::l_sha256(lua_State *L)
 	auto data = readParam<std::string_view>(L, 1);
 	bool hex = !lua_isboolean(L, 2) || !readParam<bool>(L, 2);
 
-	std::string data_sha256;
+	std::string data_sha256 = hashing::sha256(data);
-	data_sha256.resize(SHA256_DIGEST_LENGTH);
-	SHA256(reinterpret_cast<const unsigned char*>(data.data()), data.size(),
-		reinterpret_cast<unsigned char *>(data_sha256.data()));
 
 	if (hex) {
 		lua_pushstring(L, hex_encode(data_sha256).c_str());

src/server.cpp

@@ -44,7 +44,7 @@
 #include "defaultsettings.h"
 #include "server/mods.h"
 #include "util/base64.h"
-#include "util/sha1.h"
+#include "util/hashing.h"
 #include "util/hex.h"
 #include "database/database.h"
 #include "chatmessage.h"
@@ -2548,14 +2548,11 @@ bool Server::addMediaFile(const std::string &filename,
 		return false;
 	}
 
-	SHA1 sha1;
+	std::string sha1 = hashing::sha1(filedata);
-	sha1.addBytes(filedata);
+	std::string sha1_base64 = base64_encode(sha1);
-
+	std::string sha1_hex = hex_encode(sha1);
-	std::string digest = sha1.getDigest();
-	std::string sha1_base64 = base64_encode(digest);
-	std::string sha1_hex = hex_encode(digest);
 	if (digest_to)
-		*digest_to = digest;
+		*digest_to = sha1;
 
 	// Put in list
 	m_media[filename] = MediaInfo(filepath, sha1_base64);

src/unittest/test_map_settings_manager.cpp

@@ -7,7 +7,7 @@
 #include "noise.h"
 #include "settings.h"
 #include "mapgen/mapgen_v5.h"
-#include "util/sha1.h"
+#include "util/hashing.h"
 #include "map_settings_manager.h"
 
 class TestMapSettingsManager : public TestBase {
@@ -171,11 +171,9 @@ void TestMapSettingsManager::testMapSettingsManager()
 		0x78, 0x56, 0x95, 0x2d, 0xdc, 0x6a, 0xf7, 0x61, 0x36, 0x5f
 	};
 
-	SHA1 ctx;
 	std::string metafile_contents;
 	UASSERT(fs::ReadFile(test_mapmeta_path, metafile_contents));
-	ctx.addBytes(metafile_contents);
+	std::string sha1_result = hashing::sha1(metafile_contents);
-	std::string sha1_result = ctx.getDigest();
 	int resultdiff = memcmp(sha1_result.data(), expected_contents_hash, 20);
 
 	UASSERT(!resultdiff);

src/util/CMakeLists.txt

@@ -1,10 +1,11 @@
 set(util_SRCS
 	${CMAKE_CURRENT_SOURCE_DIR}/areastore.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/auth.cpp
-	${CMAKE_CURRENT_SOURCE_DIR}/colorize.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/base64.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/colorize.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/directiontables.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/enriched_string.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/hashing.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/ieee_float.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/metricsbackend.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/numeric.cpp

src/util/auth.cpp

@@ -6,7 +6,7 @@
 #include <string>
 #include "auth.h"
 #include "base64.h"
-#include "sha1.h"
+#include "util/hashing.h"
 #include "srp.h"
 #include "util/string.h"
 #include "debug.h"
@@ -23,9 +23,7 @@ std::string translate_password(const std::string &name,
 		return "";
 
 	std::string slt = name + password;
-	SHA1 sha1;
+	std::string digest = hashing::sha1(slt);
-	sha1.addBytes(slt);
-	std::string digest = sha1.getDigest();
 	std::string pwd = base64_encode(digest);
 	return pwd;
 }

src/util/hashing.cpp

@@ -0,0 +1,50 @@
+// SPDX-FileCopyrightText: 2024 Luanti Contributors
+//
+// SPDX-License-Identifier: LGPL-2.1-or-later
+
+#include "hashing.h"
+
+#include "debug.h"
+#include "config.h"
+#if USE_OPENSSL
+#include <openssl/sha.h>
+#include <openssl/evp.h>
+#else
+#include "util/sha1.h"
+#include "my_sha256.h"
+#endif
+
+namespace hashing
+{
+
+std::string sha1(std::string_view data)
+{
+#if USE_OPENSSL
+	std::string digest(SHA1_DIGEST_SIZE, '\000');
+	auto src = reinterpret_cast<const uint8_t *>(data.data());
+	auto dst = reinterpret_cast<uint8_t *>(digest.data());
+	SHA1(src, data.size(), dst);
+	return digest;
+#else
+	SHA1 sha1;
+	sha1.addBytes(data);
+	return sha1.getDigest();
+#endif
+}
+
+std::string sha256(std::string_view data)
+{
+	std::string digest(SHA256_DIGEST_SIZE, '\000');
+	auto src = reinterpret_cast<const uint8_t *>(data.data());
+	auto dst = reinterpret_cast<uint8_t *>(digest.data());
+#if USE_OPENSSL
+	// can't call SHA256(), because it's defined by our sha256.c fallback
+	auto success = EVP_Digest(src, data.size(), dst, nullptr, EVP_sha256(), nullptr) == 1;
+	FATAL_ERROR_IF(!success, "sha256 failed");
+#else
+	SHA256(src, data.size(), dst);
+#endif
+	return digest;
+}
+
+}

src/util/hashing.h

@@ -0,0 +1,21 @@
+// SPDX-FileCopyrightText: 2024 Luanti Contributors
+//
+// SPDX-License-Identifier: LGPL-2.1-or-later
+
+#pragma once
+
+#include <string>
+#include <string_view>
+
+namespace hashing
+{
+
+// Size of raw digest in bytes
+constexpr size_t SHA1_DIGEST_SIZE = 20;
+constexpr size_t SHA256_DIGEST_SIZE = 32;
+
+// Returns the digest of data
+std::string sha1(std::string_view data);
+std::string sha256(std::string_view data);
+
+}

util/ci/common.sh

@@ -7,6 +7,7 @@ install_linux_deps() {
 		libpng-dev libjpeg-dev libgl1-mesa-dev libsdl2-dev libfreetype-dev
 		libsqlite3-dev libhiredis-dev libogg-dev libgmp-dev libvorbis-dev
 		libopenal-dev libpq-dev libleveldb-dev libcurl4-openssl-dev libzstd-dev
+		libssl-dev
 	)
 
 	sudo apt-get update