Call malloc_trim() regularly to improve deallocation behavior (#14707)

This commit is contained in:
sfan5 2024-06-07 16:57:30 +02:00 committed by GitHub
parent 08485f6781
commit 71893807b3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 99 additions and 4 deletions

@ -176,6 +176,32 @@ public:
}
return 0;
}
//! Calculate size of vertices and indices in memory
virtual size_t getSize() const
{
size_t ret = 0;
switch (getVertexType()) {
case video::EVT_STANDARD:
ret += sizeof(video::S3DVertex) * getVertexCount();
break;
case video::EVT_2TCOORDS:
ret += sizeof(video::S3DVertex2TCoords) * getVertexCount();
break;
case video::EVT_TANGENTS:
ret += sizeof(video::S3DVertexTangents) * getVertexCount();
break;
}
switch (getIndexType()) {
case video::EIT_16BIT:
ret += sizeof(u16) * getIndexCount();
break;
case video::EIT_32BIT:
ret += sizeof(u32) * getIndexCount();
break;
}
return ret;
}
};
} // end namespace scene

@ -351,6 +351,12 @@ endif()
include(CheckSymbolExists)
check_symbol_exists(strlcpy "string.h" HAVE_STRLCPY)
if(UNIX)
check_symbol_exists(malloc_trim "malloc.h" HAVE_MALLOC_TRIM)
else()
set(HAVE_MALLOC_TRIM FALSE)
endif()
check_include_files(endian.h HAVE_ENDIAN_H)
configure_file(

@ -800,11 +800,16 @@ MapBlockMesh::MapBlockMesh(Client *client, MeshMakeData *data, v3s16 camera_offs
MapBlockMesh::~MapBlockMesh()
{
size_t sz = 0;
for (scene::IMesh *m : m_mesh) {
for (u32 i = 0; i < m->getMeshBufferCount(); i++)
sz += m->getMeshBuffer(i)->getSize();
m->drop();
}
for (MinimapMapblock *block : m_minimap_mapblocks)
delete block;
porting::TrackFreedMemory(sz);
}
bool MapBlockMesh::animate(bool faraway, float time, int crack,

@ -31,6 +31,7 @@
#cmakedefine01 USE_REDIS
#cmakedefine01 HAVE_ENDIAN_H
#cmakedefine01 HAVE_STRLCPY
#cmakedefine01 HAVE_MALLOC_TRIM
#cmakedefine01 CURSES_HAVE_CURSES_H
#cmakedefine01 CURSES_HAVE_NCURSES_H
#cmakedefine01 CURSES_HAVE_NCURSES_NCURSES_H

@ -85,6 +85,7 @@ MapBlock::~MapBlock()
#endif
delete[] data;
porting::TrackFreedMemory(sizeof(MapNode) * nodecount);
}
bool MapBlock::onObjectsActivation()

@ -19,7 +19,6 @@ with this program; if not, write to the Free Software Foundation, Inc.,
*/
#include <fstream>
#include <typeinfo>
#include "mg_schematic.h"
#include "server.h"
#include "mapgen.h"
@ -32,6 +31,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "serialization.h"
#include "filesys.h"
#include "voxelalgorithms.h"
#include "porting.h"
///////////////////////////////////////////////////////////////////////////////
@ -80,6 +80,8 @@ Schematic::~Schematic()
{
delete []schemdata;
delete []slice_probs;
u32 nodecount = size.X * size.Y * size.Z;
porting::TrackFreedMemory(nodecount * sizeof(MapNode));
}
ObjDef *Schematic::clone() const

@ -63,7 +63,11 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include <FindDirectory.h>
#endif
#include "config.h"
#if HAVE_MALLOC_TRIM
// glibc-only pretty much
#include <malloc.h>
#endif
#include "debug.h"
#include "filesys.h"
#include "log.h"
@ -72,6 +76,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include <cstdarg>
#include <cstdio>
#include <signal.h>
#include <atomic>
#if !defined(SERVER) && defined(_WIN32)
// On Windows export some driver-specific variables to encourage Minetest to be
@ -903,4 +908,37 @@ double perf_freq = get_perf_freq();
#endif
#if HAVE_MALLOC_TRIM
/*
* On Linux/glibc we found that after deallocating bigger chunks of data (esp. MapBlocks)
* the memory would not be given back to the OS and would stay at peak usage.
* This appears to be a combination of unfortunate allocation order/fragmentation
* and the fact that glibc does not call madvise(MADV_DONTNEED) on its own.
* Some other allocators were also affected, jemalloc and musl libc were not.
* read more: <https://forum.minetest.net/viewtopic.php?t=30509>
*
* As a workaround we track freed memory coarsely and call malloc_trim() once a
* certain amount is reached.
*/
static std::atomic<size_t> memory_freed;
constexpr size_t MEMORY_TRIM_THRESHOLD = 128 * 1024 * 1024;
void TrackFreedMemory(size_t amount)
{
constexpr auto MO = std::memory_order_relaxed;
memory_freed.fetch_add(amount, MO);
if (memory_freed.load(MO) >= MEMORY_TRIM_THRESHOLD) {
// Synchronize call
if (memory_freed.exchange(0, MO) < MEMORY_TRIM_THRESHOLD)
return;
// Leave some headroom for future allocations
malloc_trim(1 * 1024 * 1024);
}
}
#endif
} //namespace porting

@ -290,6 +290,17 @@ void osSpecificInit();
// This attaches to the parents process console, or creates a new one if it doesnt exist.
void attachOrCreateConsole();
/**
* Call this after freeing bigger blocks of memory. Used on some platforms to
* properly give memory back to the OS.
* @param amount Number of bytes freed
*/
#if HAVE_MALLOC_TRIM
void TrackFreedMemory(size_t amount);
#else
inline void TrackFreedMemory(size_t amount) { (void)amount; }
#endif
#ifdef _WIN32
// Quotes an argument for use in a CreateProcess() commandline (not cmd.exe!!)
std::string QuoteArgv(const std::string &arg);
@ -298,6 +309,7 @@ std::string QuoteArgv(const std::string &arg);
std::string ConvertError(DWORD error_code);
#endif
// snprintf wrapper
int mt_snprintf(char *buf, const size_t buf_size, const char *fmt, ...);
/**

@ -23,6 +23,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "nodedef.h"
#include "util/directiontables.h"
#include "util/timetaker.h"
#include "porting.h"
#include <cstring> // memcpy, memset
/*
@ -40,12 +41,15 @@ VoxelManipulator::~VoxelManipulator()
void VoxelManipulator::clear()
{
// Reset area to volume=0
m_area = VoxelArea();
// Reset area to empty volume
VoxelArea old;
std::swap(m_area, old);
delete[] m_data;
m_data = nullptr;
delete[] m_flags;
m_flags = nullptr;
porting::TrackFreedMemory((sizeof(*m_data) + sizeof(*m_flags)) * old.getVolume());
}
void VoxelManipulator::print(std::ostream &o, const NodeDefManager *ndef,