Texture cache -> Media cache WIP

This commit is contained in:
Perttu Ahola 2012-03-25 11:50:29 +03:00
parent 4bf5065a9c
commit f801e16b78
8 changed files with 235 additions and 241 deletions

@ -41,16 +41,16 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "filecache.h" #include "filecache.h"
#include "sound.h" #include "sound.h"
static std::string getTextureCacheDir() static std::string getMediaCacheDir()
{ {
return porting::path_user + DIR_DELIM + "cache" + DIR_DELIM + "textures"; return porting::path_user + DIR_DELIM + "cache" + DIR_DELIM + "media";
} }
struct TextureRequest struct MediaRequest
{ {
std::string name; std::string name;
TextureRequest(const std::string &name_=""): MediaRequest(const std::string &name_=""):
name(name_) name(name_)
{} {}
}; };
@ -256,9 +256,9 @@ Client::Client(
m_map_seed(0), m_map_seed(0),
m_password(password), m_password(password),
m_access_denied(false), m_access_denied(false),
m_texture_cache(getTextureCacheDir()), m_media_cache(getMediaCacheDir()),
m_texture_receive_progress(0), m_media_receive_progress(0),
m_textures_received(false), m_media_received(false),
m_itemdef_received(false), m_itemdef_received(false),
m_nodedef_received(false), m_nodedef_received(false),
m_time_of_day_set(false), m_time_of_day_set(false),
@ -1391,7 +1391,7 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
event.deathscreen.camera_point_target_z = camera_point_target.Z; event.deathscreen.camera_point_target_z = camera_point_target.Z;
m_client_event_queue.push_back(event); m_client_event_queue.push_back(event);
} }
else if(command == TOCLIENT_ANNOUNCE_TEXTURES) else if(command == TOCLIENT_ANNOUNCE_MEDIA)
{ {
io::IFileSystem *irrfs = m_device->getFileSystem(); io::IFileSystem *irrfs = m_device->getFileSystem();
video::IVideoDriver *vdrv = m_device->getVideoDriver(); video::IVideoDriver *vdrv = m_device->getVideoDriver();
@ -1403,33 +1403,36 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
// updating content definitions // updating content definitions
assert(!m_mesh_update_thread.IsRunning()); assert(!m_mesh_update_thread.IsRunning());
int num_textures = readU16(is); int num_files = readU16(is);
verbosestream<<"Client received TOCLIENT_ANNOUNCE_MEDIA ("
<<num_files<<" files)"<<std::endl;
core::list<TextureRequest> texture_requests; core::list<MediaRequest> file_requests;
for(int i=0; i<num_textures; i++){ for(int i=0; i<num_files; i++){
bool texture_found = false; bool file_found = false;
//read texture from cache //read file from cache
std::string name = deSerializeString(is); std::string name = deSerializeString(is);
std::string sha1_texture = deSerializeString(is); std::string sha1_file = deSerializeString(is);
// if name contains illegal characters, ignore the texture // if name contains illegal characters, ignore the file
if(!string_allowed(name, TEXTURENAME_ALLOWED_CHARS)){ if(!string_allowed(name, TEXTURENAME_ALLOWED_CHARS)){
errorstream<<"Client: ignoring illegal texture name " errorstream<<"Client: ignoring illegal file name "
<<"sent by server: \""<<name<<"\""<<std::endl; <<"sent by server: \""<<name<<"\""<<std::endl;
continue; continue;
} }
std::string sha1_decoded = base64_decode(sha1_texture); std::string sha1_decoded = base64_decode(sha1_file);
std::ostringstream tmp_os(std::ios_base::binary); std::ostringstream tmp_os(std::ios_base::binary);
bool tex_in_cache = m_texture_cache.loadByChecksum(name, bool file_in_cache = m_media_cache.loadByChecksum(sha1_decoded,
tmp_os, sha1_decoded); tmp_os);
m_texture_name_sha1_map.set(name, sha1_decoded); m_media_name_sha1_map.set(name, sha1_decoded);
if(tex_in_cache) {
if(file_in_cache)
{
SHA1 sha1; SHA1 sha1;
sha1.addBytes(tmp_os.str().c_str(), tmp_os.str().length()); sha1.addBytes(tmp_os.str().c_str(), tmp_os.str().length());
@ -1437,7 +1440,7 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
std::string digest_string = base64_encode(digest, 20); std::string digest_string = base64_encode(digest, 20);
if (digest_string == sha1_texture) { if (digest_string == sha1_file) {
// Silly irrlicht's const-incorrectness // Silly irrlicht's const-incorrectness
Buffer<char> data_rw(tmp_os.str().c_str(), tmp_os.str().size()); Buffer<char> data_rw(tmp_os.str().c_str(), tmp_os.str().size());
@ -1449,7 +1452,7 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
video::IImage *img = vdrv->createImageFromFile(rfile); video::IImage *img = vdrv->createImageFromFile(rfile);
if(!img){ if(!img){
infostream<<"Client: Cannot create image from data of " infostream<<"Client: Cannot create image from data of "
<<"received texture \""<<name<<"\""<<std::endl; <<"received file \""<<name<<"\""<<std::endl;
rfile->drop(); rfile->drop();
} }
else { else {
@ -1457,22 +1460,22 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
img->drop(); img->drop();
rfile->drop(); rfile->drop();
texture_found = true; file_found = true;
} }
} }
else { else {
infostream<<"Client::Texture cached sha1 hash not matching server hash: " infostream<<"Client::Media cached sha1 hash not matching server hash: "
<<name << ": server ->"<<sha1_texture <<" client -> "<<digest_string<<std::endl; <<name << ": server ->"<<sha1_file <<" client -> "<<digest_string<<std::endl;
} }
free(digest); free(digest);
} }
//add texture request //add file request
if (!texture_found) { if (!file_found) {
infostream<<"Client: Adding texture to request list: \"" infostream<<"Client: Adding file to request list: \""
<<name<<"\""<<std::endl; <<name<<"\""<<std::endl;
texture_requests.push_back(TextureRequest(name)); file_requests.push_back(MediaRequest(name));
} }
} }
@ -1482,11 +1485,11 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
m_client_event_queue.push_back(event); m_client_event_queue.push_back(event);
//send Texture request //send Media request
/* /*
u16 command u16 command
u16 number of textures requested u16 number of files requested
for each texture { for each file {
u16 length of name u16 length of name
string name string name
} }
@ -1496,15 +1499,15 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
// Write command // Write command
writeU16(buf, TOSERVER_REQUEST_TEXTURES); writeU16(buf, TOSERVER_REQUEST_MEDIA);
os.write((char*)buf, 2); os.write((char*)buf, 2);
writeU16(buf,texture_requests.size()); writeU16(buf,file_requests.size());
os.write((char*)buf, 2); os.write((char*)buf, 2);
for(core::list<TextureRequest>::Iterator i = texture_requests.begin(); for(core::list<MediaRequest>::Iterator i = file_requests.begin();
i != texture_requests.end(); i++) { i != file_requests.end(); i++) {
os<<serializeString(i->name); os<<serializeString(i->name);
} }
@ -1513,10 +1516,13 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
SharedBuffer<u8> data((u8*)s.c_str(), s.size()); SharedBuffer<u8> data((u8*)s.c_str(), s.size());
// Send as reliable // Send as reliable
Send(0, data, true); Send(0, data, true);
infostream<<"Client: Sending request list to server " <<std::endl; infostream<<"Client: Sending media request list to server ("
<<file_requests.size()<<" files)"<<std::endl;
} }
else if(command == TOCLIENT_TEXTURES) else if(command == TOCLIENT_MEDIA)
{ {
verbosestream<<"Client received TOCLIENT_MEDIA"<<std::endl;
io::IFileSystem *irrfs = m_device->getFileSystem(); io::IFileSystem *irrfs = m_device->getFileSystem();
video::IVideoDriver *vdrv = m_device->getVideoDriver(); video::IVideoDriver *vdrv = m_device->getVideoDriver();
@ -1529,10 +1535,10 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
/* /*
u16 command u16 command
u16 total number of texture bunches u16 total number of file bunches
u16 index of this bunch u16 index of this bunch
u32 number of textures in this bunch u32 number of files in this bunch
for each texture { for each file {
u16 length of name u16 length of name
string name string name
u32 length of data u32 length of data
@ -1541,20 +1547,20 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
*/ */
int num_bunches = readU16(is); int num_bunches = readU16(is);
int bunch_i = readU16(is); int bunch_i = readU16(is);
m_texture_receive_progress = (float)bunch_i / (float)(num_bunches - 1); m_media_receive_progress = (float)bunch_i / (float)(num_bunches - 1);
if(bunch_i == num_bunches - 1) if(bunch_i == num_bunches - 1)
m_textures_received = true; m_media_received = true;
int num_textures = readU32(is); int num_files = readU32(is);
infostream<<"Client: Received textures: bunch "<<bunch_i<<"/" infostream<<"Client: Received files: bunch "<<bunch_i<<"/"
<<num_bunches<<" textures="<<num_textures <<num_bunches<<" files="<<num_files
<<" size="<<datasize<<std::endl; <<" size="<<datasize<<std::endl;
for(int i=0; i<num_textures; i++){ for(int i=0; i<num_files; i++){
std::string name = deSerializeString(is); std::string name = deSerializeString(is);
std::string data = deSerializeLongString(is); std::string data = deSerializeLongString(is);
// if name contains illegal characters, ignore the texture // if name contains illegal characters, ignore the file
if(!string_allowed(name, TEXTURENAME_ALLOWED_CHARS)){ if(!string_allowed(name, TEXTURENAME_ALLOWED_CHARS)){
errorstream<<"Client: ignoring illegal texture name " errorstream<<"Client: ignoring illegal file name "
<<"sent by server: \""<<name<<"\""<<std::endl; <<"sent by server: \""<<name<<"\""<<std::endl;
continue; continue;
} }
@ -1569,22 +1575,25 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
video::IImage *img = vdrv->createImageFromFile(rfile); video::IImage *img = vdrv->createImageFromFile(rfile);
if(!img){ if(!img){
errorstream<<"Client: Cannot create image from data of " errorstream<<"Client: Cannot create image from data of "
<<"received texture \""<<name<<"\""<<std::endl; <<"received file \""<<name<<"\""<<std::endl;
rfile->drop(); rfile->drop();
continue; continue;
} }
fs::CreateAllDirs(getTextureCacheDir()); bool did = fs::CreateAllDirs(getMediaCacheDir());
if(!did){
errorstream<<"Could not create media cache directory"
<<std::endl;
}
{ {
core::map<std::string, std::string>::Node *n; core::map<std::string, std::string>::Node *n;
n = m_texture_name_sha1_map.find(name); n = m_media_name_sha1_map.find(name);
if(n == NULL) if(n == NULL)
errorstream<<"The server sent a texture that has not been announced." errorstream<<"The server sent a file that has not "
<<std::endl; <<"been announced."<<std::endl;
else else
m_texture_cache.updateByChecksum(name, m_media_cache.updateByChecksum(n->getValue(), data);
data, n->getValue());
} }
m_tsrc->insertSourceImage(name, img); m_tsrc->insertSourceImage(name, img);
@ -2358,11 +2367,11 @@ void Client::afterContentReceived()
{ {
assert(m_itemdef_received); assert(m_itemdef_received);
assert(m_nodedef_received); assert(m_nodedef_received);
assert(m_textures_received); assert(m_media_received);
// remove the information about which checksum each texture // remove the information about which checksum each texture
// ought to have // ought to have
m_texture_name_sha1_map.clear(); m_media_name_sha1_map.clear();
// Rebuild inherited images and recreate textures // Rebuild inherited images and recreate textures
m_tsrc->rebuildImagesAndTextures(); m_tsrc->rebuildImagesAndTextures();

@ -290,11 +290,11 @@ public:
std::wstring accessDeniedReason() std::wstring accessDeniedReason()
{ return m_access_denied_reason; } { return m_access_denied_reason; }
float textureReceiveProgress() float mediaReceiveProgress()
{ return m_texture_receive_progress; } { return m_media_receive_progress; }
bool texturesReceived() bool texturesReceived()
{ return m_textures_received; } { return m_media_received; }
bool itemdefReceived() bool itemdefReceived()
{ return m_itemdef_received; } { return m_itemdef_received; }
bool nodedefReceived() bool nodedefReceived()
@ -367,12 +367,11 @@ private:
bool m_access_denied; bool m_access_denied;
std::wstring m_access_denied_reason; std::wstring m_access_denied_reason;
Queue<ClientEvent> m_client_event_queue; Queue<ClientEvent> m_client_event_queue;
FileCache m_texture_cache; FileCache m_media_cache;
// a map of the name and SHA1 checksum of each texture; // Mapping from media file name to SHA1 checksum
// cleared after content has been recieved core::map<std::string, std::string> m_media_name_sha1_map;
core::map<std::string, std::string> m_texture_name_sha1_map; float m_media_receive_progress;
float m_texture_receive_progress; bool m_media_received;
bool m_textures_received;
bool m_itemdef_received; bool m_itemdef_received;
bool m_nodedef_received; bool m_nodedef_received;
friend class FarMesh; friend class FarMesh;

@ -28,7 +28,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
PROTOCOL_VERSION 3: PROTOCOL_VERSION 3:
Base for writing changes here Base for writing changes here
PROTOCOL_VERSION 4: PROTOCOL_VERSION 4:
Add TOCLIENT_TEXTURES Add TOCLIENT_MEDIA
Add TOCLIENT_TOOLDEF Add TOCLIENT_TOOLDEF
Add TOCLIENT_NODEDEF Add TOCLIENT_NODEDEF
Add TOCLIENT_CRAFTITEMDEF Add TOCLIENT_CRAFTITEMDEF
@ -215,13 +215,13 @@ enum ToClientCommand
v3f1000 camera point target (to point the death cause or whatever) v3f1000 camera point target (to point the death cause or whatever)
*/ */
TOCLIENT_TEXTURES = 0x38, TOCLIENT_MEDIA = 0x38,
/* /*
u16 command u16 command
u16 total number of texture bunches u16 total number of texture bunches
u16 index of this bunch u16 index of this bunch
u32 number of textures in this bunch u32 number of files in this bunch
for each texture { for each file {
u16 length of name u16 length of name
string name string name
u32 length of data u32 length of data
@ -250,11 +250,11 @@ enum ToClientCommand
serialized CraftiItemDefManager serialized CraftiItemDefManager
*/ */
TOCLIENT_ANNOUNCE_TEXTURES = 0x3c, TOCLIENT_ANNOUNCE_MEDIA = 0x3c,
/* /*
u16 command u16 command
u32 number of textures u32 number of files
for each texture { for each texture {
u16 length of name u16 length of name
string name string name
@ -468,11 +468,11 @@ enum ToServerCommand
s32[len] sound_id s32[len] sound_id
*/ */
TOSERVER_REQUEST_TEXTURES = 0x40, TOSERVER_REQUEST_MEDIA = 0x40,
/* /*
u16 command u16 command
u16 number of textures requested u16 number of files requested
for each texture { for each file {
u16 length of name u16 length of name
string name string name
} }

@ -28,14 +28,13 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include <string> #include <string>
#include <iostream> #include <iostream>
bool FileCache::loadByPath(const std::string &name, std::ostream &os, bool FileCache::loadByPath(const std::string &path, std::ostream &os)
const std::string &path)
{ {
std::ifstream fis(path.c_str(), std::ios_base::binary); std::ifstream fis(path.c_str(), std::ios_base::binary);
if(!fis.good()){ if(!fis.good()){
infostream<<"FileCache: File not found in cache: " verbosestream<<"FileCache: File not found in cache: "
<<name << " expected it at: "<<path<<std::endl; <<path<<std::endl;
return false; return false;
} }
@ -53,15 +52,14 @@ bool FileCache::loadByPath(const std::string &name, std::ostream &os,
} }
} }
if(bad){ if(bad){
infostream<<"FileCache: Failed to read file from cache: \"" errorstream<<"FileCache: Failed to read file from cache: \""
<<path<<"\""<<std::endl; <<path<<"\""<<std::endl;
} }
return !bad; return !bad;
} }
bool FileCache::updateByPath(const std::string &name, const std::string &data, bool FileCache::updateByPath(const std::string &path, const std::string &data)
const std::string &path)
{ {
std::ofstream file(path.c_str(), std::ios_base::binary | std::ofstream file(path.c_str(), std::ios_base::binary |
std::ios_base::trunc); std::ios_base::trunc);
@ -69,7 +67,7 @@ bool FileCache::updateByPath(const std::string &name, const std::string &data,
if(!file.good()) if(!file.good())
{ {
errorstream<<"FileCache: Can't write to file at " errorstream<<"FileCache: Can't write to file at "
<<path<<std::endl; <<path<<std::endl;
return false; return false;
} }
@ -82,36 +80,31 @@ bool FileCache::updateByPath(const std::string &name, const std::string &data,
bool FileCache::loadByName(const std::string &name, std::ostream &os) bool FileCache::loadByName(const std::string &name, std::ostream &os)
{ {
std::string path = m_dir + DIR_DELIM + name; std::string path = m_dir + DIR_DELIM + name;
return loadByPath(name, os, path); return loadByPath(path, os);
} }
bool FileCache::updateByName(const std::string &name, const std::string &data) bool FileCache::updateByName(const std::string &name, const std::string &data)
{ {
std::string path = m_dir + DIR_DELIM + name; std::string path = m_dir + DIR_DELIM + name;
return updateByPath(name, data, path); return updateByPath(path, data);
} }
std::string FileCache::getPathFromChecksum(const std::string &name, std::string FileCache::getPathFromChecksum(const std::string &checksum)
const std::string &checksum)
{ {
std::string checksum_hex = hex_encode(checksum.c_str(), checksum.length()); std::string checksum_hex = hex_encode(checksum.c_str(), checksum.length());
size_t dot = name.find_last_of('.');; return m_dir + DIR_DELIM + checksum_hex;
std::string ext = (dot == std::string::npos)? "" :
name.substr(dot, std::string::npos);
return m_dir + DIR_DELIM + checksum_hex + ext;
} }
bool FileCache::loadByChecksum(const std::string &name, std::ostream &os, bool FileCache::loadByChecksum(const std::string &checksum, std::ostream &os)
const std::string &checksum)
{ {
std::string path = getPathFromChecksum(name, checksum); std::string path = getPathFromChecksum(checksum);
return loadByPath(name, os, path); return loadByPath(path, os);
} }
bool FileCache::updateByChecksum(const std::string &name, bool FileCache::updateByChecksum(const std::string &checksum,
const std::string &data, const std::string &checksum) const std::string &data)
{ {
std::string path = getPathFromChecksum(name, checksum); std::string path = getPathFromChecksum(checksum);
return updateByPath(name, data, path); return updateByPath(path, data);
} }

@ -52,28 +52,21 @@ public:
Loads a file based on a check sum, which may be any kind of Loads a file based on a check sum, which may be any kind of
rather unique byte sequence. Returns true, if the file could rather unique byte sequence. Returns true, if the file could
be written into os, false otherwise. be written into os, false otherwise.
A file name is required to give the disk file a name that
has the right file name extension (e.g. ".png").
*/ */
bool loadByChecksum(const std::string &name, std::ostream &os, bool loadByChecksum(const std::string &checksum, std::ostream &os);
const std::string &checksum);
/* /*
Stores a file in the cache based on its checksum. Stores a file in the cache based on its checksum.
Returns true on success, false otherwise. Returns true on success, false otherwise.
*/ */
bool updateByChecksum(const std::string &name, const std::string &data, bool updateByChecksum(const std::string &checksum, const std::string &data);
const std::string &checksum);
private: private:
std::string m_dir; std::string m_dir;
bool loadByPath(const std::string &name, std::ostream &os, bool loadByPath(const std::string &path, std::ostream &os);
const std::string &path); bool updateByPath(const std::string &path, const std::string &data);
bool updateByPath(const std::string &name, const std::string &data, std::string getPathFromChecksum(const std::string &checksum);
const std::string &path);
std::string getPathFromChecksum(const std::string &name,
const std::string &checksum);
}; };
#endif #endif

@ -56,7 +56,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "clientmap.h" #include "clientmap.h"
#include "sky.h" #include "sky.h"
#include "sound.h" #include "sound.h"
#if USE_AUDIO #if USE_SOUND
#include "sound_openal.h" #include "sound_openal.h"
#endif #endif
#include "event_manager.h" #include "event_manager.h"
@ -953,7 +953,7 @@ void the_game(
// Sound manager // Sound manager
ISoundManager *sound = NULL; ISoundManager *sound = NULL;
bool sound_is_dummy = false; bool sound_is_dummy = false;
#if USE_AUDIO #if USE_SOUND
infostream<<"Attempting to use OpenAL audio"<<std::endl; infostream<<"Attempting to use OpenAL audio"<<std::endl;
sound = createOpenALSoundManager(&soundfetcher); sound = createOpenALSoundManager(&soundfetcher);
if(!sound) if(!sound)
@ -1163,9 +1163,8 @@ void the_game(
ss<<L" Item definitions\n"; ss<<L" Item definitions\n";
ss<<(client.nodedefReceived()?L"[X]":L"[ ]"); ss<<(client.nodedefReceived()?L"[X]":L"[ ]");
ss<<L" Node definitions\n"; ss<<L" Node definitions\n";
//ss<<(client.texturesReceived()?L"[X]":L"[ ]"); ss<<L"["<<(int)(client.mediaReceiveProgress()*100+0.5)<<L"%] ";
ss<<L"["<<(int)(client.textureReceiveProgress()*100+0.5)<<L"%] "; ss<<L" Media\n";
ss<<L" Textures\n";
draw_load_screen(ss.str(), driver, font); draw_load_screen(ss.str(), driver, font);

@ -957,7 +957,7 @@ Server::Server(
} }
// Read Textures and calculate sha1 sums // Read Textures and calculate sha1 sums
PrepareTextures(); fillMediaCache();
// Apply item aliases in the node definition manager // Apply item aliases in the node definition manager
m_nodedef->updateAliases(m_itemdef); m_nodedef->updateAliases(m_itemdef);
@ -2183,7 +2183,7 @@ void Server::ProcessData(u8 *data, u32 datasize, u16 peer_id)
SendNodeDef(m_con, peer_id, m_nodedef); SendNodeDef(m_con, peer_id, m_nodedef);
// Send texture announcement // Send texture announcement
SendTextureAnnouncement(peer_id); sendMediaAnnouncement(peer_id);
// Send player info to all players // Send player info to all players
//SendPlayerInfos(); //SendPlayerInfos();
@ -2842,29 +2842,28 @@ void Server::ProcessData(u8 *data, u32 datasize, u16 peer_id)
// ActiveObject is added to environment in AsyncRunStep after // ActiveObject is added to environment in AsyncRunStep after
// the previous addition has been succesfully removed // the previous addition has been succesfully removed
} }
else if(command == TOSERVER_REQUEST_TEXTURES) { else if(command == TOSERVER_REQUEST_MEDIA) {
std::string datastring((char*)&data[2], datasize-2); std::string datastring((char*)&data[2], datasize-2);
std::istringstream is(datastring, std::ios_base::binary); std::istringstream is(datastring, std::ios_base::binary);
core::list<MediaRequest> tosend;
core::list<TextureRequest> tosend; u16 numfiles = readU16(is);
u16 numtextures = readU16(is);
infostream<<"Sending "<<numtextures<<" textures to " infostream<<"Sending "<<numfiles<<" files to "
<<getPlayerName(peer_id)<<std::endl; <<getPlayerName(peer_id)<<std::endl;
verbosestream<<"TOSERVER_REQUEST_TEXTURES: "<<std::endl; verbosestream<<"TOSERVER_REQUEST_MEDIA: "<<std::endl;
for(int i = 0; i < numtextures; i++) { for(int i = 0; i < numfiles; i++) {
std::string name = deSerializeString(is); std::string name = deSerializeString(is);
tosend.push_back(TextureRequest(name)); tosend.push_back(MediaRequest(name));
verbosestream<<"TOSERVER_REQUEST_TEXTURES: requested texture " verbosestream<<"TOSERVER_REQUEST_MEDIA: requested file "
<<name<<std::endl; <<name<<std::endl;
} }
SendTexturesRequested(peer_id, tosend); sendRequestedMedia(peer_id, tosend);
// Now the client should know about everything // Now the client should know about everything
// (definitions and textures) // (definitions and files)
getClient(peer_id)->definitions_sent = true; getClient(peer_id)->definitions_sent = true;
} }
else if(command == TOSERVER_INTERACT) else if(command == TOSERVER_INTERACT)
@ -3928,32 +3927,32 @@ void Server::SendBlocks(float dtime)
} }
} }
void Server::PrepareTextures() void Server::fillMediaCache()
{ {
DSTACK(__FUNCTION_NAME); DSTACK(__FUNCTION_NAME);
infostream<<"Server: Calculating texture checksums"<<std::endl; infostream<<"Server: Calculating file checksums"<<std::endl;
for(core::list<ModSpec>::Iterator i = m_mods.begin(); for(core::list<ModSpec>::Iterator i = m_mods.begin();
i != m_mods.end(); i++){ i != m_mods.end(); i++){
const ModSpec &mod = *i; const ModSpec &mod = *i;
std::string texturepath = mod.path + DIR_DELIM + "textures"; std::string filepath = mod.path + DIR_DELIM + "textures";
std::vector<fs::DirListNode> dirlist = fs::GetDirListing(texturepath); std::vector<fs::DirListNode> dirlist = fs::GetDirListing(filepath);
for(u32 j=0; j<dirlist.size(); j++){ for(u32 j=0; j<dirlist.size(); j++){
if(dirlist[j].dir) // Ignode dirs if(dirlist[j].dir) // Ignode dirs
continue; continue;
std::string tname = dirlist[j].name; std::string tname = dirlist[j].name;
// if name contains illegal characters, ignore the texture // if name contains illegal characters, ignore the file
if(!string_allowed(tname, TEXTURENAME_ALLOWED_CHARS)){ if(!string_allowed(tname, TEXTURENAME_ALLOWED_CHARS)){
errorstream<<"Server: ignoring illegal texture name: \"" errorstream<<"Server: ignoring illegal file name: \""
<<tname<<"\""<<std::endl; <<tname<<"\""<<std::endl;
continue; continue;
} }
std::string tpath = texturepath + DIR_DELIM + tname; std::string tpath = filepath + DIR_DELIM + tname;
// Read data // Read data
std::ifstream fis(tpath.c_str(), std::ios_base::binary); std::ifstream fis(tpath.c_str(), std::ios_base::binary);
if(fis.good() == false){ if(fis.good() == false){
errorstream<<"Server::PrepareTextures(): Could not open \"" errorstream<<"Server::fillMediaCache(): Could not open \""
<<tname<<"\" for reading"<<std::endl; <<tname<<"\" for reading"<<std::endl;
continue; continue;
} }
@ -3972,12 +3971,12 @@ void Server::PrepareTextures()
} }
} }
if(bad){ if(bad){
errorstream<<"Server::PrepareTextures(): Failed to read \"" errorstream<<"Server::fillMediaCache(): Failed to read \""
<<tname<<"\""<<std::endl; <<tname<<"\""<<std::endl;
continue; continue;
} }
if(tmp_os.str().length() == 0){ if(tmp_os.str().length() == 0){
errorstream<<"Server::PrepareTextures(): Empty file \"" errorstream<<"Server::fillMediaCache(): Empty file \""
<<tpath<<"\""<<std::endl; <<tpath<<"\""<<std::endl;
continue; continue;
} }
@ -3991,60 +3990,60 @@ void Server::PrepareTextures()
free(digest); free(digest);
// Put in list // Put in list
this->m_Textures[tname] = TextureInformation(tpath,digest_string); this->m_media[tname] = MediaInfo(tpath,digest_string);
verbosestream<<"Server: sha1 for "<<tname<<"\tis "<<std::endl; verbosestream<<"Server: sha1 for "<<tname<<"\tis "<<std::endl;
} }
} }
} }
struct SendableTextureAnnouncement struct SendableMediaAnnouncement
{ {
std::string name; std::string name;
std::string sha1_digest; std::string sha1_digest;
SendableTextureAnnouncement(const std::string name_="", SendableMediaAnnouncement(const std::string name_="",
const std::string sha1_digest_=""): const std::string sha1_digest_=""):
name(name_), name(name_),
sha1_digest(sha1_digest_) sha1_digest(sha1_digest_)
{ {}
} };
};
void Server::SendTextureAnnouncement(u16 peer_id){ void Server::sendMediaAnnouncement(u16 peer_id)
{
DSTACK(__FUNCTION_NAME); DSTACK(__FUNCTION_NAME);
verbosestream<<"Server: Announcing textures to id("<<peer_id<<")" verbosestream<<"Server: Announcing files to id("<<peer_id<<")"
<<std::endl; <<std::endl;
core::list<SendableTextureAnnouncement> texture_announcements; core::list<SendableMediaAnnouncement> file_announcements;
for (std::map<std::string,TextureInformation>::iterator i = m_Textures.begin();i != m_Textures.end(); i++ ) {
for(std::map<std::string, MediaInfo>::iterator i = m_media.begin();
i != m_media.end(); i++){
// Put in list // Put in list
texture_announcements.push_back( file_announcements.push_back(
SendableTextureAnnouncement(i->first, i->second.sha1_digest)); SendableMediaAnnouncement(i->first, i->second.sha1_digest));
} }
//send announcements // Make packet
std::ostringstream os(std::ios_base::binary);
/* /*
u16 command u16 command
u32 number of textures u32 number of files
for each texture { for each texture {
u16 length of name u16 length of name
string name string name
u16 length of digest string u16 length of sha1_digest
string sha1_digest string sha1_digest
} }
*/ */
std::ostringstream os(std::ios_base::binary);
writeU16(os, TOCLIENT_ANNOUNCE_MEDIA);
writeU16(os, file_announcements.size());
writeU16(os, TOCLIENT_ANNOUNCE_TEXTURES); for(core::list<SendableMediaAnnouncement>::Iterator
writeU16(os, texture_announcements.size()); j = file_announcements.begin();
j != file_announcements.end(); j++){
for(core::list<SendableTextureAnnouncement>::Iterator
j = texture_announcements.begin();
j != texture_announcements.end(); j++){
os<<serializeString(j->name); os<<serializeString(j->name);
os<<serializeString(j->sha1_digest); os<<serializeString(j->sha1_digest);
} }
@ -4058,13 +4057,13 @@ void Server::SendTextureAnnouncement(u16 peer_id){
} }
struct SendableTexture struct SendableMedia
{ {
std::string name; std::string name;
std::string path; std::string path;
std::string data; std::string data;
SendableTexture(const std::string &name_="", const std::string path_="", SendableMedia(const std::string &name_="", const std::string path_="",
const std::string &data_=""): const std::string &data_=""):
name(name_), name(name_),
path(path_), path(path_),
@ -4072,36 +4071,40 @@ struct SendableTexture
{} {}
}; };
void Server::SendTexturesRequested(u16 peer_id,core::list<TextureRequest> tosend) { void Server::sendRequestedMedia(u16 peer_id,
const core::list<MediaRequest> &tosend)
{
DSTACK(__FUNCTION_NAME); DSTACK(__FUNCTION_NAME);
verbosestream<<"Server::SendTexturesRequested(): " verbosestream<<"Server::sendRequestedMedia(): "
<<"Sending textures to client"<<std::endl; <<"Sending files to client"<<std::endl;
/* Read textures */ /* Read files */
// Put 5kB in one bunch (this is not accurate) // Put 5kB in one bunch (this is not accurate)
u32 bytes_per_bunch = 5000; u32 bytes_per_bunch = 5000;
core::array< core::list<SendableTexture> > texture_bunches; core::array< core::list<SendableMedia> > file_bunches;
texture_bunches.push_back(core::list<SendableTexture>()); file_bunches.push_back(core::list<SendableMedia>());
u32 texture_size_bunch_total = 0; u32 file_size_bunch_total = 0;
for(core::list<TextureRequest>::Iterator i = tosend.begin(); i != tosend.end(); i++) { for(core::list<MediaRequest>::ConstIterator i = tosend.begin();
if(m_Textures.find(i->name) == m_Textures.end()){ i != tosend.end(); i++)
errorstream<<"Server::SendTexturesRequested(): Client asked for " {
<<"unknown texture \""<<(i->name)<<"\""<<std::endl; if(m_media.find(i->name) == m_media.end()){
errorstream<<"Server::sendRequestedMedia(): Client asked for "
<<"unknown file \""<<(i->name)<<"\""<<std::endl;
continue; continue;
} }
//TODO get path + name //TODO get path + name
std::string tpath = m_Textures[(*i).name].path; std::string tpath = m_media[(*i).name].path;
// Read data // Read data
std::ifstream fis(tpath.c_str(), std::ios_base::binary); std::ifstream fis(tpath.c_str(), std::ios_base::binary);
if(fis.good() == false){ if(fis.good() == false){
errorstream<<"Server::SendTexturesRequested(): Could not open \"" errorstream<<"Server::sendRequestedMedia(): Could not open \""
<<tpath<<"\" for reading"<<std::endl; <<tpath<<"\" for reading"<<std::endl;
continue; continue;
} }
@ -4112,7 +4115,7 @@ void Server::SendTexturesRequested(u16 peer_id,core::list<TextureRequest> tosend
fis.read(buf, 1024); fis.read(buf, 1024);
std::streamsize len = fis.gcount(); std::streamsize len = fis.gcount();
tmp_os.write(buf, len); tmp_os.write(buf, len);
texture_size_bunch_total += len; file_size_bunch_total += len;
if(fis.eof()) if(fis.eof())
break; break;
if(!fis.good()){ if(!fis.good()){
@ -4121,67 +4124,66 @@ void Server::SendTexturesRequested(u16 peer_id,core::list<TextureRequest> tosend
} }
} }
if(bad){ if(bad){
errorstream<<"Server::SendTexturesRequested(): Failed to read \"" errorstream<<"Server::sendRequestedMedia(): Failed to read \""
<<(*i).name<<"\""<<std::endl; <<(*i).name<<"\""<<std::endl;
continue; continue;
} }
/*infostream<<"Server::SendTexturesRequested(): Loaded \"" /*infostream<<"Server::sendRequestedMedia(): Loaded \""
<<tname<<"\""<<std::endl;*/ <<tname<<"\""<<std::endl;*/
// Put in list // Put in list
texture_bunches[texture_bunches.size()-1].push_back( file_bunches[file_bunches.size()-1].push_back(
SendableTexture((*i).name, tpath, tmp_os.str())); SendableMedia((*i).name, tpath, tmp_os.str()));
// Start next bunch if got enough data // Start next bunch if got enough data
if(texture_size_bunch_total >= bytes_per_bunch){ if(file_size_bunch_total >= bytes_per_bunch){
texture_bunches.push_back(core::list<SendableTexture>()); file_bunches.push_back(core::list<SendableMedia>());
texture_size_bunch_total = 0; file_size_bunch_total = 0;
} }
} }
/* Create and send packets */ /* Create and send packets */
u32 num_bunches = texture_bunches.size(); u32 num_bunches = file_bunches.size();
for(u32 i=0; i<num_bunches; i++) for(u32 i=0; i<num_bunches; i++)
{ {
/* std::ostringstream os(std::ios_base::binary);
u16 command
u16 total number of texture bunches
u16 index of this bunch
u32 number of textures in this bunch
for each texture {
u16 length of name
string name
u32 length of data
data
}
*/
std::ostringstream os(std::ios_base::binary);
writeU16(os, TOCLIENT_TEXTURES); /*
writeU16(os, num_bunches); u16 command
writeU16(os, i); u16 total number of texture bunches
writeU32(os, texture_bunches[i].size()); u16 index of this bunch
u32 number of files in this bunch
for(core::list<SendableTexture>::Iterator for each file {
j = texture_bunches[i].begin(); u16 length of name
j != texture_bunches[i].end(); j++){ string name
os<<serializeString(j->name); u32 length of data
os<<serializeLongString(j->data); data
} }
*/
// Make data buffer writeU16(os, TOCLIENT_MEDIA);
std::string s = os.str(); writeU16(os, num_bunches);
verbosestream<<"Server::SendTexturesRequested(): bunch " writeU16(os, i);
<<i<<"/"<<num_bunches writeU32(os, file_bunches[i].size());
<<" textures="<<texture_bunches[i].size()
<<" size=" <<s.size()<<std::endl; for(core::list<SendableMedia>::Iterator
SharedBuffer<u8> data((u8*)s.c_str(), s.size()); j = file_bunches[i].begin();
// Send as reliable j != file_bunches[i].end(); j++){
m_con.Send(peer_id, 0, data, true); os<<serializeString(j->name);
os<<serializeLongString(j->data);
} }
// Make data buffer
std::string s = os.str();
verbosestream<<"Server::sendRequestedMedia(): bunch "
<<i<<"/"<<num_bunches
<<" files="<<file_bunches[i].size()
<<" size=" <<s.size()<<std::endl;
SharedBuffer<u8> data((u8*)s.c_str(), s.size());
// Send as reliable
m_con.Send(peer_id, 0, data, true);
}
} }
/* /*

@ -253,21 +253,21 @@ struct PrioritySortedBlockTransfer
u16 peer_id; u16 peer_id;
}; };
struct TextureRequest struct MediaRequest
{ {
std::string name; std::string name;
TextureRequest(const std::string &name_=""): MediaRequest(const std::string &name_=""):
name(name_) name(name_)
{} {}
}; };
struct TextureInformation struct MediaInfo
{ {
std::string path; std::string path;
std::string sha1_digest; std::string sha1_digest;
TextureInformation(const std::string path_="", MediaInfo(const std::string path_="",
const std::string sha1_digest_=""): const std::string sha1_digest_=""):
path(path_), path(path_),
sha1_digest(sha1_digest_) sha1_digest(sha1_digest_)
@ -644,11 +644,10 @@ private:
// Sends blocks to clients (locks env and con on its own) // Sends blocks to clients (locks env and con on its own)
void SendBlocks(float dtime); void SendBlocks(float dtime);
void PrepareTextures(); void fillMediaCache();
void sendMediaAnnouncement(u16 peer_id);
void SendTextureAnnouncement(u16 peer_id); void sendRequestedMedia(u16 peer_id,
const core::list<MediaRequest> &tosend);
void SendTexturesRequested(u16 peer_id,core::list<TextureRequest> tosend);
/* /*
Something random Something random
@ -832,7 +831,7 @@ private:
friend class EmergeThread; friend class EmergeThread;
friend class RemoteClient; friend class RemoteClient;
std::map<std::string,TextureInformation> m_Textures; std::map<std::string,MediaInfo> m_media;
/* /*
Sounds Sounds