ShaderSource and silly example shaders

client/shaders/the_darkness_of_light/opengl_fragment.asm

@@ -0,0 +1,17 @@
+!!ARBfp1.0
+
+#Input
+ATTRIB inTexCoord = fragment.texcoord;      # texture coordinates
+ATTRIB inColor = fragment.color.primary; # interpolated diffuse color
+
+#Output
+OUTPUT outColor = result.color;
+
+TEMP texelColor;
+TXP texelColor, inTexCoord, texture, 2D; 
+MUL texelColor, texelColor, inColor;  # multiply with color   
+SUB outColor, {1.0,1.0,1.0,1.0}, texelColor;
+MOV outColor.w, 1.0;
+
+END
+

client/shaders/the_darkness_of_light/opengl_fragment.glsl

@@ -0,0 +1,9 @@
+
+uniform sampler2D myTexture;
+
+void main (void)
+{
+    vec4 col = texture2D(myTexture, vec2(gl_TexCoord[0]));
+    col *= gl_Color;
+    gl_FragColor = vec4(1.0-col.r, 1.0-col.g, 1.0-col.b, 1.0);
+}

client/shaders/the_darkness_of_light/opengl_vertex.asm

@@ -0,0 +1,38 @@
+!!ARBvp1.0
+
+#input
+ATTRIB InPos = vertex.position;
+ATTRIB InColor = vertex.color;
+ATTRIB InNormal = vertex.normal;
+ATTRIB InTexCoord = vertex.texcoord;
+
+#output
+OUTPUT OutPos = result.position;
+OUTPUT OutColor = result.color;
+OUTPUT OutTexCoord = result.texcoord;
+
+PARAM MVP[4] = { state.matrix.mvp }; # modelViewProjection matrix.
+TEMP Temp;
+TEMP TempColor;
+TEMP TempCompare;
+
+#transform position to clip space 
+DP4 Temp.x, MVP[0], InPos;
+DP4 Temp.y, MVP[1], InPos;
+DP4 Temp.z, MVP[2], InPos;
+DP4 Temp.w, MVP[3], InPos;
+
+# check if normal.y > 0.5
+SLT TempCompare, InNormal, {0.5,0.5,0.5,0.5};
+MUL TempCompare.z, TempCompare.y, 0.5;
+SUB TempCompare.x, 1.0, TempCompare.z;
+MOV TempCompare.y, TempCompare.x;
+MOV TempCompare.z, TempCompare.x;
+
+# calculate light color
+MUL OutColor, InColor, TempCompare;
+MOV OutColor.w, 1.0;          # we want alpha to be always 1
+MOV OutTexCoord, InTexCoord; # store texture coordinate
+MOV OutPos, Temp;
+
+END

client/shaders/the_darkness_of_light/opengl_vertex.glsl

@@ -0,0 +1,16 @@
+
+uniform mat4 mWorldViewProj;
+uniform mat4 mInvWorld;
+uniform mat4 mTransWorld;
+
+void main(void)
+{
+	gl_Position = mWorldViewProj * gl_Vertex;
+	
+	if(gl_Normal.y > 0.5)
+		gl_FrontColor = gl_BackColor = gl_Color;
+	else
+		gl_FrontColor = gl_BackColor = gl_Color * 0.5;
+
+	gl_TexCoord[0] = gl_MultiTexCoord0;
+}

src/CMakeLists.txt

@@ -262,6 +262,7 @@ set(minetest_SRCS
 	client.cpp
 	filecache.cpp
 	tile.cpp
+	shader.cpp
 	game.cpp
 	main.cpp
 )

src/client.cpp

@@ -33,6 +33,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "nodemetadata.h"
 #include "nodedef.h"
 #include "itemdef.h"
+#include "shader.h"
 #include <IFileSystem.h>
 #include "sha1.h"
 #include "base64.h"
@@ -228,12 +229,14 @@ Client::Client(
 		std::string password,
 		MapDrawControl &control,
 		IWritableTextureSource *tsrc,
+		IWritableShaderSource *shsrc,
 		IWritableItemDefManager *itemdef,
 		IWritableNodeDefManager *nodedef,
 		ISoundManager *sound,
 		MtEventManager *event
 ):
 	m_tsrc(tsrc),
+	m_shsrc(shsrc),
 	m_itemdef(itemdef),
 	m_nodedef(nodedef),
 	m_sound(sound),
@@ -2456,6 +2459,9 @@ void Client::afterContentReceived()
 	if(g_settings->getBool("enable_texture_atlas"))
 		m_tsrc->buildMainAtlas(this);
 
+	// Rebuild shaders
+	m_shsrc->rebuildShaders();
+
 	// Update node aliases
 	infostream<<"- Updating node aliases"<<std::endl;
 	m_nodedef->updateAliases(m_itemdef);
@@ -2512,6 +2518,10 @@ ITextureSource* Client::getTextureSource()
 {
 	return m_tsrc;
 }
+IShaderSource* Client::getShaderSource()
+{
+	return m_shsrc;
+}
 u16 Client::allocateUnknownNodeId(const std::string &name)
 {
 	errorstream<<"Client::allocateUnknownNodeId(): "

src/client.h

@@ -39,6 +39,7 @@ struct MeshMakeData;
 class MapBlockMesh;
 class IGameDef;
 class IWritableTextureSource;
+class IWritableShaderSource;
 class IWritableItemDefManager;
 class IWritableNodeDefManager;
 //class IWritableCraftDefManager;
@@ -174,6 +175,7 @@ public:
 			std::string password,
 			MapDrawControl &control,
 			IWritableTextureSource *tsrc,
+			IWritableShaderSource *shsrc,
 			IWritableItemDefManager *itemdef,
 			IWritableNodeDefManager *nodedef,
 			ISoundManager *sound,
@@ -305,6 +307,7 @@ public:
 	virtual INodeDefManager* getNodeDefManager();
 	virtual ICraftDefManager* getCraftDefManager();
 	virtual ITextureSource* getTextureSource();
+	virtual IShaderSource* getShaderSource();
 	virtual u16 allocateUnknownNodeId(const std::string &name);
 	virtual ISoundManager* getSoundManager();
 	virtual MtEventManager* getEventManager();
@@ -337,6 +340,7 @@ private:
 	IntervalLimiter m_map_timer_and_unload_interval;
 
 	IWritableTextureSource *m_tsrc;
+	IWritableShaderSource *m_shsrc;
 	IWritableItemDefManager *m_itemdef;
 	IWritableNodeDefManager *m_nodedef;
 	ISoundManager *m_sound;

src/defaultsettings.cpp

@@ -92,6 +92,7 @@ void set_default_settings(Settings *settings)
 	settings->setDefault("smooth_lighting", "true");
 	settings->setDefault("enable_texture_atlas", "true");
 	settings->setDefault("texture_path", "");
+	settings->setDefault("shader_path", "");
 	settings->setDefault("video_driver", "opengl");
 	settings->setDefault("free_move", "false");
 	settings->setDefault("continuous_forward", "false");
@@ -103,6 +104,7 @@ void set_default_settings(Settings *settings)
 	settings->setDefault("view_bobbing_amount", "1.0");
 	settings->setDefault("enable_3d_clouds", "true");
 	settings->setDefault("opaque_water", "false");
+	settings->setDefault("enable_shaders", "2");
 	settings->setDefault("console_color", "(0,0,0)");
 	settings->setDefault("console_alpha", "200");
 	settings->setDefault("enable_sound", "true");

src/game.cpp

@@ -50,6 +50,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "main.h" // For g_settings
 #include "itemdef.h"
 #include "tile.h" // For TextureSource
+#include "shader.h" // For ShaderSource
 #include "logoutputbuffer.h"
 #include "subgame.h"
 #include "quicktune_shortcutter.h"
@@ -876,6 +877,9 @@ void the_game(
 	// Create texture source
 	IWritableTextureSource *tsrc = createTextureSource(device);
 	
+	// Create shader source
+	IWritableShaderSource *shsrc = createShaderSource(device);
+	
 	// These will be filled by data received from the server
 	// Create item definition manager
 	IWritableItemDefManager *itemdef = createItemDefManager();
@@ -943,7 +947,7 @@ void the_game(
 	MapDrawControl draw_control;
 
 	Client client(device, playername.c_str(), password, draw_control,
-			tsrc, itemdef, nodedef, sound, &eventmgr);
+			tsrc, shsrc, itemdef, nodedef, sound, &eventmgr);
 	
 	// Client acts as our GameDef
 	IGameDef *gamedef = &client;
@@ -1422,6 +1426,11 @@ void the_game(
 		/* Process ItemDefManager's queue */
 		itemdef->processQueue(gamedef);
 
+		/*
+			Process ShaderSource's queue
+		*/
+		shsrc->processQueue();
+
 		/*
 			Random calculations
 		*/
@@ -3002,9 +3011,11 @@ void the_game(
 	
 	if(!sound_is_dummy)
 		delete sound;
+
+	delete tsrc;
+	delete shsrc;
 	delete nodedef;
 	delete itemdef;
-	delete tsrc;
 }
 
 

src/gamedef.h

@@ -28,6 +28,7 @@ class INodeDefManager;
 class ICraftDefManager;
 class ITextureSource;
 class ISoundManager;
+class IShaderSource;
 class MtEventManager;
 class IRollbackReportSink;
 
@@ -49,6 +50,8 @@ public:
 	// pointers in other threads than main thread will make things explode.
 	virtual ITextureSource* getTextureSource()=0;
 
+	virtual IShaderSource* getShaderSource()=0;
+	
 	// Used for keeping track of names/ids of unknown nodes
 	virtual u16 allocateUnknownNodeId(const std::string &name)=0;
 	
@@ -70,6 +73,7 @@ public:
 	ICraftDefManager* cdef(){return getCraftDefManager();}
 	ITextureSource* tsrc(){return getTextureSource();}
 	ISoundManager* sound(){return getSoundManager();}
+	IShaderSource* shsrc(){return getShaderSource();}
 	MtEventManager* event(){return getEventManager();}
 	IRollbackReportSink* rollback(){return getRollbackReportSink();}
 };

src/mapblock_mesh.cpp

@@ -28,6 +28,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "mesh.h"
 #include "content_mapblock.h"
 #include "noise.h"
+#include "shader.h"
 #include "settings.h"
 #include "util/directiontables.h"
 
@@ -1011,6 +1012,8 @@ MapBlockMesh::MapBlockMesh(MeshMakeData *data):
 		Convert MeshCollector to SMesh
 		Also store animation info
 	*/
+	video::E_MATERIAL_TYPE shadermat = m_gamedef->getShaderSource()->
+			getShader("the_darkness_of_light").material;
 	for(u32 i = 0; i < collector.prebuffers.size(); i++)
 	{
 		PreMeshBuffer &p = collector.prebuffers[i];
@@ -1077,6 +1080,9 @@ MapBlockMesh::MapBlockMesh(MeshMakeData *data):
 		material.setTexture(0, p.tile.texture.atlas);
 		p.tile.applyMaterialOptions(material);
 
+		//if(material.MaterialType == video::EMT_TRANSPARENT_ALPHA_CHANNEL_REF)
+			material.MaterialType = shadermat;
+
 		// Create meshbuffer
 
 		// This is a "Standard MeshBuffer",

src/server.cpp

@@ -4688,6 +4688,10 @@ ITextureSource* Server::getTextureSource()
 {
 	return NULL;
 }
+IShaderSource* Server::getShaderSource()
+{
+	return NULL;
+}
 u16 Server::allocateUnknownNodeId(const std::string &name)
 {
 	return m_nodedef->allocateDummy(name);

src/server.h

@@ -560,6 +560,7 @@ public:
 	virtual INodeDefManager* getNodeDefManager();
 	virtual ICraftDefManager* getCraftDefManager();
 	virtual ITextureSource* getTextureSource();
+	virtual IShaderSource* getShaderSource();
 	virtual u16 allocateUnknownNodeId(const std::string &name);
 	virtual ISoundManager* getSoundManager();
 	virtual MtEventManager* getEventManager();

src/shader.cpp

@@ -0,0 +1,731 @@
+/*
+Minetest-c55
+Copyright (C) 2012 celeron55, Perttu Ahola <celeron55@gmail.com>
+Copyright (C) 2012 Kahrl <kahrl@gmx.net>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#include "shader.h"
+#include "irrlichttypes_extrabloated.h"
+#include "debug.h"
+#include "main.h" // for g_settings
+#include "filesys.h"
+#include "util/container.h"
+#include "util/thread.h"
+#include "settings.h"
+#include <iterator>
+#include <ICameraSceneNode.h>
+#include <IGPUProgrammingServices.h>
+#include <IMaterialRenderer.h>
+#include <IMaterialRendererServices.h>
+#include <IShaderConstantSetCallBack.h>
+#include "EShaderTypes.h"
+#include "log.h"
+#include "gamedef.h"
+
+/*
+	A cache from shader name to shader path
+*/
+MutexedMap<std::string, std::string> g_shadername_to_path_cache;
+
+/*
+	Gets the path to a shader by first checking if the file
+	  name_of_shader/filename
+	exists in shader_path and if not, using the data path.
+
+	If not found, returns "".
+
+	Utilizes a thread-safe cache.
+*/
+std::string getShaderPath(const std::string &name_of_shader,
+		const std::string &filename)
+{
+	std::string combined = name_of_shader + DIR_DELIM + filename;
+	std::string fullpath = "";
+	/*
+		Check from cache
+	*/
+	bool incache = g_shadername_to_path_cache.get(combined, &fullpath);
+	if(incache)
+		return fullpath;
+
+	/*
+		Check from shader_path
+	*/
+	std::string shader_path = g_settings->get("shader_path");
+	if(shader_path != "")
+	{
+		std::string testpath = shader_path + DIR_DELIM + combined;
+		if(fs::PathExists(testpath))
+			fullpath = testpath;
+	}
+
+	/*
+		Check from default data directory
+	*/
+	if(fullpath == "")
+	{
+		std::string rel_path = std::string("client") + DIR_DELIM
+				+ "shaders" + DIR_DELIM
+				+ name_of_shader + DIR_DELIM
+				+ filename;
+		std::string testpath = porting::path_share + DIR_DELIM + rel_path;
+		if(fs::PathExists(testpath))
+			fullpath = testpath;
+	}
+
+	// Add to cache (also an empty result is cached)
+	g_shadername_to_path_cache.set(combined, fullpath);
+
+	// Finally return it
+	return fullpath;
+}
+
+/*
+	SourceShaderCache: A cache used for storing source shaders.
+*/
+
+class SourceShaderCache
+{
+public:
+	void insert(const std::string &name_of_shader,
+			const std::string &filename,
+			const std::string &program,
+			bool prefer_local)
+	{
+		std::string combined = name_of_shader + DIR_DELIM + filename;
+		// Try to use local shader instead if asked to
+		if(prefer_local){
+			std::string path = getShaderPath(name_of_shader, filename);
+			if(path != ""){
+				std::string p = readFile(path);
+				if(p != ""){
+					m_programs[combined] = p;
+					return;
+				}
+			}
+		}
+		m_programs[combined] = program;
+	}
+	std::string get(const std::string &name_of_shader,
+			const std::string &filename)
+	{
+		std::string combined = name_of_shader + DIR_DELIM + filename;
+		core::map<std::string, std::string>::Node *n;
+		n = m_programs.find(combined);
+		if(n)
+			return n->getValue();
+		return "";
+	}
+	// Primarily fetches from cache, secondarily tries to read from filesystem
+	std::string getOrLoad(const std::string &name_of_shader,
+			const std::string &filename)
+	{
+		std::string combined = name_of_shader + DIR_DELIM + filename;
+		core::map<std::string, std::string>::Node *n;
+		n = m_programs.find(combined);
+		if(n)
+			return n->getValue();
+		std::string path = getShaderPath(name_of_shader, filename);
+		if(path == ""){
+			infostream<<"SourceShaderCache::getOrLoad(): No path found for \""
+					<<combined<<"\""<<std::endl;
+			return "";
+		}
+		infostream<<"SourceShaderCache::getOrLoad(): Loading path \""<<path
+				<<"\""<<std::endl;
+		std::string p = readFile(path);
+		if(p != ""){
+			m_programs[combined] = p;
+			return p;
+		}
+		return "";
+	}
+private:
+	core::map<std::string, std::string> m_programs;
+	std::string readFile(const std::string &path)
+	{
+		std::ifstream is(path.c_str(), std::ios::binary);
+		if(!is.is_open())
+			return "";
+		std::ostringstream tmp_os;
+		tmp_os << is.rdbuf();
+		return tmp_os.str();
+	}
+};
+
+/*
+	ShaderCallback: Sets constants that can be used in shaders
+*/
+
+class ShaderCallback : public video::IShaderConstantSetCallBack
+{
+public:
+	ShaderCallback(IrrlichtDevice *device): m_device(device) {}
+	~ShaderCallback() {}
+
+	virtual void OnSetConstants(video::IMaterialRendererServices *services, s32 userData)
+	{
+		video::IVideoDriver *driver = services->getVideoDriver();
+		assert(driver);
+
+		bool is_highlevel = userData;
+
+		// set inverted world matrix
+		core::matrix4 invWorld = driver->getTransform(video::ETS_WORLD);
+		invWorld.makeInverse();
+		if(is_highlevel)
+			services->setVertexShaderConstant("mInvWorld", invWorld.pointer(), 16);
+		else
+			services->setVertexShaderConstant(invWorld.pointer(), 0, 4);
+
+		// set clip matrix
+		core::matrix4 worldViewProj;
+		worldViewProj = driver->getTransform(video::ETS_PROJECTION);
+		worldViewProj *= driver->getTransform(video::ETS_VIEW);
+		worldViewProj *= driver->getTransform(video::ETS_WORLD);
+		if(is_highlevel)
+			services->setVertexShaderConstant("mWorldViewProj", worldViewProj.pointer(), 16);
+		else
+			services->setVertexShaderConstant(worldViewProj.pointer(), 4, 4);
+
+		// set transposed world matrix
+		core::matrix4 world = driver->getTransform(video::ETS_WORLD);
+		world = world.getTransposed();
+		if(is_highlevel)
+			services->setVertexShaderConstant("mTransWorld", world.pointer(), 16);
+		else
+			services->setVertexShaderConstant(world.pointer(), 8, 4);
+	}
+
+private:
+	IrrlichtDevice *m_device;
+};
+
+/*
+	ShaderSource
+*/
+
+class ShaderSource : public IWritableShaderSource
+{
+public:
+	ShaderSource(IrrlichtDevice *device);
+	~ShaderSource();
+
+	/*
+		Gets a shader material id from cache or
+		- if main thread, from getShaderIdDirect
+		- if other thread, adds to request queue and waits for main thread
+	*/
+	u32 getShaderId(const std::string &name);
+
+	/*
+		- If shader material specified by name is found from cache,
+		  return the cached id.
+		- Otherwise generate the shader material, add to cache and return id.
+
+		The id 0 points to a null shader. Its material is EMT_SOLID.
+	*/
+	u32 getShaderIdDirect(const std::string &name);
+
+	// Finds out the name of a cached shader.
+	std::string getShaderName(u32 id);
+
+	/*
+		If shader specified by the name pointed by the id doesn't
+		exist, create it, then return the cached shader.
+
+		Can be called from any thread. If called from some other thread
+		and not found in cache, the call is queued to the main thread
+		for processing.
+	*/
+	ShaderInfo getShader(u32 id);
+
+	ShaderInfo getShader(const std::string &name)
+	{
+		return getShader(getShaderId(name));
+	}
+
+	// Processes queued shader requests from other threads.
+	// Shall be called from the main thread.
+	void processQueue();
+
+	// Insert a shader program into the cache without touching the
+	// filesystem. Shall be called from the main thread.
+	void insertSourceShader(const std::string &name_of_shader,
+		const std::string &filename, const std::string &program);
+
+	// Rebuild shaders from the current set of source shaders
+	// Shall be called from the main thread.
+	void rebuildShaders();
+
+private:
+
+	// The id of the thread that is allowed to use irrlicht directly
+	threadid_t m_main_thread;
+	// The irrlicht device
+	IrrlichtDevice *m_device;
+	// The set-constants callback
+	ShaderCallback *m_shader_callback;
+
+	// Cache of source shaders
+	// This should be only accessed from the main thread
+	SourceShaderCache m_sourcecache;
+
+	// A shader id is index in this array.
+	// The first position contains a dummy shader.
+	core::array<ShaderInfo> m_shaderinfo_cache;
+	// Maps a shader name to an index in the former.
+	core::map<std::string, u32> m_name_to_id;
+	// The two former containers are behind this mutex
+	JMutex m_shaderinfo_cache_mutex;
+
+	// Queued shader fetches (to be processed by the main thread)
+	RequestQueue<std::string, u32, u8, u8> m_get_shader_queue;
+};
+
+IWritableShaderSource* createShaderSource(IrrlichtDevice *device)
+{
+	return new ShaderSource(device);
+}
+
+/*
+	Generate shader given the shader name.
+*/
+ShaderInfo generate_shader(std::string name, IrrlichtDevice *device,
+		video::IShaderConstantSetCallBack *callback,
+		SourceShaderCache *sourcecache);
+
+/*
+	Load shader programs
+*/
+void load_shaders(std::string name, SourceShaderCache *sourcecache,
+		video::E_DRIVER_TYPE drivertype, s32 enable_shaders,
+		std::string &vertex_program, std::string &pixel_program,
+		std::string &geometry_program, bool &is_highlevel);
+
+ShaderSource::ShaderSource(IrrlichtDevice *device):
+		m_device(device)
+{
+	assert(m_device);
+
+	m_shader_callback = new ShaderCallback(device);
+
+	m_shaderinfo_cache_mutex.Init();
+
+	m_main_thread = get_current_thread_id();
+
+	// Add a dummy ShaderInfo as the first index, named ""
+	m_shaderinfo_cache.push_back(ShaderInfo());
+	m_name_to_id[""] = 0;
+}
+
+ShaderSource::~ShaderSource()
+{
+	//m_shader_callback->drop();
+}
+
+u32 ShaderSource::getShaderId(const std::string &name)
+{
+	//infostream<<"getShaderId(): \""<<name<<"\""<<std::endl;
+
+	{
+		/*
+			See if shader already exists
+		*/
+		JMutexAutoLock lock(m_shaderinfo_cache_mutex);
+		core::map<std::string, u32>::Node *n;
+		n = m_name_to_id.find(name);
+		if(n != NULL)
+			return n->getValue();
+	}
+
+	/*
+		Get shader
+	*/
+	if(get_current_thread_id() == m_main_thread){
+		return getShaderIdDirect(name);
+	} else {
+		infostream<<"getShaderId(): Queued: name=\""<<name<<"\""<<std::endl;
+
+		// We're gonna ask the result to be put into here
+		ResultQueue<std::string, u32, u8, u8> result_queue;
+
+		// Throw a request in
+		m_get_shader_queue.add(name, 0, 0, &result_queue);
+
+		infostream<<"Waiting for shader from main thread, name=\""
+				<<name<<"\""<<std::endl;
+
+		try{
+			// Wait result for a second
+			GetResult<std::string, u32, u8, u8>
+					result = result_queue.pop_front(1000);
+
+			// Check that at least something worked OK
+			assert(result.key == name);
+
+			return result.item;
+		}
+		catch(ItemNotFoundException &e){
+			infostream<<"Waiting for shader timed out."<<std::endl;
+			return 0;
+		}
+	}
+
+	infostream<<"getShaderId(): Failed"<<std::endl;
+
+	return 0;
+}
+
+/*
+	This method generates all the shaders
+*/
+u32 ShaderSource::getShaderIdDirect(const std::string &name)
+{
+	//infostream<<"getShaderIdDirect(): name=\""<<name<<"\""<<std::endl;
+
+	// Empty name means shader 0
+	if(name == ""){
+		infostream<<"getShaderIdDirect(): name is empty"<<std::endl;
+		return 0;
+	}
+
+	/*
+		Calling only allowed from main thread
+	*/
+	if(get_current_thread_id() != m_main_thread){
+		errorstream<<"ShaderSource::getShaderIdDirect() "
+				"called not from main thread"<<std::endl;
+		return 0;
+	}
+
+	/*
+		See if shader already exists
+	*/
+	{
+		JMutexAutoLock lock(m_shaderinfo_cache_mutex);
+
+		core::map<std::string, u32>::Node *n;
+		n = m_name_to_id.find(name);
+		if(n != NULL){
+			/*infostream<<"getShaderIdDirect(): \""<<name
+					<<"\" found in cache"<<std::endl;*/
+			return n->getValue();
+		}
+	}
+
+	/*infostream<<"getShaderIdDirect(): \""<<name
+			<<"\" NOT found in cache. Creating it."<<std::endl;*/
+
+	ShaderInfo info = generate_shader(name, m_device,
+			m_shader_callback, &m_sourcecache);
+
+	/*
+		Add shader to caches (add dummy shaders too)
+	*/
+
+	JMutexAutoLock lock(m_shaderinfo_cache_mutex);
+
+	u32 id = m_shaderinfo_cache.size();
+	m_shaderinfo_cache.push_back(info);
+	m_name_to_id.insert(name, id);
+
+	/*infostream<<"getShaderIdDirect(): "
+			<<"Returning id="<<id<<" for name \""<<name<<"\""<<std::endl;*/
+
+	return id;
+}
+
+std::string ShaderSource::getShaderName(u32 id)
+{
+	JMutexAutoLock lock(m_shaderinfo_cache_mutex);
+
+	if(id >= m_shaderinfo_cache.size()){
+		errorstream<<"ShaderSource::getShaderName(): id="<<id
+				<<" >= m_shaderinfo_cache.size()="
+				<<m_shaderinfo_cache.size()<<std::endl;
+		return "";
+	}
+
+	return m_shaderinfo_cache[id].name;
+}
+
+ShaderInfo ShaderSource::getShader(u32 id)
+{
+	JMutexAutoLock lock(m_shaderinfo_cache_mutex);
+
+	if(id >= m_shaderinfo_cache.size())
+		return ShaderInfo();
+
+	return m_shaderinfo_cache[id];
+}
+
+void ShaderSource::processQueue()
+{
+	/*
+		Fetch shaders
+	*/
+	if(m_get_shader_queue.size() > 0){
+		GetRequest<std::string, u32, u8, u8>
+				request = m_get_shader_queue.pop();
+
+		/*infostream<<"ShaderSource::processQueue(): "
+				<<"got shader request with "
+				<<"name=\""<<request.key<<"\""
+				<<std::endl;*/
+
+		GetResult<std::string, u32, u8, u8>
+				result;
+		result.key = request.key;
+		result.callers = request.callers;
+		result.item = getShaderIdDirect(request.key);
+
+		request.dest->push_back(result);
+	}
+}
+
+void ShaderSource::insertSourceShader(const std::string &name_of_shader,
+		const std::string &filename, const std::string &program)
+{
+	/*infostream<<"ShaderSource::insertSourceShader(): "
+			"name_of_shader=\""<<name_of_shader<<"\", "
+			"filename=\""<<filename<<"\""<<std::endl;*/
+
+	assert(get_current_thread_id() == m_main_thread);
+
+	m_sourcecache.insert(name_of_shader, filename, program, true);
+}
+
+void ShaderSource::rebuildShaders()
+{
+	JMutexAutoLock lock(m_shaderinfo_cache_mutex);
+
+	/*// Oh well... just clear everything, they'll load sometime.
+	m_shaderinfo_cache.clear();
+	m_name_to_id.clear();*/
+
+	/*
+		FIXME: Old shader materials can't be deleted in Irrlicht,
+		or can they?
+		(This would be nice to do in the destructor too)
+	*/
+
+	// Recreate shaders
+	for(u32 i=0; i<m_shaderinfo_cache.size(); i++){
+		ShaderInfo *info = &m_shaderinfo_cache[i];
+		*info = generate_shader(info->name, m_device,
+				m_shader_callback, &m_sourcecache);
+	}
+}
+ 
+ShaderInfo generate_shader(std::string name, IrrlichtDevice *device,
+		video::IShaderConstantSetCallBack *callback,
+		SourceShaderCache *sourcecache)
+{
+	/*infostream<<"generate_shader(): "
+			"\""<<name<<"\""<<std::endl;*/
+
+	ShaderInfo shaderinfo;
+	shaderinfo.name = name;
+	shaderinfo.material = video::EMT_SOLID;
+
+	/*
+		Get the base material
+	*/
+	std::string base_material_name = sourcecache->getOrLoad(name, "base.txt");
+	for(s32 i = 0; video::sBuiltInMaterialTypeNames[i] != 0; i++){
+		if(video::sBuiltInMaterialTypeNames[i] == base_material_name){
+			shaderinfo.material = (video::E_MATERIAL_TYPE) i;
+			break;
+		}
+	}
+
+	// 0 = off, 1 = assembly shaders only, 2 = highlevel or assembly
+	s32 enable_shaders = g_settings->getS32("enable_shaders");
+	if(enable_shaders <= 0)
+		return shaderinfo;
+
+	video::IVideoDriver* driver = device->getVideoDriver();
+	assert(driver);
+
+	video::IGPUProgrammingServices *gpu = driver->getGPUProgrammingServices();
+	if(!gpu){
+		errorstream<<"generate_shader(): "
+				"failed to generate \""<<name<<"\", "
+				"GPU programming not supported."
+				<<std::endl;
+		return shaderinfo;
+	}
+
+	// Choose shader language depending on driver type and settings
+	// Then load shaders
+	std::string vertex_program;
+	std::string pixel_program;
+	std::string geometry_program;
+	bool is_highlevel;
+	load_shaders(name, sourcecache, driver->getDriverType(),
+			enable_shaders, vertex_program, pixel_program,
+			geometry_program, is_highlevel);
+
+	// Check hardware/driver support
+	if(vertex_program != "" &&
+			!driver->queryFeature(video::EVDF_VERTEX_SHADER_1_1) &&
+			!driver->queryFeature(video::EVDF_ARB_VERTEX_PROGRAM_1)){
+		infostream<<"generate_shader(): vertex shaders disabled "
+				"because of missing driver/hardware support."
+				<<std::endl;
+		vertex_program = "";
+	}
+	if(pixel_program != "" &&
+			!driver->queryFeature(video::EVDF_PIXEL_SHADER_1_1) &&
+			!driver->queryFeature(video::EVDF_ARB_FRAGMENT_PROGRAM_1)){
+		infostream<<"generate_shader(): pixel shaders disabled "
+				"because of missing driver/hardware support."
+				<<std::endl;
+		pixel_program = "";
+	}
+	if(geometry_program != "" &&
+			!driver->queryFeature(video::EVDF_GEOMETRY_SHADER)){
+		infostream<<"generate_shader(): geometry shaders disabled "
+				"because of missing driver/hardware support."
+				<<std::endl;
+		geometry_program = "";
+	}
+
+	// If no shaders are used, don't make a separate material type
+	if(vertex_program == "" && pixel_program == "" && geometry_program == "")
+		return shaderinfo;
+
+	// Call addHighLevelShaderMaterial() or addShaderMaterial()
+	const c8* vertex_program_ptr = 0;
+	const c8* pixel_program_ptr = 0;
+	const c8* geometry_program_ptr = 0;
+	if(vertex_program != "")
+		vertex_program_ptr = vertex_program.c_str();
+	if(pixel_program != "")
+		pixel_program_ptr = pixel_program.c_str();
+	if(geometry_program != "")
+		geometry_program_ptr = geometry_program.c_str();
+	s32 shadermat = -1;
+	if(is_highlevel){
+		infostream<<"Compiling high level shaders for "<<name<<std::endl;
+		shadermat = gpu->addHighLevelShaderMaterial(
+			vertex_program_ptr,   // Vertex shader program
+			"vertexMain",         // Vertex shader entry point
+			video::EVST_VS_1_1,   // Vertex shader version
+			pixel_program_ptr,    // Pixel shader program
+			"pixelMain",          // Pixel shader entry point
+			video::EPST_PS_1_1,   // Pixel shader version
+			geometry_program_ptr, // Geometry shader program
+			"geometryMain",       // Geometry shader entry point
+			video::EGST_GS_4_0,   // Geometry shader version
+			scene::EPT_TRIANGLES,      // Geometry shader input
+			scene::EPT_TRIANGLE_STRIP, // Geometry shader output
+			0,                         // Support maximum number of vertices
+			callback,             // Set-constant callback
+			shaderinfo.material,  // Base material
+			1                     // Userdata passed to callback
+			);
+
+		if(shadermat == -1){
+			errorstream<<"generate_shader(): "
+					"failed to generate \""<<name<<"\", "
+					"addHighLevelShaderMaterial failed."
+					<<std::endl;
+			return shaderinfo;
+		}
+	}
+	else{
+		infostream<<"Compiling assembly shaders for "<<name<<std::endl;
+		shadermat = gpu->addShaderMaterial(
+			vertex_program_ptr,   // Vertex shader program
+			pixel_program_ptr,    // Pixel shader program
+			callback,             // Set-constant callback
+			shaderinfo.material,  // Base material
+			0                     // Userdata passed to callback
+			);
+
+		if(shadermat == -1){
+			errorstream<<"generate_shader(): "
+					"failed to generate \""<<name<<"\", "
+					"addShaderMaterial failed."
+					<<std::endl;
+			return shaderinfo;
+		}
+	}
+
+	// HACK, TODO: investigate this better
+	// Grab the material renderer once more so minetest doesn't crash on exit
+	driver->getMaterialRenderer(shadermat)->grab();
+
+	// Apply the newly created material type
+	shaderinfo.material = (video::E_MATERIAL_TYPE) shadermat;
+	return shaderinfo;
+}
+
+void load_shaders(std::string name, SourceShaderCache *sourcecache,
+		video::E_DRIVER_TYPE drivertype, s32 enable_shaders,
+		std::string &vertex_program, std::string &pixel_program,
+		std::string &geometry_program, bool &is_highlevel)
+{
+	vertex_program = "";
+	pixel_program = "";
+	geometry_program = "";
+	is_highlevel = false;
+
+	if(enable_shaders >= 2){
+		// Look for high level shaders
+		if(drivertype == video::EDT_DIRECT3D9){
+			// Direct3D 9: HLSL
+			// (All shaders in one file)
+			vertex_program = sourcecache->getOrLoad(name, "d3d9.hlsl");
+			pixel_program = vertex_program;
+			geometry_program = vertex_program;
+		}
+		else if(drivertype == video::EDT_OPENGL){
+			// OpenGL: GLSL
+			vertex_program = sourcecache->getOrLoad(name, "opengl_vertex.glsl");
+			pixel_program = sourcecache->getOrLoad(name, "opengl_fragment.glsl");
+			geometry_program = sourcecache->getOrLoad(name, "opengl_geometry.glsl");
+		}
+		if(vertex_program != "" || pixel_program != "" || geometry_program != ""){
+			is_highlevel = true;
+			return;
+		}
+	}
+
+	if(enable_shaders >= 1){
+		// Look for assembly shaders
+		if(drivertype == video::EDT_DIRECT3D8){
+			// Direct3D 8 assembly shaders
+			vertex_program = sourcecache->getOrLoad(name, "d3d8_vertex.asm");
+			pixel_program = sourcecache->getOrLoad(name, "d3d8_pixel.asm");
+		}
+		else if(drivertype == video::EDT_DIRECT3D9){
+			// Direct3D 9 assembly shaders
+			vertex_program = sourcecache->getOrLoad(name, "d3d9_vertex.asm");
+			pixel_program = sourcecache->getOrLoad(name, "d3d9_pixel.asm");
+		}
+		else if(drivertype == video::EDT_OPENGL){
+			// OpenGL assembly shaders
+			vertex_program = sourcecache->getOrLoad(name, "opengl_vertex.asm");
+			pixel_program = sourcecache->getOrLoad(name, "opengl_fragment.asm");
+		}
+		if(vertex_program != "" || pixel_program != "")
+			return;
+	}
+}

src/shader.h

@@ -0,0 +1,89 @@
+/*
+Minetest-c55
+Copyright (C) 2012 celeron55, Perttu Ahola <celeron55@gmail.com>
+Copyright (C) 2012 Kahrl <kahrl@gmx.net>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#ifndef SHADER_HEADER
+#define SHADER_HEADER
+
+#include "irrlichttypes_extrabloated.h"
+#include "threads.h"
+#include <string>
+
+class IGameDef;
+
+/*
+	shader.{h,cpp}: Shader handling stuff.
+*/
+
+/*
+	Gets the path to a shader by first checking if the file
+	  name_of_shader/filename
+	exists in shader_path and if not, using the data path.
+
+	If not found, returns "".
+
+	Utilizes a thread-safe cache.
+*/
+std::string getShaderPath(const std::string &name_of_shader,
+		const std::string &filename);
+
+struct ShaderInfo
+{
+	std::string name;
+	video::E_MATERIAL_TYPE material;
+
+	ShaderInfo(): name(""), material(video::EMT_SOLID) {}
+};
+
+/*
+	ShaderSource creates and caches shaders.
+*/
+
+class IShaderSource
+{
+public:
+	IShaderSource(){}
+	virtual ~IShaderSource(){}
+	virtual u32 getShaderId(const std::string &name){return 0;}
+	virtual u32 getShaderIdDirect(const std::string &name){return 0;}
+	virtual std::string getShaderName(u32 id){return "";}
+	virtual ShaderInfo getShader(u32 id){return ShaderInfo();}
+	virtual ShaderInfo getShader(const std::string &name){return ShaderInfo();}
+};
+
+class IWritableShaderSource : public IShaderSource
+{
+public:
+	IWritableShaderSource(){}
+	virtual ~IWritableShaderSource(){}
+	virtual u32 getShaderId(const std::string &name){return 0;}
+	virtual u32 getShaderIdDirect(const std::string &name){return 0;}
+	virtual std::string getShaderName(u32 id){return "";}
+	virtual ShaderInfo getShader(u32 id){return ShaderInfo();}
+	virtual ShaderInfo getShader(const std::string &name){return ShaderInfo();}
+
+	virtual void processQueue()=0;
+	virtual void insertSourceShader(const std::string &name_of_shader,
+		const std::string &filename, const std::string &program)=0;
+	virtual void rebuildShaders()=0;
+};
+
+IWritableShaderSource* createShaderSource(IrrlichtDevice *device);
+
+#endif