Build: Fast global need-rebuild detection

.gitignore

@@ -3,6 +3,7 @@
 *.lib
 *.obj
 *.exe
+*.dep
 *.ldflags
 *.files
 docs

Makefile

@@ -63,13 +63,15 @@ bsnes/out/libsnes.$(ARCHIVE_SUFFIX): forcelook
 	$(REALRANLIB) bsnes/out/libsnes.$(ARCHIVE_SUFFIX)
 
 
-src/__all_files__: src/core/version.cpp forcelook
+src/__all_files__: src/core/version.cpp buildaux/mkdeps.exe forcelook
 	$(MAKE) -C src precheck
 	$(MAKE) -C src
 	cp src/lsnes$(DOT_EXECUTABLE_SUFFIX) .
 
 buildaux/version.exe: buildaux/version.cpp VERSION
 	$(HOSTCC) $(HOSTCCFLAGS) -o $@ $<
+buildaux/mkdeps.exe: buildaux/mkdeps.cpp VERSION
+	$(HOSTCC) $(HOSTCCFLAGS) -o $@ $< -lboost_filesystem -lboost_system
 src/core/version.cpp: buildaux/version.exe forcelook
 	buildaux/version.exe >$@
 

buildaux/mkdeps.cpp

@@ -0,0 +1,106 @@
+#include <boost/filesystem.hpp>
+#include <sys/time.h>
+#include <fstream>
+#include <cctype>
+#include <set>
+#include <map>
+#include <iostream>
+#include <string>
+
+namespace boost_fs = boost::filesystem;
+
+std::string search_include(const std::list<std::string>& searchpath, const std::string& filename,
+	const std::string& ref_by)
+{
+	size_t p = ref_by.find_last_of("/");
+	if(p < ref_by.length()) {
+		std::string i = ref_by;
+		i = i.substr(0, p);
+		std::string real_fn = i + "/" + filename;
+		boost_fs::path p(real_fn);
+		if(boost_fs::exists(p) && boost_fs::is_regular_file(p))
+			return real_fn;
+	}
+	for(auto& i : searchpath) {
+		std::string real_fn = i + "/" + filename;
+		boost_fs::path p(real_fn);
+		if(boost_fs::exists(p) && boost_fs::is_regular_file(p))
+			return real_fn;
+	}
+	std::cerr << "WARNING: Include file '" << filename << "' not found." << std::endl;
+	return "";
+}
+
+time_t get_timestamp(const std::string& path)
+{
+	boost_fs::path p(path);
+	if(!boost_fs::exists(p)) return 0;
+	return boost_fs::last_write_time(p);
+}
+
+time_t recursive_scan(const std::list<std::string>& searchpath, const std::string& filename,
+	std::map<std::string, time_t>& scanned)
+{
+	if(filename == "")
+		return 0;
+	if(scanned.count(filename))
+		return 0;
+	std::ifstream fp(filename);
+	if(!fp) {
+		std::cerr << "WARNING: File '" << filename << "' can't be opened." << std::endl;
+		return 0;
+	}
+	time_t newest = get_timestamp(filename);
+	scanned[filename] = newest;
+	std::string tmp;
+	while(std::getline(fp, tmp)) {
+		if(tmp.length() > 0 && tmp[0] == '#') {
+			//Possibly include.
+			std::string included;
+			if(strncmp(tmp.c_str(), "#include", 8))
+				continue;
+			size_t ptr = 8;
+			while(ptr < tmp.length() && isspace((unsigned char)tmp[ptr]))
+				ptr++;
+			if(ptr == tmp.length())
+				continue;
+			if(tmp[ptr] != '\"')
+				continue;
+			size_t iptr = ++ptr;
+			while(ptr < tmp.length() && tmp[ptr] != '\"')
+				ptr++;
+			if(ptr == tmp.length())
+				continue;
+			included = tmp.substr(iptr, ptr - iptr);
+			newest = std::max(newest, recursive_scan(searchpath, search_include(searchpath, included,
+				filename), scanned));
+		}
+	}
+	return newest;
+}
+
+int main(int argc, char** argv)
+{
+	std::list<std::string> searchpath;
+	std::list<std::string> files;
+	bool step = false;
+	for(int i = 1; i < argc; i++) {
+		if(!step && !strcmp(argv[i], "--"))
+			step = true;
+		else if(!step)
+			searchpath.push_back(argv[i]);
+		else
+			files.push_back(argv[i]);
+	}
+	searchpath.push_back(".");
+	for(auto& i : files) {
+		std::map<std::string, time_t> x;
+		time_t t = recursive_scan(searchpath, i, x);
+		if(get_timestamp(i + ".dep") < t) {
+			std::ofstream y(i + ".dep");
+			for(auto& j : x)
+				y << j.second << " " << j.first << std::endl;
+		}
+	}
+	return 0;
+}

src/Makefile

@@ -78,6 +78,7 @@ lsnes$(DOT_EXECUTABLE_SUFFIX): __all_common__.files __all_platform__.files
 precheck:
 	$(MAKE) -C core precheck
 	$(MAKE) -C emulation precheck
+	$(MAKE) -C fonts precheck
 	$(MAKE) -C interface precheck
 	$(MAKE) -C library precheck
 	$(MAKE) -C lua precheck
@@ -87,6 +88,7 @@ precheck:
 
 clean:
 	rm -f *.$(OBJECT_SUFFIX) *.ldflags
+	find . -name "*.dep" -exec rm -f {} \;
 	$(MAKE) -C core clean
 	$(MAKE) -C emulation clean
 	$(MAKE) -C fonts clean

src/core/Makefile

@@ -6,10 +6,11 @@ __all__.files: $(OBJECTS)
 	lua ../genfilelist.lua $^ >$@
 	touch __all__.ldflags
 
-%.$(OBJECT_SUFFIX): %.cpp
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep
 	$(REALCC) $(CFLAGS) -c -o $@ $< -I../../include
 
 precheck:
+	../../buildaux/mkdeps.exe ../../include -- *.cpp
 	@true
 
 clean:

src/emulation/bsnes-legacy/Makefile

@@ -38,7 +38,7 @@ __all__.files: $(OBJECTS) ../../../$(BSNES_LIBRARY)
 ports.inc: ports.json ../make-ports.exe
 	../make-ports.exe <$< >$@
 
-%.$(OBJECT_SUFFIX): %.cpp ports.inc
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep ports.inc
 	$(REALCC) -c -o $@ $< -I../../../include -I../../../bsnes $(CFLAGS) $(BSNES_CFLAGS)
 else
 
@@ -53,6 +53,7 @@ forcelook:
 	@true
 
 precheck:
+	../../../buildaux/mkdeps.exe ../../../include -- *.cpp
 	@true
 
 clean:

src/emulation/gambatte/Makefile

@@ -18,7 +18,7 @@ __all__.files: $(OBJECTS) ../../../$(GAMBATTE_LIBRARY)
 ports.inc: ports.json ../make-ports.exe
 	../make-ports.exe <$< >$@
 
-%.$(OBJECT_SUFFIX): %.cpp ports.inc
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep ports.inc
 	$(REALCC) -c -o $@ $< -I../../../include -I../../../gambatte $(CFLAGS) $(GAMBATTE_CFLAGS)
 
 else
@@ -34,6 +34,7 @@ forcelook:
 	@true
 
 precheck:
+	../../../buildaux/mkdeps.exe ../../../include ../../../gambatte -- *.cpp
 	@true
 
 clean:

src/emulation/sky/Makefile

@@ -6,13 +6,14 @@ __all__.files: $(OBJECTS)
 	lua ../../genfilelist.lua $^ >$@
 	echo >__all__.ldflags
 
-%.$(OBJECT_SUFFIX): %.cpp
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep
 	$(REALCC) -c -o $@ $< -I../../../include -I. $(CFLAGS)
 
 forcelook:
 	@true
 
 precheck:
+	../../../buildaux/mkdeps.exe ../../../include -- *.cpp
 	@true
 
 clean:

src/emulation/test/Makefile

@@ -9,13 +9,14 @@ __all__.files: $(OBJECTS)
 ports.inc: ports.json ../make-ports.exe
 	../make-ports.exe <$< >$@
 
-%.$(OBJECT_SUFFIX): %.cpp ports.inc
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep ports.inc
 	$(REALCC) -c -o $@ $< -I../../../include -I../../../gambatte $(CFLAGS)
 
 forcelook:
 	@true
 
 precheck:
+	../../../buildaux/mkdeps.exe ../../../include -- *.cpp
 	@true
 
 clean:

src/fonts/Makefile

@@ -8,16 +8,21 @@ __all__.files: $(OBJECTS)
 
 .PRECIOUS: font.$(OBJECT_SUFFIX) font.cpp
 
-%.$(OBJECT_SUFFIX): %.cpp
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep
 	$(REALCC) $(CFLAGS) -c -o $@ $< -I../../include
 
-font.cpp : $(FONT_SRC)
+font.cpp font.cpp.dep: $(FONT_SRC)
 	echo "extern const char* font_hex_data;" >font.cpp
 	echo "const char* font_hex_data = " >>font.cpp
 	sed -E -f fonttransform.sed <$^ >>font.cpp
 	echo ";" >>font.cpp
+	touch font.cpp.dep
 
 precheck:
+	../../buildaux/mkdeps.exe ../../include -- *.cpp
+	@true
+
+forcelook:
 	@true
 
 clean:

src/interface/Makefile

@@ -6,10 +6,14 @@ __all__.files: $(OBJECTS)
 	lua ../genfilelist.lua $^ >$@
 	touch __all__.ldflags
 
-%.$(OBJECT_SUFFIX): %.cpp
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep
 	$(REALCC) $(CFLAGS) -c -o $@ $< -I../../include
 
 precheck:
+	../../buildaux/mkdeps.exe ../../include -- *.cpp
+	@true
+
+forcelook:
 	@true
 
 clean:

src/library/Makefile

@@ -6,10 +6,14 @@ __all__.files: $(OBJECTS)
 	lua ../genfilelist.lua $^ >$@
 	touch __all__.ldflags
 
-%.$(OBJECT_SUFFIX): %.cpp
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep
 	$(REALCC) $(CFLAGS) -c -o $@ $< -I../../include/library
 
 precheck:
+	../../buildaux/mkdeps.exe ../../include/library -- *.cpp
+	@true
+
+forcelook:
 	@true
 
 clean:

src/library/skein.cpp

@@ -204,8 +204,8 @@ void skein_hash::read(uint8_t* output)
 
 #ifdef TEST_SKEIN_CODE
 #define SKEIN_DEBUG
-#include "skein.h"
+#include <skein.h>
-#include "skein_debug.h"
+#include <skein_debug.h>
 
 
 

src/lua/Makefile

@@ -6,10 +6,14 @@ __all__.files: $(OBJECTS)
 	lua ../genfilelist.lua $^ >$@
 	echo >__all__.ldflags
 
-%.$(OBJECT_SUFFIX): %.cpp
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep
 	$(REALCC) -c -o $@ $< -I../../include $(CFLAGS)
 
 precheck:
+	../../buildaux/mkdeps.exe ../../include -- *.cpp
+	@true
+
+forcelook:
 	@true
 
 clean:

src/platform/evdev/Makefile

@@ -10,10 +10,14 @@ __all__.files: $(OBJECTS)
 	lua ../../genfilelist.lua $^ >$@
 	touch __all__.ldflags
 
-%.$(OBJECT_SUFFIX): %.cpp
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep
 	$(REALCC) -c -o $@ $< -I../../../include $(CFLAGS)
 
 precheck:
+	../../../buildaux/mkdeps.exe ../../../include -- *.cpp
+	@true
+
+forcelook:
 	@true
 
 clean:

src/platform/libao/Makefile

@@ -12,10 +12,14 @@ __all__.files: $(OBJECTS)
 	lua ../../genfilelist.lua $^ >$@
 	echo $(PA_LDFLAGS) >__all__.ldflags
 
-%.$(OBJECT_SUFFIX): %.cpp
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep
 	$(REALCC) -c -o $@ $< -I../../../include $(CFLAGS) $(PA_CFLAGS)
 
 precheck:
+	../../../buildaux/mkdeps.exe ../../../include -- *.cpp
+	@true
+
+forcelook:
 	@true
 
 clean:

src/platform/macosx/Makefile

@@ -10,13 +10,17 @@ __all__.files: $(OBJECTS)
 	lua ../../genfilelist.lua $^ >$@
 	touch __all__.ldflags
 
-%.$(OBJECT_SUFFIX): %.cpp
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep
 	$(REALCC) -c -o $@ $< -I../../../include $(CFLAGS)
 
-%.$(OBJECT_SUFFIX): %.c
+%.$(OBJECT_SUFFIX): %.c %c.dep
 	$(REALCC) -DSDL_JOYSTICK_IOKIT -x c --std=gnu99 -c -o $@ $< $(CSTDFLAGS)
 
 precheck:
+	../../../buildaux/mkdeps.exe ../../../include -- *.cpp *.c
+	@true
+
+forcelook:
 	@true
 
 clean:

src/platform/portaudio/Makefile

@@ -13,10 +13,14 @@ __all__.files: $(OBJECTS)
 	lua ../../genfilelist.lua $^ >$@
 	echo $(PA_LDFLAGS) >__all__.ldflags
 
-%.$(OBJECT_SUFFIX): %.cpp
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep
 	$(REALCC) -c -o $@ $< -I../../../include $(CFLAGS) $(PA_CFLAGS)
 
 precheck:
+	../../../buildaux/mkdeps.exe ../../../include -- *.cpp
+	@true
+
+forcelook:
 	@true
 
 clean:

src/platform/win32mm/Makefile

@@ -10,10 +10,14 @@ __all__.files: $(OBJECTS)
 	lua ../../genfilelist.lua $^ >$@
 	touch __all__.ldflags
 
-%.$(OBJECT_SUFFIX): %.cpp
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep
 	$(REALCC) -c -o $@ $< -I../../../include $(CFLAGS)
 
 precheck:
+	../../../buildaux/mkdeps.exe ../../../include -- *.cpp
+	@true
+
+forcelook:
 	@true
 
 clean:

src/platform/wxwidgets/Makefile

@@ -20,10 +20,14 @@ __all__.files: $(OBJECTS)
 	lua ../../genfilelist.lua $^ >$@
 	echo $(WXW_LDFLAGS) >__all__.ldflags
 
-%.$(OBJECT_SUFFIX): %.cpp
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep
 	$(REALCC) -c -o $@ $< -I../../../include $(CFLAGS) $(WXW_CFLAGS)
 
 precheck:
+	../../../buildaux/mkdeps.exe ../../../include -- *.cpp
+	@true
+
+forcelook:
 	@true
 
 clean:

src/util/Makefile

@@ -5,11 +5,14 @@ OBJECTS=$(patsubst %.cpp,%.$(OBJECT_SUFFIX),$(wildcard *.cpp))
 __all_files__: $(OBJECTS)
 	@true
 
-%.$(OBJECT_SUFFIX): %.cpp
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep
 	$(REALCC) -c -o $@ $< -I../../include $(CFLAGS)
-	touch $(patsubst %.$(OBJECT_SUFFIX),%.ldflags,$^)
 
 precheck:
+	../../buildaux/mkdeps.exe ../../include -- *.cpp
+	@true
+
+forcelook:
 	@true
 
 clean:

src/video/Makefile

@@ -14,13 +14,14 @@ __all__.files: avi/__all__.files $(OBJECTS)
 avi/__all__.files: forcelook
 	$(MAKE) -C avi
 
-%.$(OBJECT_SUFFIX): %.cpp
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep
 	$(REALCC) $(CFLAGS) -c -o $@ $< -I../../include $(AVI_CFLAGS)
 
 forcelook:
 	@true
 
 precheck:
+	../../buildaux/mkdeps.exe ../../include -- *.cpp
 	$(MAKE) -C avi precheck
 
 clean:

src/video/avi/Makefile

@@ -8,13 +8,14 @@ __all__.files: codec/__all__.files $(OBJECTS)
 codec/__all__.files: forcelook
 	$(MAKE) -C codec
 
-%.$(OBJECT_SUFFIX): %.cpp
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep
 	$(REALCC) $(CFLAGS) -c -o $@ $< -I../../../include
 
 forcelook:
 	@true
 
 precheck:
+	../../../buildaux/mkdeps.exe ../../../include -- *.cpp
 	$(MAKE) -C codec precheck
 
 clean:

src/video/avi/codec/Makefile

@@ -11,7 +11,7 @@ audio/__all__.files: forcelook
 video/__all__.files: forcelook
 	$(MAKE) -C video
 
-%.$(OBJECT_SUFFIX): %.cpp
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep
 	$(REALCC) $(CFLAGS) -c -o $@ $< -I../../include
 
 forcelook:

src/video/avi/codec/audio/Makefile

@@ -5,10 +5,14 @@ OBJECTS=$(patsubst %.cpp,%.$(OBJECT_SUFFIX),$(wildcard *.cpp))
 __all__.files: $(OBJECTS)
 	lua ../../../../genfilelist.lua $^ >$@
 
-%.$(OBJECT_SUFFIX): %.cpp
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep
 	$(REALCC) $(CFLAGS) -c -o $@ $< -I../../../../../include
 
 precheck:
+	../../../../../buildaux/mkdeps.exe ../../../../../include -- *.cpp
+	@true
+
+forcelook:
 	@true
 
 clean:

src/video/avi/codec/video/Makefile

@@ -5,10 +5,14 @@ OBJECTS=$(patsubst %.cpp,%.$(OBJECT_SUFFIX),$(wildcard *.cpp))
 __all__.files: $(OBJECTS)
 	lua ../../../../genfilelist.lua $^ >$@
 
-%.$(OBJECT_SUFFIX): %.cpp
+%.$(OBJECT_SUFFIX): %.cpp %.cpp.dep
 	$(REALCC) $(CFLAGS) -c -o $@ $< -I../../../../../include
 
 precheck:
+	../../../../../buildaux/mkdeps.exe ../../../../../include -- *.cpp
+	@true
+
+forcelook:
 	@true
 
 clean: