Tests: Added basic way of finding potentially broken games

Core/AutomaticRomTest.cpp

@@ -0,0 +1,167 @@
+#include "stdafx.h"
+#include "../Utilities/FolderUtilities.h"
+#include "AutomaticRomTest.h"
+#include "EmulationSettings.h"
+#include "Console.h"
+#include "PPU.h"
+#include "VideoDecoder.h"
+#include "StandardController.h"
+
+bool AutomaticRomTest::_running = false;
+
+AutomaticRomTest::AutomaticRomTest()
+{
+	_running = true;
+	_errorCode = 0;
+	MessageManager::RegisterNotificationListener(this);
+}
+
+AutomaticRomTest::~AutomaticRomTest()
+{
+	_running = false;
+	MessageManager::UnregisterNotificationListener(this);
+}
+
+void AutomaticRomTest::ProcessNotification(ConsoleNotificationType type, void* parameter)
+{
+	if(type == ConsoleNotificationType::PpuFrameDone) {
+		uint16_t *frameBuffer = (uint16_t*)parameter;
+		if(PPU::GetFrameCount() == 5) {
+			memcpy(_prevFrameBuffer, frameBuffer, sizeof(_prevFrameBuffer));
+		} else if(PPU::GetFrameCount() == 300) {
+			if(memcmp(_prevFrameBuffer, frameBuffer, sizeof(_prevFrameBuffer)) == 0) {
+				//No change
+				_errorCode |= 0x20;
+			}
+			memcpy(_prevFrameBuffer, frameBuffer, sizeof(_prevFrameBuffer));
+			VideoDecoder::GetInstance()->TakeScreenshot();
+		} else if(PPU::GetFrameCount() == 900) {
+			if(memcmp(_prevFrameBuffer, frameBuffer, sizeof(_prevFrameBuffer)) == 0) {
+				//No change
+				_errorCode |= 0x01;
+			}
+
+			bool allZeros = true;
+			for(int i = 0; i < 256 * 240; i++) {
+				if(frameBuffer[i] != 0) {
+					allZeros = false;
+					break;
+				}
+			}
+
+			if(allZeros) {
+				_errorCode |= 0x04;
+			}
+
+			memcpy(_prevFrameBuffer, frameBuffer, sizeof(_prevFrameBuffer));
+			VideoDecoder::GetInstance()->TakeScreenshot();
+		} else if(PPU::GetFrameCount() == 1800) {
+			bool continueTest = false;
+			if(memcmp(_prevFrameBuffer, frameBuffer, sizeof(_prevFrameBuffer)) == 0) {
+				//No change, change input pattern and keep trying
+				continueTest = true;
+			}
+
+			bool allZeros = true;
+			for(int i = 0; i < 256 * 240; i++) {
+				if(frameBuffer[i] != 0) {
+					allZeros = false;
+					break;
+				}
+			}
+
+			if(allZeros) {
+				_errorCode |= 0x08;
+			}
+
+			VideoDecoder::GetInstance()->TakeScreenshot();
+
+			if(!continueTest) {
+				//Stop test
+				_signal.Signal();
+			}
+		} else if(PPU::GetFrameCount() == 3600) {
+			if(memcmp(_prevFrameBuffer, frameBuffer, sizeof(_prevFrameBuffer)) == 0) {
+				//No change
+				_errorCode |= 0x02;
+			}
+
+			bool allZeros = true;
+			for(int i = 0; i < 256 * 240; i++) {
+				if(frameBuffer[i] != 0) {
+					allZeros = false;
+					break;
+				}
+			}
+
+			if(allZeros) {
+				_errorCode |= 0x40;
+			}
+
+			VideoDecoder::GetInstance()->TakeScreenshot();
+
+			//Stop test
+			_signal.Signal();
+		}
+	}
+}
+
+int32_t AutomaticRomTest::Run(string filename)
+{
+	EmulationSettings::SetEmulationSpeed(0);
+	EmulationSettings::SetMasterVolume(0);
+	Console::Pause();
+	if(Console::LoadROM(filename)) {
+		Console::Resume();
+		EmulationSettings::ClearFlags(EmulationFlags::Paused);
+		_signal.Wait();
+
+		EmulationSettings::SetFlags(EmulationFlags::Paused);
+
+		if(PPU::GetFrameCount() < 1800) {
+			//Finished early
+			_errorCode |= 0x10;
+		}
+
+		EmulationSettings::SetEmulationSpeed(100);
+		EmulationSettings::SetMasterVolume(1.0);
+
+		Console::GetInstance()->Stop();
+
+		return _errorCode;
+	}
+
+	return -1;
+}
+
+bool AutomaticRomTest::Running()
+{
+	return _running;
+}
+
+uint8_t AutomaticRomTest::GetControllerState(uint8_t port)
+{
+	if(port == 0) {
+		uint32_t frameNumber = PPU::GetFrameCount();
+
+		if(frameNumber <= 1800) {
+			if(frameNumber % 30 < 10) {
+				//Press 1 button for 10 frames every second
+				if((frameNumber / 30) % 8 != 1) {
+					return 1 << ((frameNumber / 60) % 8);
+				}
+			}
+		} else {
+			if(frameNumber % 30 < 10) {
+				if((frameNumber / 30) % 2) {
+					return 0x01;
+				} else {
+					return 0x08;
+				}
+			}
+		}
+		return 0;
+	} else {
+		return 0;
+	}
+}

Core/AutomaticRomTest.h

@@ -0,0 +1,23 @@
+#pragma once
+#include "stdafx.h"
+#include "INotificationListener.h"
+#include "../Utilities/AutoResetEvent.h"
+
+class AutomaticRomTest : public INotificationListener
+{
+private:
+	AutoResetEvent _signal;
+	static bool _running;
+	uint16_t _prevFrameBuffer[256 * 240];
+	uint32_t _errorCode;
+
+public:
+	AutomaticRomTest();
+	~AutomaticRomTest();
+
+	void ProcessNotification(ConsoleNotificationType type, void* parameter) override;
+	int32_t Run(string filename);
+
+	static bool Running();
+	static uint8_t GetControllerState(uint8_t port);
+};

Core/BaseControlDevice.cpp

@@ -5,6 +5,7 @@
 #include "EmulationSettings.h"
 #include "GameClient.h"
 #include "GameServerConnection.h"
+#include "AutomaticRomTest.h"
 
 BaseControlDevice::BaseControlDevice(uint8_t port)
 {
@@ -63,6 +64,8 @@ uint8_t BaseControlDevice::GetControlState()
 		_currentState = MovieManager::GetState(_port);
 	} else if(GameClient::Connected()) {
 		_currentState = GameClient::GetControllerState(_port);
+	} else if(AutomaticRomTest::Running()) {
+		_currentState = AutomaticRomTest::GetControllerState(_port);
 	} else if(netPlayDevice) {
 		_currentState = ProcessNetPlayState(netPlayDevice->GetState());
 	} else {

Core/Console.cpp

@@ -43,7 +43,7 @@ void Console::Release()
 	Console::Instance.reset(new Console());
 }
 
-void Console::Initialize(string romFilename, stringstream *filestream, string patchFilename, int32_t archiveFileIndex)
+bool Console::Initialize(string romFilename, stringstream *filestream, string patchFilename, int32_t archiveFileIndex)
 {
 	SoundMixer::StopAudio();
 
@@ -105,16 +105,19 @@ void Console::Initialize(string romFilename, stringstream *filestream, string pa
 		if(EmulationSettings::GetOverclockRate() != 100) {
 			MessageManager::DisplayMessage("ClockRate", std::to_string(EmulationSettings::GetOverclockRate()) + "%");
 		}
+		return true;
 	} else {
 		MessageManager::DisplayMessage("Error", "CouldNotLoadFile", FolderUtilities::GetFilename(romFilename, true));
+		return false;
 	}
 }
 
-void Console::LoadROM(string filepath, stringstream *filestream, int32_t archiveFileIndex, string patchFilepath)
+bool Console::LoadROM(string filepath, stringstream *filestream, int32_t archiveFileIndex, string patchFilepath)
 {
 	Console::Pause();
-	Instance->Initialize(filepath, filestream, patchFilepath, archiveFileIndex);
+	bool result = Instance->Initialize(filepath, filestream, patchFilepath, archiveFileIndex);
 	Console::Resume();
+	return result;
 }
 
 bool Console::LoadROM(string romName, uint32_t crc32Hash)
@@ -145,8 +148,7 @@ bool Console::LoadROM(string romName, HashInfo hashInfo)
 	for(string folder : FolderUtilities::GetKnownGameFolders()) {
 		string match = RomLoader::FindMatchingRomInFolder(folder, romName, hashInfo, true, archiveFileIndex);
 		if(!match.empty()) {
-			Console::LoadROM(match, nullptr, archiveFileIndex);
+			return Console::LoadROM(match, nullptr, archiveFileIndex);
-			return true;
 		}
 	}
 
@@ -154,8 +156,7 @@ bool Console::LoadROM(string romName, HashInfo hashInfo)
 	for(string folder : FolderUtilities::GetKnownGameFolders()) {
 		string match = RomLoader::FindMatchingRomInFolder(folder, romName, hashInfo, false, archiveFileIndex);
 		if(!match.empty()) {
-			Console::LoadROM(match, nullptr, archiveFileIndex);
+			return Console::LoadROM(match, nullptr, archiveFileIndex);
-			return true;
 		}
 	}
 

Core/Console.h

@@ -47,7 +47,7 @@ class Console
 		bool _initialized = false;
 
 		void ResetComponents(bool softReset);
-		void Initialize(string filename, stringstream *filestream = nullptr, string patchFilename = "", int32_t archiveFileIndex = -1);
+		bool Initialize(string filename, stringstream *filestream = nullptr, string patchFilename = "", int32_t archiveFileIndex = -1);
 		void UpdateNesModel(bool sendNotification);
 		double GetFrameDelay();
 
@@ -73,7 +73,7 @@ class Console
 		static void LoadState(istream &loadStream);
 		static void LoadState(uint8_t *buffer, uint32_t bufferSize);
 
-		static void LoadROM(string filepath, stringstream *filestream = nullptr, int32_t archiveFileIndex = -1, string patchFilepath = "");
+		static bool LoadROM(string filepath, stringstream *filestream = nullptr, int32_t archiveFileIndex = -1, string patchFilepath = "");
 		static bool LoadROM(string romName, HashInfo hashInfo);
 		static bool LoadROM(string romName, uint32_t crc32Hash);
 		static bool LoadROM(string romName, string sha1Hash);

Core/Core.vcxproj

@@ -410,7 +410,8 @@
     <ClInclude Include="APU.h" />
     <ClInclude Include="ArkanoidController.h" />
     <ClInclude Include="Assembler.h" />
-    <ClInclude Include="AutoRomTest.h" />
+    <ClInclude Include="AutomaticRomTest.h" />
+    <ClInclude Include="RecordedRomTest.h" />
     <ClInclude Include="AutoSaveManager.h" />
     <ClInclude Include="AviRecorder.h" />
     <ClInclude Include="Ax5705.h" />
@@ -764,7 +765,8 @@
     <ClCompile Include="ApuLengthCounter.cpp" />
     <ClCompile Include="ArkanoidController.cpp" />
     <ClCompile Include="Assembler.cpp" />
-    <ClCompile Include="AutoRomTest.cpp" />
+    <ClCompile Include="AutomaticRomTest.cpp" />
+    <ClCompile Include="RecordedRomTest.cpp" />
     <ClCompile Include="AutoSaveManager.cpp" />
     <ClCompile Include="AviRecorder.cpp" />
     <ClCompile Include="BaseControlDevice.cpp" />

Core/Core.vcxproj.filters

@@ -553,9 +553,6 @@
     <ClInclude Include="Snapshotable.h">
       <Filter>Nes</Filter>
     </ClInclude>
-    <ClInclude Include="AutoRomTest.h">
-      <Filter>Misc</Filter>
-    </ClInclude>
     <ClInclude Include="EmulationSettings.h">
       <Filter>Misc</Filter>
     </ClInclude>
@@ -1156,6 +1153,12 @@
     <ClInclude Include="BizhawkMovie.h">
       <Filter>Movies</Filter>
     </ClInclude>
+    <ClInclude Include="RecordedRomTest.h">
+      <Filter>Misc</Filter>
+    </ClInclude>
+    <ClInclude Include="AutomaticRomTest.h">
+      <Filter>Misc</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="stdafx.cpp">
@@ -1185,9 +1188,6 @@
     <ClCompile Include="CodeDataLogger.cpp">
       <Filter>Debugger</Filter>
     </ClCompile>
-    <ClCompile Include="AutoRomTest.cpp">
-      <Filter>Misc</Filter>
-    </ClCompile>
     <ClCompile Include="BaseVideoFilter.cpp">
       <Filter>VideoDecoder</Filter>
     </ClCompile>
@@ -1365,5 +1365,11 @@
     <ClCompile Include="BizhawkMovie.cpp">
       <Filter>Movies</Filter>
     </ClCompile>
+    <ClCompile Include="RecordedRomTest.cpp">
+      <Filter>Misc</Filter>
+    </ClCompile>
+    <ClCompile Include="AutomaticRomTest.cpp">
+      <Filter>Misc</Filter>
+    </ClCompile>
   </ItemGroup>
 </Project>

Core/RecordedRomTest.cpp

@@ -1,6 +1,6 @@
 #include "stdafx.h"
 
-#include "AutoRomTest.h"
+#include "RecordedRomTest.h"
 #include "Console.h"
 #include "EmulationSettings.h"
 #include "MessageManager.h"
@@ -11,21 +11,21 @@
 #include "../Utilities/ZipWriter.h"
 #include "../Utilities/ZipReader.h"
 
-AutoRomTest::AutoRomTest()
+RecordedRomTest::RecordedRomTest()
 {
 	Reset();
 
 	MessageManager::RegisterNotificationListener(this);
 }
 
-AutoRomTest::~AutoRomTest()
+RecordedRomTest::~RecordedRomTest()
 {
 	Reset();
 
 	MessageManager::UnregisterNotificationListener(this);
 }
 
-void AutoRomTest::SaveFrame(uint16_t* ppuFrameBuffer)
+void RecordedRomTest::SaveFrame(uint16_t* ppuFrameBuffer)
 {
 	uint8_t md5Hash[16];
 	GetMd5Sum(md5Hash, ppuFrameBuffer, PPU::PixelCount * sizeof(uint16_t));
@@ -47,7 +47,7 @@ void AutoRomTest::SaveFrame(uint16_t* ppuFrameBuffer)
 	}
 }
 
-void AutoRomTest::ValidateFrame(uint16_t* ppuFrameBuffer)
+void RecordedRomTest::ValidateFrame(uint16_t* ppuFrameBuffer)
 {
 	uint8_t md5Hash[16];
 	GetMd5Sum(md5Hash, ppuFrameBuffer, PPU::PixelCount * sizeof(uint16_t));
@@ -71,7 +71,7 @@ void AutoRomTest::ValidateFrame(uint16_t* ppuFrameBuffer)
 	}
 }
 
-void AutoRomTest::ProcessNotification(ConsoleNotificationType type, void* parameter)
+void RecordedRomTest::ProcessNotification(ConsoleNotificationType type, void* parameter)
 {
 	switch(type) {
 		case ConsoleNotificationType::PpuFrameDone:
@@ -89,7 +89,7 @@ void AutoRomTest::ProcessNotification(ConsoleNotificationType type, void* parame
 	}
 }
 
-void AutoRomTest::Reset()
+void RecordedRomTest::Reset()
 {
 	memset(_previousHash, 0xFF, 16);
 	
@@ -107,7 +107,7 @@ void AutoRomTest::Reset()
 	_recordingFromMovie = false;
 }
 
-void AutoRomTest::Record(string filename, bool reset)
+void RecordedRomTest::Record(string filename, bool reset)
 {
 	_filename = filename;
 
@@ -127,7 +127,7 @@ void AutoRomTest::Record(string filename, bool reset)
 	}
 }
 
-void AutoRomTest::RecordFromMovie(string testFilename, stringstream &movieStream, bool autoLoadRom)
+void RecordedRomTest::RecordFromMovie(string testFilename, stringstream &movieStream, bool autoLoadRom)
 {
 	_filename = testFilename;
 
@@ -151,7 +151,7 @@ void AutoRomTest::RecordFromMovie(string testFilename, stringstream &movieStream
 	}
 }
 
-void AutoRomTest::RecordFromMovie(string testFilename, string movieFilename)
+void RecordedRomTest::RecordFromMovie(string testFilename, string movieFilename)
 {
 	stringstream ss;
 	ifstream file(movieFilename, ios::in | ios::binary);
@@ -162,7 +162,7 @@ void AutoRomTest::RecordFromMovie(string testFilename, string movieFilename)
 	}
 }
 
-void AutoRomTest::RecordFromTest(string newTestFilename, string existingTestFilename)
+void RecordedRomTest::RecordFromTest(string newTestFilename, string existingTestFilename)
 {
 	ZipReader zipReader;
 	zipReader.LoadArchive(existingTestFilename);
@@ -180,7 +180,7 @@ void AutoRomTest::RecordFromTest(string newTestFilename, string existingTestFile
 	}
 }
 
-int32_t AutoRomTest::Run(string filename)
+int32_t RecordedRomTest::Run(string filename)
 {
 	string testName = FolderUtilities::GetFilename(filename, false);
 	if(testName.compare("5.MMC3_rev_A") == 0 || testName.compare("6-MMC6") == 0 || testName.compare("6-MMC3_alt") == 0) {
@@ -255,7 +255,7 @@ int32_t AutoRomTest::Run(string filename)
 	return -1;
 }
 
-void AutoRomTest::Stop()
+void RecordedRomTest::Stop()
 {
 	if(_recording) {
 		Save();
@@ -263,7 +263,7 @@ void AutoRomTest::Stop()
 	Reset();
 }
 
-void AutoRomTest::Save()
+void RecordedRomTest::Save()
 {
 	//Wait until the next frame is captured to end the recording
 	_signal.Wait();

Core/RecordedRomTest.h

@@ -5,7 +5,7 @@
 #include "INotificationListener.h"
 #include "../Utilities/AutoResetEvent.h"
 
-class AutoRomTest : public INotificationListener
+class RecordedRomTest : public INotificationListener
 {
 private:
 	bool _recording;
@@ -35,8 +35,8 @@ private:
 	void RecordFromMovie(string testFilename, stringstream &movieStream, bool autoLoadRom);
 
 public:
-	AutoRomTest();
+	RecordedRomTest();
-	virtual ~AutoRomTest();
+	virtual ~RecordedRomTest();
 
 	void ProcessNotification(ConsoleNotificationType type, void* parameter) override;
 	void Record(string filename, bool reset);

GUI.NET/Forms/frmMain.Designer.cs

@@ -179,6 +179,7 @@ namespace Mesen.GUI.Forms
 			this.toolStripMenuItem5 = new System.Windows.Forms.ToolStripSeparator();
 			this.mnuHelpWindow = new System.Windows.Forms.ToolStripMenuItem();
 			this.mnuAbout = new System.Windows.Forms.ToolStripMenuItem();
+			this.mnuRunAutomaticTest = new System.Windows.Forms.ToolStripMenuItem();
 			this.panelRenderer.SuspendLayout();
 			this.menuStrip.SuspendLayout();
 			this.SuspendLayout();
@@ -1238,7 +1239,8 @@ namespace Mesen.GUI.Forms
             this.mnuTestRecordFrom,
             this.mnuTestStopRecording,
             this.mnuRunAllTests,
-            this.mnuRunAllGameTests});
+            this.mnuRunAllGameTests,
+            this.mnuRunAutomaticTest});
 			this.mnuTests.Name = "mnuTests";
 			this.mnuTests.Size = new System.Drawing.Size(231, 22);
 			this.mnuTests.Text = "Tests";
@@ -1265,28 +1267,28 @@ namespace Mesen.GUI.Forms
 			// 
 			this.mnuTestRecordStart.Name = "mnuTestRecordStart";
 			this.mnuTestRecordStart.ShortcutKeys = ((System.Windows.Forms.Keys)((System.Windows.Forms.Keys.Control | System.Windows.Forms.Keys.W)));
-			this.mnuTestRecordStart.Size = new System.Drawing.Size(143, 22);
+			this.mnuTestRecordStart.Size = new System.Drawing.Size(152, 22);
 			this.mnuTestRecordStart.Text = "Start";
 			this.mnuTestRecordStart.Click += new System.EventHandler(this.mnuTestRecordStart_Click);
 			// 
 			// mnuTestRecordNow
 			// 
 			this.mnuTestRecordNow.Name = "mnuTestRecordNow";
-			this.mnuTestRecordNow.Size = new System.Drawing.Size(143, 22);
+			this.mnuTestRecordNow.Size = new System.Drawing.Size(152, 22);
 			this.mnuTestRecordNow.Text = "Now";
 			this.mnuTestRecordNow.Click += new System.EventHandler(this.mnuTestRecordNow_Click);
 			// 
 			// mnuTestRecordMovie
 			// 
 			this.mnuTestRecordMovie.Name = "mnuTestRecordMovie";
-			this.mnuTestRecordMovie.Size = new System.Drawing.Size(143, 22);
+			this.mnuTestRecordMovie.Size = new System.Drawing.Size(152, 22);
 			this.mnuTestRecordMovie.Text = "Movie";
 			this.mnuTestRecordMovie.Click += new System.EventHandler(this.mnuTestRecordMovie_Click);
 			// 
 			// mnuTestRecordTest
 			// 
 			this.mnuTestRecordTest.Name = "mnuTestRecordTest";
-			this.mnuTestRecordTest.Size = new System.Drawing.Size(143, 22);
+			this.mnuTestRecordTest.Size = new System.Drawing.Size(152, 22);
 			this.mnuTestRecordTest.Text = "Test";
 			this.mnuTestRecordTest.Click += new System.EventHandler(this.mnuTestRecordTest_Click);
 			// 
@@ -1406,6 +1408,13 @@ namespace Mesen.GUI.Forms
 			this.mnuAbout.Text = "About";
 			this.mnuAbout.Click += new System.EventHandler(this.mnuAbout_Click);
 			// 
+			// mnuRunAutomaticTest
+			// 
+			this.mnuRunAutomaticTest.Name = "mnuRunAutomaticTest";
+			this.mnuRunAutomaticTest.Size = new System.Drawing.Size(192, 22);
+			this.mnuRunAutomaticTest.Text = "Run automatic test";
+			this.mnuRunAutomaticTest.Click += new System.EventHandler(this.mnuRunAutomaticTest_Click);
+			// 
 			// frmMain
 			// 
 			this.AllowDrop = true;
@@ -1580,6 +1589,7 @@ namespace Mesen.GUI.Forms
 		private System.Windows.Forms.ToolStripMenuItem mnuPrescale6xFilter;
 		private System.Windows.Forms.ToolStripMenuItem mnuPrescale8xFilter;
 		private System.Windows.Forms.ToolStripMenuItem mnuPrescale10xFilter;
+		private System.Windows.Forms.ToolStripMenuItem mnuRunAutomaticTest;
 	}
 }
 

GUI.NET/Forms/frmMain.cs

@@ -1060,7 +1060,7 @@ namespace Mesen.GUI.Forms
 
 				Task.Run(() => {
 					foreach(string filename in ofd.FileNames) {
-						int result = InteropEmu.RomTestRun(filename);
+						int result = InteropEmu.RunRecordedTest(filename);
 
 						if(result == 0) {
 							passedTests.Add(Path.GetFileNameWithoutExtension(filename));
@@ -1207,6 +1207,20 @@ namespace Mesen.GUI.Forms
 			InteropEmu.SetNesModel(ConfigManager.Config.Region);
 		}
 
+		private void mnuRunAutomaticTest_Click(object sender, EventArgs e)
+		{
+			using(OpenFileDialog ofd = new OpenFileDialog()) {
+				ofd.SetFilter("*.nes|*.nes");
+				if(ofd.ShowDialog() == System.Windows.Forms.DialogResult.OK) {
+					string filename = ofd.FileName;
+					
+					Task.Run(() => {
+						int result = InteropEmu.RunAutomaticTest(filename);
+					});
+				}
+			}
+		}
+
 		private void mnuRunAllTests_Click(object sender, EventArgs e)
 		{
 			string workingDirectory = Path.Combine(Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location));

GUI.NET/InteropEmu.cs

@@ -99,7 +99,8 @@ namespace Mesen.GUI
 		[DllImport(DLLPath)] public static extern void WaveStop();
 		[DllImport(DLLPath)] [return: MarshalAs(UnmanagedType.I1)] public static extern bool WaveIsRecording();
 
-		[DllImport(DLLPath)] public static extern Int32 RomTestRun([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UTF8Marshaler))]string filename);
+		[DllImport(DLLPath)] public static extern Int32 RunRecordedTest([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UTF8Marshaler))]string filename);
+		[DllImport(DLLPath)] public static extern Int32 RunAutomaticTest([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UTF8Marshaler))]string filename);
 		[DllImport(DLLPath)] public static extern void RomTestRecord([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UTF8Marshaler))]string filename, [MarshalAs(UnmanagedType.I1)]bool reset);
 		[DllImport(DLLPath)] public static extern void RomTestRecordFromMovie([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UTF8Marshaler))]string testFilename, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UTF8Marshaler))]string movieFilename);
 		[DllImport(DLLPath)] public static extern void RomTestRecordFromTest([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UTF8Marshaler))]string newTestFilename, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UTF8Marshaler))]string existingTestFilename);

InteropDLL/ConsoleWrapper.cpp

@@ -9,7 +9,8 @@
 #include "../Core/CheatManager.h"
 #include "../Core/EmulationSettings.h"
 #include "../Core/VideoDecoder.h"
-#include "../Core/AutoRomTest.h"
+#include "../Core/AutomaticRomTest.h"
+#include "../Core/RecordedRomTest.h"
 #include "../Core/FDS.h"
 #include "../Core/VsControlManager.h"
 #include "../Core/SoundMixer.h"
@@ -37,7 +38,7 @@ void*  _windowHandle = nullptr;
 void* _viewerHandle = nullptr;
 string _returnString;
 string _logString;
-AutoRomTest *_autoRomTest = nullptr;
+RecordedRomTest *_recordedRomTest = nullptr;
 
 typedef void (__stdcall *NotificationListenerCallback)(int);
 
@@ -313,43 +314,49 @@ namespace InteropEmu {
 		DllExport void __stdcall WaveStop() { SoundMixer::StopRecording(); }
 		DllExport bool __stdcall WaveIsRecording() { return SoundMixer::IsRecording(); }
 
-		DllExport int32_t __stdcall RomTestRun(char* filename)
+		DllExport int32_t __stdcall RunRecordedTest(char* filename)
 		{
-			AutoRomTest romTest; 
+			RecordedRomTest romTest; 
+			return romTest.Run(filename);
+		}
+
+		DllExport int32_t __stdcall RunAutomaticTest(char* filename)
+		{
+			AutomaticRomTest romTest;
 			return romTest.Run(filename);
 		}
 
 		DllExport void __stdcall RomTestRecord(char* filename, bool reset) 
 		{
-			if(_autoRomTest) {
+			if(_recordedRomTest) {
-				delete _autoRomTest;
+				delete _recordedRomTest;
 			}
-			_autoRomTest = new AutoRomTest();
+			_recordedRomTest = new RecordedRomTest();
-			_autoRomTest->Record(filename, reset); 
+			_recordedRomTest->Record(filename, reset);
 		}
 
 		DllExport void __stdcall RomTestRecordFromMovie(char* testFilename, char* movieFilename) 
 		{
-			_autoRomTest = new AutoRomTest();
+			_recordedRomTest = new RecordedRomTest();
-			_autoRomTest->RecordFromMovie(testFilename, movieFilename);
+			_recordedRomTest->RecordFromMovie(testFilename, movieFilename);
 		}
 
 		DllExport void __stdcall RomTestRecordFromTest(char* newTestFilename, char* existingTestFilename) 
 		{
-			_autoRomTest = new AutoRomTest();
+			_recordedRomTest = new RecordedRomTest();
-			_autoRomTest->RecordFromTest(newTestFilename, existingTestFilename);
+			_recordedRomTest->RecordFromTest(newTestFilename, existingTestFilename);
 		}
 
 		DllExport void __stdcall RomTestStop() 
 		{
-			if(_autoRomTest) {
+			if(_recordedRomTest) {
-				_autoRomTest->Stop();
+				_recordedRomTest->Stop();
-				delete _autoRomTest;
+				delete _recordedRomTest;
-				_autoRomTest = nullptr;
+				_recordedRomTest = nullptr;
 			}
 		}
 
-		DllExport bool __stdcall RomTestRecording() { return _autoRomTest != nullptr; }
+		DllExport bool __stdcall RomTestRecording() { return _recordedRomTest != nullptr; }
 
 		DllExport void __stdcall SetCheats(CheatInfo cheats[], uint32_t length) { CheatManager::SetCheats(cheats, length); }
 

TestHelper/TestHelper.cpp

@@ -46,7 +46,8 @@ public:
 extern "C" {
 	void __stdcall InitializeEmu(const char* homeFolder, void*, void*, bool, bool, bool);
 	void __stdcall SetControllerType(uint32_t port, ControllerType type);
-	int __stdcall RomTestRun(char* filename);
+	int __stdcall RunAutomaticTest(char* filename);
+	int __stdcall RunRecordedTest(char* filename);
 	void __stdcall LoadROM(char* filename);
 	void __stdcall Run();
 	void __stdcall Stop();
@@ -57,21 +58,27 @@ std::thread *runThread = nullptr;
 std::atomic<int> testIndex;
 vector<string> testFilenames;
 vector<string> failedTests;
+vector<int32_t> failedTestErrorCode;
 SimpleLock lock;
 Timer timer;
+bool automaticTests = false;
+
+void RunEmu()
+{
+	try {
+		Run();
+	} catch(std::exception ex) {
+
+	}
+}
 
 void __stdcall OnNotificationReceived(ConsoleNotificationType type)
 {
 	if(type == ConsoleNotificationType::GameLoaded) {
-		runThread = new std::thread(Run);
+		runThread = new std::thread(RunEmu);
 	}
 }
 
-void RunEmu()
-{
-	Run();
-}
-
 void RunTest()
 {
 	while(true) {
@@ -82,11 +89,21 @@ void RunTest()
 		if(index < testFilenames.size()) {
 			string filepath = testFilenames[index];
 			string filename = FolderUtilities::GetFilename(filepath, false);
-			#ifdef _WIN32
+
-				string command = "TestHelper.exe /testrom \"" + filepath + "\"";
+			string command;
-			#else
+			if(automaticTests) {
-				string command = "./testhelper /testrom \"" + filepath + "\"";
+				#ifdef _WIN32
-			#endif
+					command = "TestHelper.exe /autotest \"" + filepath + "\"";
+				#else
+					command = "./testhelper /autotest \"" + filepath + "\"";
+				#endif
+			} else {
+				#ifdef _WIN32
+					command = "TestHelper.exe /testrom \"" + filepath + "\"";
+				#else
+					command = "./testhelper /testrom \"" + filepath + "\"";
+				#endif
+			}
 
 			lock.Acquire();
 			std::cout << std::to_string(index) << ") " << filename << std::endl;
@@ -101,6 +118,7 @@ void RunTest()
 				//Test failed
 				lock.Acquire();
 				failedTests.push_back(filename);
+				failedTestErrorCode.push_back(failedFrames);
 				std::cout << "  ****  " << std::to_string(index) << ") " << filename << " failed (" << failedFrames << ")" << std::endl;
 				lock.Release();
 			}
@@ -132,18 +150,24 @@ int main(int argc, char* argv[])
 		signal(SIGSEGV, handler);		
 	#endif
 
-	if(argc <= 2) {
+	if(argc >= 3 && strcmp(argv[1], "/auto") == 0) {
+		string romFolder = argv[2];
+		testFilenames = FolderUtilities::GetFilesInFolder(romFolder, ".nes", true);
+		automaticTests = true;
+	} else if(argc <= 2) {
 		string testFolder;
 		if(argc == 1) {
 			testFolder = FolderUtilities::CombinePath(mesenFolder, "Tests");
 		} else {
 			testFolder = argv[1];
 		}
-
-		vector<std::thread*> testThreads;
 		testFilenames = FolderUtilities::GetFilesInFolder(testFolder, ".mtp", true);
-		testIndex = 0;
+		automaticTests = false;
+	}
 
+	if(!testFilenames.empty()) {
+		vector<std::thread*> testThreads;
+		testIndex = 0;
 		timer.Reset();
 
 		int numberOfThreads = 4;
@@ -162,9 +186,10 @@ int main(int argc, char* argv[])
 			std::cout << "------------" << std::endl;
 			std::cout << "Failed tests" << std::endl;
 			std::cout << "------------" << std::endl;
-			for(string failedTest : failedTests) {
+			for(int i = 0; i < failedTests.size(); i++) {
-				std::cout << failedTest << std::endl;
+				std::cout << failedTests[i] << " (" << std::to_string(failedTestErrorCode[i]) << ")" << std::endl;
 			}
+			std::cout << std::endl << std::to_string(failedTests.size()) << " tests failed." << std::endl;
 		} else {
 			std::cout << std::endl << std::endl << "All tests passed.";
 		}
@@ -179,7 +204,13 @@ int main(int argc, char* argv[])
 		SetControllerType(0, ControllerType::StandardController);
 		SetControllerType(1, ControllerType::StandardController);
 
-		int result = RomTestRun(testFilename);
+		int result = 0;
+		if(strcmp(argv[1], "/testrom") == 0) {
+			result = RunRecordedTest(testFilename);
+		} else {
+			result = RunAutomaticTest(testFilename);
+		}
+		
 		if(runThread != nullptr) {
 			runThread->join();
 			delete runThread;