From 5ea3c304571cd956b3bb6be79806733c82558a77 Mon Sep 17 00:00:00 2001 From: empathicqubit Date: Wed, 28 Apr 2021 18:54:39 -0400 Subject: [PATCH] Reuse processes for more efficiency --- config.lua | 3 +- game.lua | 31 ++++++++--- pool.lua | 28 +++++----- runner-process.lua | 130 +++++++++++++++++++++++++++++---------------- runner-wrapper.lua | 92 +++++++++++++++++--------------- runner.lua | 11 ++-- state-test.lua | 29 ++++------ util.lua | 87 ++++++++++++++++++++++++++++++ 8 files changed, 277 insertions(+), 134 deletions(-) diff --git a/config.lua b/config.lua index 126f2e6..bf76cef 100644 --- a/config.lua +++ b/config.lua @@ -35,8 +35,7 @@ _M.Filename = _M.PoolDir .. _M.State[1] _M.StartPowerup = 0 _M.NeatConfig = { -Threads = 2, -ThreadDontQuit = false, +Threads = 4, --Filename = "DP1.state", SaveFile = _M.Filename .. ".pool", Filename = _M.Filename, diff --git a/game.lua b/game.lua index d0f6994..e4125c0 100644 --- a/game.lua +++ b/game.lua @@ -349,7 +349,6 @@ function _M.getExtendedSprites() return extended end -callcount = 0 function _M.getInputs() _M.getPositions() @@ -432,7 +431,7 @@ function _M.onEmptyHit(handler) table.insert(emptyHitQueue, handler) end -function processEmptyHit(addr, val) +local function processEmptyHit(addr, val) local idx = math.floor((bit.band(addr, 0xffff) - bit.band(SPRITE_BASE, 0xffff)) / SPRITE_SIZE) local pow = _M.getSprite(idx) if pow == nil or @@ -454,23 +453,41 @@ function processEmptyHit(addr, val) end end -function processAreaLoad() +local function processAreaLoad() for i=#areaLoadedQueue,1,-1 do table.remove(areaLoadedQueue, i)() end end -function processMapLoad() +local function processMapLoad() for i=#mapLoadedQueue,1,-1 do table.remove(mapLoadedQueue, i)() end end +local handlers = {} +local function registerHandler(space, regname, addr, callback) + table.insert(handlers, { + + fn = space[regname](space, addr, callback), + unregisterFn = space['un'..regname], + space = space, + addr = addr, + }) +end + +function _M.unregisterHandlers() + for i=#handlers,1,-1 do + local handler = table.remove(handlers, i) + handler.unregisterFn(handler.space, handler.addr, handler.fn) + end +end + function _M.registerHandlers() - memory2.BUS:registerwrite(0xb517b2, processAreaLoad) - memory2.WRAM:registerread(0x06b1, processMapLoad) + registerHandler(memory2.BUS, 'registerwrite', 0xb517b2, processAreaLoad) + registerHandler(memory2.WRAM, 'registerread', 0x06b1, processMapLoad) for i=2,22,1 do - memory2.WRAM:registerwrite(bit.band(SPRITE_BASE + SPRITE_SIZE * i, 0xffff), processEmptyHit) + registerHandler(memory2.WRAM, 'registerwrite', bit.band(SPRITE_BASE + SPRITE_SIZE * i, 0xffff), processEmptyHit) end end diff --git a/pool.lua b/pool.lua index 6fee97f..b422ffe 100644 --- a/pool.lua +++ b/pool.lua @@ -640,6 +640,20 @@ local function newGeneration() writeFile(_M.saveLoadFile .. ".gen" .. pool.generation .. ".pool") end +local runner = Runner() +runner.onMessage(function(msg, color) + message(msg, color) +end) +runner.onSave(function(filename) + _M.requestSave(filename) +end) +runner.onLoad(function(filename) + _M.requestLoad(filename) +end) +runner.onRenderForm(function(form) + processRenderForm(form) +end) + local loadRequested = false local saveRequested = false local function mainLoop(currentSpecies) @@ -667,19 +681,7 @@ local function mainLoop(currentSpecies) if currentSpecies == nil then currentSpecies = 1 end - local runner = Runner() - runner.onMessage(function(msg, color) - message(msg, color) - end) - runner.onSave(function(filename) - _M.requestSave(filename) - end) - runner.onLoad(function(filename) - _M.requestLoad(filename) - end) - runner.onRenderForm(function(form) - processRenderForm(form) - end) + local slice = pool.species[currentSpecies] if config.NeatConfig.Threads > 1 then slice = {} diff --git a/runner-process.lua b/runner-process.lua index e84cf39..e336521 100644 --- a/runner-process.lua +++ b/runner-process.lua @@ -1,38 +1,30 @@ local base = string.gsub(@@LUA_SCRIPT_FILENAME@@, "(.*[/\\])(.*)", "%1") +local Promise = dofile(base.."/promise.lua") + local Runner = dofile(base.."/runner.lua") local serpent = dofile(base.."/serpent.lua") local util = dofile(base.."/util.lua") -local runnerDataFile = io.open(os.getenv("RUNNER_DATA"), 'r') -local ok, runnerData = serpent.load(runnerDataFile:read('*a')) -runnerDataFile:close() +local inputFilePath = os.getenv("RUNNER_INPUT_FILE") +local outputFilePath = os.getenv("RUNNER_OUTPUT_FILE") -if not ok then - print("Deserialization error") - return -end - -local species = runnerData[1] - -local speciesId = species.id - -local generationIndex = runnerData[2] - -local filename = runnerData[3] - -local outFile = io.open(filename, "w") +local first = false local outContents = {} local statusLine = nil local statusColor = 0x0000ff00 +local species = nil +local speciesId = nil +local generationIndex = nil + local runner = Runner() runner.onMessage(function(msg, color) statusLine = msg statusColor = color - + print(msg) table.insert( outContents, serpent.dump({ @@ -84,32 +76,80 @@ runner.onLoad(function(filename) ) end) -runner.run( - species, - generationIndex, - function(genome, index) - table.insert( - outContents, - serpent.dump({ - type = 'onGenome', - genome = genome, - genomeIndex = index, - speciesId = speciesId, - }) - ) - end, - function() - table.insert( - outContents, - serpent.dump({ - type = 'onFinish', - speciesId = speciesId, - }) - ) - outFile:write(table.concat(outContents, "\n")) +local waiter = util.waitForChange(inputFilePath) + +local function waitLoop() + if not first then + local sec, usec = utime() + local ts = sec * 1000000 + usec + + local outFile = io.open(outputFilePath, "w") + outFile:write(serpent.dump({ type = 'onInit', ts = ts })) outFile:close() - if os.getenv("RUNNER_DONT_QUIT") == nil then - exec('quit-emulator') - end + + print(string.format('Wrote init to output at %d', ts)) + + first = true end -) + + print('Waiting for input from master process') + + waiter:read("*a") + util.closeCmd(waiter) + + print('Received input from master process') + + local inputFile = io.open(inputFilePath, 'r') + local ok, inputData = serpent.load(inputFile:read('*a')) + inputFile:close() + + if not ok then + print("Deserialization error") + return + end + + species = inputData[1] + + speciesId = species.id + + generationIndex = inputData[2] + + outContents = {} + + print('Running') + + runner.run( + species, + generationIndex, + function(genome, index) + table.insert( + outContents, + serpent.dump({ + type = 'onGenome', + genome = genome, + genomeIndex = index, + speciesId = speciesId, + }) + ) + end, + function() + table.insert( + outContents, + serpent.dump({ + type = 'onFinish', + speciesId = speciesId, + }) + ) + + waiter = util.waitForChange(inputFilePath) + + local outFile = io.open(outputFilePath, "w") + outFile:write(table.concat(outContents, "\n")) + outFile:close() + + waitLoop() + end + ) +end + +waitLoop() \ No newline at end of file diff --git a/runner-wrapper.lua b/runner-wrapper.lua index 4e24e4f..d450b0a 100644 --- a/runner-wrapper.lua +++ b/runner-wrapper.lua @@ -19,7 +19,10 @@ for i=1,#temps,1 do end end -local tmpFileName = tempDir.."/donk_runner" +local tmpFileName = tempDir.."/donk_runner_".. + + string.hex(math.floor(random.integer(0, 0xffffffff))).. + string.hex(math.floor(random.integer(0, 0xffffffff))) local function message(_M, msg, color) if color == nil then @@ -60,6 +63,7 @@ return function() onMessageHandler = {}, onSaveHandler = {}, onLoadHandler = {}, + poppets = {}, } _M.onRenderForm = function(handler) @@ -82,55 +86,59 @@ return function() end _M.run = function(species, generationIdx, genomeCallback, finishCallback) - local poppets = {} - for i=1,#species,1 do + local hostProcess = "lsnes" + if util.isWin then + hostProcess = util.scrapeCmd('*l', 'powershell "(Get-WmiObject Win32_Process -Filter ProcessId=$((Get-WmiObject Win32_Process -Filter ProcessId=$((Get-WmiObject Win32_Process -Filter ProcessId=$PID).ParentProcessId)).ParentProcessId)").ExecutablePath') + if hostProcess == nil or hostProcess == "" then + hostProcess = "lsnes-bsnes.exe" + end + else + -- FIXME Linux + end + + while #_M.poppets < #species do + local i = #_M.poppets+1 local outputFileName = tmpFileName..'_output_'..i - local inputFileName = tmpFileName.."_input_"..i - print(inputFileName) - local inputFile = io.open(inputFileName, 'w') - inputFile:write(serpent.dump({species[i], generationIdx, outputFileName})) - inputFile:close() - - local proc = "lsnes" - if util.isWin then - local checkParent = io.popen('powershell "(Get-WmiObject Win32_Process -Filter ProcessId=$((Get-WmiObject Win32_Process -Filter ProcessId=$((Get-WmiObject Win32_Process -Filter ProcessId=$PID).ParentProcessId)).ParentProcessId)").ExecutablePath') - proc = checkParent:read("*l") - checkParent:close() - else - -- FIXME Linux - end - print(proc) - local cmd = "\""..proc.."\" \"--rom="..config.ROM.."\" --unpause \"--lua="..base.."/runner-process.lua\"" + + message(_M, hostProcess) + local envs = { - RUNNER_DATA = inputFileName + RUNNER_INPUT_FILE = inputFileName, + RUNNER_OUTPUT_FILE = outputFileName, } - if config.NeatConfig.ThreadDontQuit then - envs.RUNNER_DONT_QUIT = "1" - end - local cmdParts = {} - for k,v in pairs(envs) do - if util.isWin then - table.insert(cmdParts, string.format("set %s=%s &&", k, v)) - else - table.insert(cmdParts, string.format("%s='%s'", k, v)) - end - end - table.insert(cmdParts, cmd) - local fullCmd = table.concat(cmdParts, " ") - message(_M, fullCmd) - local poppet = io.popen(fullCmd, 'r') - table.insert(poppets, poppet) + local waiter = util.waitForChange(outputFileName) + + local cmd = '"'..hostProcess..'" "--rom='..config.ROM..'" --unpause "--lua='..base..'/runner-process.lua"' + local poppet = util.popenCmd(cmd, nil, envs) + table.insert(_M.poppets, poppet) + + waiter:read("*a") + util.closeCmd(waiter) end - for i=1,#poppets,1 do - local poppet = poppets[i] - poppet:read('*a') - poppet:close() - end - + local outputFileName = tmpFileName..'_output_*' + local waiter = util.waitForChange(outputFileName, #species) + + message(_M, 'Setting up child processes') + for i=1,#species,1 do + local inputFileName = tmpFileName.."_input_"..i + local inputFile = io.open(inputFileName, 'w') + inputFile:write(serpent.dump({species[i], generationIdx})) + inputFile:close() + end + + message(_M, 'Waiting for child processes to finish') + + waiter:read("*a") + util.closeCmd(waiter) + + message(_M, 'Child processes finished') + + for i=1,#species,1 do + message(_M, "Processing output "..i) local outputFileName = tmpFileName..'_output_'..i local outputFile = io.open(outputFileName, "r") local line = "" diff --git a/runner.lua b/runner.lua index 9a7bb29..7eb83fa 100644 --- a/runner.lua +++ b/runner.lua @@ -236,7 +236,6 @@ function displayButtons() gui.renderctx.setnull() end -local frame = 0 local formCtx = nil local form = nil function displayForm(_M) @@ -244,7 +243,7 @@ function displayForm(_M) return end - if form ~= nil and frame % 10 ~= 0 then + if form ~= nil and _M.drawFrame % 10 ~= 0 then for i=#_M.onRenderFormHandler,1,-1 do _M.onRenderFormHandler[i](form) end @@ -299,7 +298,7 @@ local function painting(_M) if formCtx == nil then formCtx = gui.renderctx.new(500, guiHeight) end - frame = frame + 1 + _M.drawFrame = _M.drawFrame + 1 displayForm(_M) end @@ -402,7 +401,7 @@ local function mainLoop(_M, genome) genome = _M.currentSpecies.genomes[_M.currentGenomeIndex] - if frame % 10 == 0 then + if _M.drawFrame % 10 == 0 then if not pcall(function() displayGenome(genome) end) then @@ -433,7 +432,7 @@ local function mainLoop(_M, genome) -- Don't punish being launched by barrels -- FIXME Will this skew mine cart levels? if game.getVelocityY() < -2104 then - message(_M, "BARREL! "..frame, 0x00ffff00) + message(_M, "BARREL! ".._M.drawFrame, 0x00ffff00) if _M.timeout < timeoutConst + 60 * 12 then _M.timeout = _M.timeout + 60 * 12 end @@ -567,6 +566,7 @@ local function mainLoop(_M, genome) while fitnessAlreadyMeasured(_M) do _M.currentGenomeIndex = _M.currentGenomeIndex + 1 if _M.currentGenomeIndex > #_M.currentSpecies.genomes then + game.unregisterHandlers() for i=#_M.dereg,1,-1 do local d = table.remove(_M.dereg, i) callback.unregister(d[1], d[2]) @@ -850,6 +850,7 @@ return function() genomeCallback = nil, currentGenomeIndex = 1, currentFrame = 0, + drawFrame = 0, maxFitness = 0, dereg = {}, diff --git a/state-test.lua b/state-test.lua index 01c8514..b4b52ea 100644 --- a/state-test.lua +++ b/state-test.lua @@ -1,20 +1,9 @@ -PARTY_X = 0x7e0a2a -TILE_SIZE = 32 - -print(memory.readword(PARTY_X)) - -function on_post_rewind() - print("Async?") - print(memory.readword(PARTY_X)) -end - -function movement(addr, val) - if memory.readword(addr) > TILE_SIZE * 20 then - local rew = movie.to_rewind("pool/PiratePanic.lsmv") - movie.unsafe_rewind(rew) - print("Sync?") - print(memory.readword(PARTY_X)) - end -end - -memory2.WRAM:registerwrite(0x0a2a, movement) +local errtest = io.popen("farts", 'r') +errtest:read("*a") +print(errtest:close()) +errtest = io.popen("dir", 'r') +errtest:read("*a") +local ok, exit, code = errtest:close() +print(ok) +print(exit) +print(code) \ No newline at end of file diff --git a/util.lua b/util.lua index a9aa3a0..f936aed 100644 --- a/util.lua +++ b/util.lua @@ -2,6 +2,93 @@ local _M = {} _M.isWin = package.config:sub(1, 1) == '\\' +--- Echo a command, run it, and return the file handle +--- @param cmd string The command to execute +--- @param workdir string The working directory +--- @param env table The environment variables +function _M.popenCmd(cmd, workdir, env) + local cmdParts = {} + if workdir ~= nil then + if _M.isWin then + table.insert(cmdParts, 'cd /d "'..workdir..'" &&') + else + table.insert(cmdParts, 'cd "'..workdir..'" &&') + end + end + if env ~= nil then + for k,v in pairs(env) do + if _M.isWin then + table.insert(cmdParts, string.format("set %s=%s&&", k, v)) + else + table.insert(cmdParts, string.format("%s='%s'", k, v)) + end + end + end + table.insert(cmdParts, cmd) + local fullCmd = table.concat(cmdParts, " ") + print(fullCmd) + +--[[ local dummy = "/dev/null" + if isWin then + dummy = "NUL" + end + return io.open(dummy, 'r') ]] + return io.popen(fullCmd, 'r') +end + +--- Echo a command, run it, and handle any errors +--- @return string string The stdout +function _M.doCmd(...) + return _M.scrapeCmd('*a', ...) +end + +--- Run a command and get the output +--- @param formats table|string|number List or single io.read() specifier +--- @return table table List of results based on read specifiers +function _M.scrapeCmd(formats, ...) + local poppet = _M.popenCmd(...) + local outputs = nil + if type(formats) ~= 'table' then + outputs = poppet:read(formats) + else + outputs = {} + for i=1,#formats,1 do + table.insert(outputs, poppet:read(formats[i])) + end + end + _M.closeCmd(poppet) + return outputs +end + +--- Check the command's exit code and throw a Lua error if it isn't right +--- @param handle file* The handle of the command +function _M.closeCmd(handle) + local ok, state, code = handle:close() + if state ~= "exit" then + return + end + if code ~= 0 then + error("The last command failed") + end +end + +function _M.waitForChange(filename, count) + if count == nil then + count = 1 + end + + if _M.isWin then + local sec, usec = utime() + print(string.format('Starting watching file at %d', sec * 1000000 + usec)) + + return _M.popenCmd([[powershell "$filename = ']]..filename.. + [[' ; $targetCount = ]]..count..[[ ; $count = 0 ; Register-ObjectEvent (New-Object IO.FileSystemWatcher (Split-Path $filename), (Split-Path -Leaf $filename) -Property @{ IncludeSubdirectories = $false ; NotifyFilter = [IO.NotifyFilters]'FileName, LastWrite'}) -EventName Changed -SourceIdentifier RunnerDataChanged -Action { $count += 1 ; if ( $count -ge $targetCount ) { [Environment]::Exit(0) } } ; while($true) { Start-Sleep -Milliseconds 1 }"]]) + else + error("Not implemented") + -- FIXME Linux + end +end + function _M.table_to_string(tbl) local result = "{" local keys = {}