Reuse processes for more efficiency

config.lua

@@ -35,8 +35,7 @@ _M.Filename = _M.PoolDir .. _M.State[1]
 _M.StartPowerup = 0
 
 _M.NeatConfig = {
-Threads = 2,
+Threads = 4,
-ThreadDontQuit = false,
 --Filename = "DP1.state",
 SaveFile = _M.Filename .. ".pool",
 Filename = _M.Filename,

game.lua

@@ -349,7 +349,6 @@ function _M.getExtendedSprites()
 	return extended
 end
 
-callcount = 0
 function _M.getInputs()
 	_M.getPositions()
 	
@@ -432,7 +431,7 @@ function _M.onEmptyHit(handler)
     table.insert(emptyHitQueue, handler)
 end
 
-function processEmptyHit(addr, val)
+local function processEmptyHit(addr, val)
     local idx = math.floor((bit.band(addr, 0xffff) - bit.band(SPRITE_BASE, 0xffff)) / SPRITE_SIZE)
     local pow = _M.getSprite(idx)
     if pow == nil or
@@ -454,23 +453,41 @@ function processEmptyHit(addr, val)
     end
 end
 
-function processAreaLoad()
+local function processAreaLoad()
     for i=#areaLoadedQueue,1,-1 do
         table.remove(areaLoadedQueue, i)()
     end
 end
 
-function processMapLoad()
+local function processMapLoad()
     for i=#mapLoadedQueue,1,-1 do
         table.remove(mapLoadedQueue, i)()
     end
 end
 
+local handlers = {}
+local function registerHandler(space, regname, addr, callback)
+    table.insert(handlers, { 
+        
+        fn = space[regname](space, addr, callback),
+        unregisterFn = space['un'..regname],
+        space = space,
+        addr = addr,
+    })
+end
+
+function _M.unregisterHandlers()
+    for i=#handlers,1,-1 do
+        local handler = table.remove(handlers, i)
+        handler.unregisterFn(handler.space, handler.addr, handler.fn)
+    end
+end
+
 function _M.registerHandlers()
-    memory2.BUS:registerwrite(0xb517b2, processAreaLoad)
+    registerHandler(memory2.BUS, 'registerwrite', 0xb517b2, processAreaLoad)
-    memory2.WRAM:registerread(0x06b1, processMapLoad)
+    registerHandler(memory2.WRAM, 'registerread', 0x06b1, processMapLoad)
     for i=2,22,1 do
-        memory2.WRAM:registerwrite(bit.band(SPRITE_BASE + SPRITE_SIZE * i, 0xffff), processEmptyHit)
+        registerHandler(memory2.WRAM, 'registerwrite', bit.band(SPRITE_BASE + SPRITE_SIZE * i, 0xffff), processEmptyHit)
     end
 end
 

pool.lua

@@ -640,6 +640,20 @@ local function newGeneration()
 	writeFile(_M.saveLoadFile .. ".gen" .. pool.generation .. ".pool")
 end
 
+local runner = Runner()
+runner.onMessage(function(msg, color)
+	message(msg, color)
+end)
+runner.onSave(function(filename)
+	_M.requestSave(filename)
+end)
+runner.onLoad(function(filename)
+	_M.requestLoad(filename)
+end)
+runner.onRenderForm(function(form)
+	processRenderForm(form)
+end)
+
 local loadRequested = false
 local saveRequested = false
 local function mainLoop(currentSpecies)
@@ -667,19 +681,7 @@ local function mainLoop(currentSpecies)
     if currentSpecies == nil then
         currentSpecies = 1
     end
-    local runner = Runner()
+
-    runner.onMessage(function(msg, color)
-        message(msg, color)
-    end)
-    runner.onSave(function(filename)
-        _M.requestSave(filename)
-    end)
-    runner.onLoad(function(filename)
-        _M.requestLoad(filename)
-    end)
-    runner.onRenderForm(function(form)
-        processRenderForm(form)
-    end)
     local slice = pool.species[currentSpecies]
     if config.NeatConfig.Threads > 1 then
         slice = {}

runner-process.lua

@@ -1,38 +1,30 @@
 local base = string.gsub(@@LUA_SCRIPT_FILENAME@@, "(.*[/\\])(.*)", "%1")
 
+local Promise = dofile(base.."/promise.lua")
+
 local Runner = dofile(base.."/runner.lua")
 local serpent = dofile(base.."/serpent.lua")
 local util = dofile(base.."/util.lua")
 
-local runnerDataFile = io.open(os.getenv("RUNNER_DATA"), 'r')
+local inputFilePath = os.getenv("RUNNER_INPUT_FILE")
-local ok, runnerData = serpent.load(runnerDataFile:read('*a'))
+local outputFilePath = os.getenv("RUNNER_OUTPUT_FILE")
-runnerDataFile:close()
 
-if not ok then
+local first = false
-    print("Deserialization error")
-    return
-end
-
-local species = runnerData[1]
-
-local speciesId = species.id
-
-local generationIndex = runnerData[2]
-
-local filename = runnerData[3]
-
-local outFile = io.open(filename, "w")
 
 local outContents = {}
 
 local statusLine = nil
 local statusColor = 0x0000ff00
 
+local species = nil
+local speciesId = nil
+local generationIndex = nil
+
 local runner = Runner()
 runner.onMessage(function(msg, color)
     statusLine = msg
     statusColor = color
-
+    print(msg)
     table.insert(
         outContents,
         serpent.dump({
@@ -84,32 +76,80 @@ runner.onLoad(function(filename)
     )
 end)
 
-runner.run(
+local waiter = util.waitForChange(inputFilePath)
-    species,
+
-    generationIndex,
+local function waitLoop()
-    function(genome, index)
+    if not first then
-        table.insert(
+        local sec, usec = utime()
-            outContents,
+        local ts = sec * 1000000 + usec
-            serpent.dump({
+
-                type = 'onGenome',
+        local outFile = io.open(outputFilePath, "w")
-                genome = genome,
+        outFile:write(serpent.dump({ type = 'onInit', ts = ts }))
-                genomeIndex = index,
-                speciesId = speciesId,
-            })
-        )
-    end,
-    function()
-        table.insert(
-            outContents,
-            serpent.dump({
-                type = 'onFinish',
-                speciesId = speciesId,
-            })
-        )
-        outFile:write(table.concat(outContents, "\n"))
         outFile:close()
-        if os.getenv("RUNNER_DONT_QUIT") == nil then
+
-            exec('quit-emulator')
+        print(string.format('Wrote init to output at %d', ts))
-        end
+
+        first = true
     end
-)
+
+    print('Waiting for input from master process')
+
+    waiter:read("*a")
+    util.closeCmd(waiter)
+
+    print('Received input from master process')
+
+    local inputFile = io.open(inputFilePath, 'r')
+    local ok, inputData = serpent.load(inputFile:read('*a'))
+    inputFile:close()
+
+    if not ok then
+        print("Deserialization error")
+        return
+    end
+
+    species = inputData[1]
+
+    speciesId = species.id
+
+    generationIndex = inputData[2]
+
+    outContents = {}
+
+    print('Running')
+
+    runner.run(
+        species,
+        generationIndex,
+        function(genome, index)
+            table.insert(
+                outContents,
+                serpent.dump({
+                    type = 'onGenome',
+                    genome = genome,
+                    genomeIndex = index,
+                    speciesId = speciesId,
+                })
+            )
+        end,
+        function()
+            table.insert(
+                outContents,
+                serpent.dump({
+                    type = 'onFinish',
+                    speciesId = speciesId,
+                })
+            )
+
+            waiter = util.waitForChange(inputFilePath)
+
+            local outFile = io.open(outputFilePath, "w")
+            outFile:write(table.concat(outContents, "\n"))
+            outFile:close()
+
+            waitLoop()
+        end
+    )
+end
+
+waitLoop()

runner-wrapper.lua

@@ -19,7 +19,10 @@ for i=1,#temps,1 do
     end
 end
 
-local tmpFileName = tempDir.."/donk_runner"
+local tmpFileName = tempDir.."/donk_runner_"..
+
+    string.hex(math.floor(random.integer(0, 0xffffffff)))..
+    string.hex(math.floor(random.integer(0, 0xffffffff)))
 
 local function message(_M, msg, color)
     if color == nil then
@@ -60,6 +63,7 @@ return function()
         onMessageHandler = {},
         onSaveHandler = {},
         onLoadHandler = {},
+        poppets = {},
     }
 
     _M.onRenderForm = function(handler)
@@ -82,55 +86,59 @@ return function()
     end
 
     _M.run = function(species, generationIdx, genomeCallback, finishCallback)
-        local poppets = {}
+        local hostProcess = "lsnes"
-        for i=1,#species,1 do
+        if util.isWin then
+            hostProcess = util.scrapeCmd('*l', 'powershell "(Get-WmiObject Win32_Process -Filter ProcessId=$((Get-WmiObject Win32_Process -Filter ProcessId=$((Get-WmiObject Win32_Process -Filter ProcessId=$PID).ParentProcessId)).ParentProcessId)").ExecutablePath')
+            if hostProcess == nil or hostProcess == "" then
+                hostProcess = "lsnes-bsnes.exe"
+            end
+        else
+            -- FIXME Linux
+        end
+
+        while #_M.poppets < #species do
+            local i = #_M.poppets+1
             local outputFileName = tmpFileName..'_output_'..i
-
             local inputFileName = tmpFileName.."_input_"..i
-            print(inputFileName)
+
-            local inputFile = io.open(inputFileName, 'w')
+            message(_M, hostProcess)
-            inputFile:write(serpent.dump({species[i], generationIdx, outputFileName}))
+
-            inputFile:close()
-            
-            local proc = "lsnes"
-            if util.isWin then
-                local checkParent = io.popen('powershell "(Get-WmiObject Win32_Process -Filter ProcessId=$((Get-WmiObject Win32_Process -Filter ProcessId=$((Get-WmiObject Win32_Process -Filter ProcessId=$PID).ParentProcessId)).ParentProcessId)").ExecutablePath')
-                proc = checkParent:read("*l")
-                checkParent:close()
-            else
-                -- FIXME Linux
-            end
-            print(proc)
-            local cmd = "\""..proc.."\" \"--rom="..config.ROM.."\" --unpause \"--lua="..base.."/runner-process.lua\""
             local envs = {
-                RUNNER_DATA = inputFileName
+                RUNNER_INPUT_FILE = inputFileName,
+                RUNNER_OUTPUT_FILE = outputFileName,
             }
-            if config.NeatConfig.ThreadDontQuit then
-                envs.RUNNER_DONT_QUIT = "1"
-            end
 
-            local cmdParts = {}
+            local waiter = util.waitForChange(outputFileName)
-            for k,v in pairs(envs) do
+
-                if util.isWin then
+            local cmd = '"'..hostProcess..'" "--rom='..config.ROM..'" --unpause "--lua='..base..'/runner-process.lua"'
-                    table.insert(cmdParts, string.format("set %s=%s &&", k, v))
+            local poppet = util.popenCmd(cmd, nil, envs)
-                else
+            table.insert(_M.poppets, poppet)
-                    table.insert(cmdParts, string.format("%s='%s'", k, v))
+
-                end
+            waiter:read("*a")
-            end
+            util.closeCmd(waiter)
-            table.insert(cmdParts, cmd)
-            local fullCmd = table.concat(cmdParts, " ")
-            message(_M, fullCmd)
-            local poppet = io.popen(fullCmd, 'r')
-            table.insert(poppets, poppet)
         end
 
-        for i=1,#poppets,1 do
+        local outputFileName = tmpFileName..'_output_*'
-            local poppet = poppets[i]
+        local waiter = util.waitForChange(outputFileName, #species)
-            poppet:read('*a')
+
-            poppet:close()
+        message(_M, 'Setting up child processes')
-        end
+
-        
         for i=1,#species,1 do
+            local inputFileName = tmpFileName.."_input_"..i
+            local inputFile = io.open(inputFileName, 'w')
+            inputFile:write(serpent.dump({species[i], generationIdx}))
+            inputFile:close()
+        end
+
+        message(_M, 'Waiting for child processes to finish')
+
+        waiter:read("*a")
+        util.closeCmd(waiter)
+
+        message(_M, 'Child processes finished')
+
+        for i=1,#species,1 do
+            message(_M, "Processing output "..i)
             local outputFileName = tmpFileName..'_output_'..i
             local outputFile = io.open(outputFileName, "r")
             local line = ""

runner.lua

@@ -236,7 +236,6 @@ function displayButtons()
     gui.renderctx.setnull()
 end
 
-local frame = 0
 local formCtx = nil
 local form = nil
 function displayForm(_M)
@@ -244,7 +243,7 @@ function displayForm(_M)
         return
     end
 
-    if form ~= nil and frame % 10 ~= 0 then
+    if form ~= nil and _M.drawFrame % 10 ~= 0 then
         for i=#_M.onRenderFormHandler,1,-1 do
             _M.onRenderFormHandler[i](form)
         end
@@ -299,7 +298,7 @@ local function painting(_M)
     if formCtx == nil then
         formCtx = gui.renderctx.new(500, guiHeight)
     end
-    frame = frame + 1
+    _M.drawFrame = _M.drawFrame + 1
     displayForm(_M)
 end
 
@@ -402,7 +401,7 @@ local function mainLoop(_M, genome)
 
         genome = _M.currentSpecies.genomes[_M.currentGenomeIndex]
 
-        if frame % 10 == 0 then
+        if _M.drawFrame % 10 == 0 then
             if not pcall(function()
                 displayGenome(genome)
             end) then
@@ -433,7 +432,7 @@ local function mainLoop(_M, genome)
         -- Don't punish being launched by barrels
         -- FIXME Will this skew mine cart levels?
         if game.getVelocityY() < -2104 then
-            message(_M, "BARREL! "..frame, 0x00ffff00)
+            message(_M, "BARREL! ".._M.drawFrame, 0x00ffff00)
             if _M.timeout < timeoutConst + 60 * 12 then
                 _M.timeout = _M.timeout + 60 * 12
             end
@@ -567,6 +566,7 @@ local function mainLoop(_M, genome)
         while fitnessAlreadyMeasured(_M) do
             _M.currentGenomeIndex = _M.currentGenomeIndex + 1
             if _M.currentGenomeIndex > #_M.currentSpecies.genomes then
+                game.unregisterHandlers()
                 for i=#_M.dereg,1,-1 do
                     local d = table.remove(_M.dereg, i)
                     callback.unregister(d[1], d[2])
@@ -850,6 +850,7 @@ return function()
         genomeCallback = nil,
         currentGenomeIndex = 1,
         currentFrame = 0,
+        drawFrame = 0,
         maxFitness = 0,
 
         dereg = {},

state-test.lua

@@ -1,20 +1,9 @@
-PARTY_X = 0x7e0a2a
+local errtest = io.popen("farts", 'r')
-TILE_SIZE = 32
+errtest:read("*a")
-
+print(errtest:close())
-print(memory.readword(PARTY_X))
+errtest = io.popen("dir", 'r')
-
+errtest:read("*a")
-function on_post_rewind()
+local ok, exit, code = errtest:close()
-    print("Async?")
+print(ok)
-    print(memory.readword(PARTY_X))
+print(exit)
-end
+print(code)
-
-function movement(addr, val)
-    if memory.readword(addr) > TILE_SIZE * 20 then
-        local rew = movie.to_rewind("pool/PiratePanic.lsmv")
-        movie.unsafe_rewind(rew)
-        print("Sync?")
-        print(memory.readword(PARTY_X))
-    end
-end
-
-memory2.WRAM:registerwrite(0x0a2a, movement)

util.lua

@@ -2,6 +2,93 @@ local _M = {}
 
 _M.isWin = package.config:sub(1, 1) == '\\'
 
+--- Echo a command, run it, and return the file handle
+--- @param cmd string The command to execute
+--- @param workdir string The working directory
+--- @param env table The environment variables
+function _M.popenCmd(cmd, workdir, env)
+    local cmdParts = {}
+    if workdir ~= nil then
+        if _M.isWin then
+            table.insert(cmdParts, 'cd /d "'..workdir..'" &&')
+        else
+            table.insert(cmdParts, 'cd "'..workdir..'" &&')
+        end
+    end
+    if env ~= nil then
+        for k,v in pairs(env) do
+            if _M.isWin then
+                table.insert(cmdParts, string.format("set %s=%s&&", k, v))
+            else
+                table.insert(cmdParts, string.format("%s='%s'", k, v))
+            end
+        end
+    end
+    table.insert(cmdParts, cmd)
+    local fullCmd = table.concat(cmdParts, " ")
+    print(fullCmd)
+
+--[[     local dummy = "/dev/null"
+    if isWin then
+        dummy = "NUL"
+    end
+    return io.open(dummy, 'r') ]]
+    return io.popen(fullCmd, 'r')
+end
+
+--- Echo a command, run it, and handle any errors
+--- @return string string The stdout
+function _M.doCmd(...)
+    return _M.scrapeCmd('*a', ...)
+end
+
+--- Run a command and get the output
+--- @param formats table|string|number List or single io.read() specifier
+--- @return table table List of results based on read specifiers
+function _M.scrapeCmd(formats, ...)
+    local poppet = _M.popenCmd(...)
+    local outputs = nil
+    if type(formats) ~= 'table' then
+        outputs = poppet:read(formats)
+    else
+        outputs = {}
+        for i=1,#formats,1 do
+            table.insert(outputs, poppet:read(formats[i]))
+        end
+    end
+    _M.closeCmd(poppet)
+    return outputs
+end
+
+--- Check the command's exit code and throw a Lua error if it isn't right
+--- @param handle file* The handle of the command
+function _M.closeCmd(handle)
+    local ok, state, code = handle:close()
+    if state ~= "exit" then
+        return
+    end
+    if code ~= 0 then
+        error("The last command failed")
+    end
+end
+
+function _M.waitForChange(filename, count)
+    if count == nil then
+        count = 1
+    end
+
+    if _M.isWin then
+        local sec, usec = utime()
+        print(string.format('Starting watching file at %d', sec * 1000000 + usec))
+
+        return _M.popenCmd([[powershell "$filename = ']]..filename..
+        [[' ; $targetCount = ]]..count..[[ ; $count = 0 ; Register-ObjectEvent (New-Object IO.FileSystemWatcher (Split-Path $filename), (Split-Path -Leaf $filename) -Property @{ IncludeSubdirectories = $false ; NotifyFilter =  [IO.NotifyFilters]'FileName, LastWrite'}) -EventName Changed -SourceIdentifier RunnerDataChanged -Action { $count += 1 ; if ( $count -ge $targetCount ) { [Environment]::Exit(0) } } ; while($true) { Start-Sleep -Milliseconds 1 }"]])
+    else
+        error("Not implemented")
+        -- FIXME Linux
+    end
+end
+
 function _M.table_to_string(tbl)
     local result = "{"
     local keys = {}