diff --git a/include/core/rom.hpp b/include/core/rom.hpp index 553fe7df..d89edc00 100644 --- a/include/core/rom.hpp +++ b/include/core/rom.hpp @@ -7,90 +7,7 @@ #include #include "core/misc.hpp" #include "interface/romtype.hpp" - -/** - * Some loaded data or indication of no data. - */ -struct loaded_slot -{ -/** - * Construct empty slot. - * - * throws std::bad_alloc: Not enough memory. - */ - loaded_slot() throw(std::bad_alloc); - -/** - * This constructor construct slot by reading data from file. If filename is "", constructs an empty slot. - * - * parameter filename: The filename to read. If "", empty slot is constructed. - * parameter base: Base filename to interpret the filename against. If "", no base filename is used. - * parameter imginfo: Image information. - * parameter xml_flag: If set, always keep trailing NUL. - * throws std::bad_alloc: Not enough memory. - * throws std::runtime_error: Can't load the data. - */ - loaded_slot(const std::string& filename, const std::string& base, const struct core_romimage_info& imginfo, - bool xml_flag = false) throw(std::bad_alloc, std::runtime_error); - -/** - * This method patches this slot using specified IPS patch. - * - * parameter patch: The patch to apply - * parameter offset: The amount to add to the offsets in the IPS file. Parts with offsets below zero are not patched. - * throws std::bad_alloc: Not enough memory. - * throws std::runtime_error: Bad IPS patch. - */ - void patch(const std::vector& patch, int32_t offset) throw(std::bad_alloc, std::runtime_error); -/** - * Is this filename? - */ - bool filename_flag; -/** - * Is this slot XML slot? - */ - bool xml; -/** - * If this slot is blank, this is set to false, data is undefined and sha256 is "". Otherwise this is set to true, - * data to apporiate data, and sha256 to hash of data. - */ - bool valid; -/** - * The actual data for this slot. - */ - std::vector data; -/** - * SHA-256 for the data in this slot if data is valid. If no valid data, this field is "". - */ - std::string sha_256; -/** - * Get pointer to loaded data - * - * returns: Pointer to loaded data, or NULL if slot is blank. - */ - operator const char*() const throw() - { - return valid ? reinterpret_cast(&data[0]) : NULL; - } -/** - * Get pointer to loaded data - * - * returns: Pointer to loaded data, or NULL if slot is blank. - */ - operator const uint8_t*() const throw() - { - return valid ? reinterpret_cast(&data[0]) : NULL; - } -/** - * Get size of slot - * - * returns: The number of bytes in slot, or 0 if slot is blank. - */ - operator unsigned() const throw() - { - return valid ? data.size() : 0; - } -}; +#include "library/fileimage.hpp" /** * ROM loaded into memory. @@ -135,11 +52,11 @@ struct loaded_rom /** * Loaded main ROM */ - loaded_slot romimg[27]; + loaded_image romimg[27]; /** * Loaded main ROM XML */ - loaded_slot romxml[27]; + loaded_image romxml[27]; /** * MSU-1 base. */ @@ -192,6 +109,11 @@ std::pair get_current_rom_info() throw(); std::map> load_sram_commandline(const std::vector& cmdline) throw(std::bad_alloc, std::runtime_error); +/** + * Set the hasher callback. + */ +void set_hasher_callback(std::function cb); + //Map of preferred cores for each extension and type. extern std::map preferred_core; //Preferred overall core. diff --git a/include/library/fileimage.hpp b/include/library/fileimage.hpp new file mode 100644 index 00000000..acaa9601 --- /dev/null +++ b/include/library/fileimage.hpp @@ -0,0 +1,225 @@ +#ifndef _library__fileimage__hpp__included__ +#define _library__fileimage__hpp__included__ + +#include +#include +#include +#include +#include "threadtypes.hpp" + +class sha256_hasher; + +/** + * Future for SHA-256 computation. + */ +class sha256_future +{ +public: +/** + * Construct a null future, never resolves. + */ + sha256_future(); +/** + * Construct a future, with value that is immediately resolved. + */ + sha256_future(const std::string& value); +/** + * Is the result known? + */ + bool ready() const; +/** + * Read the result (or throw error). Waits until result is ready. + */ + std::string read() const; +/** + * Copy a future. + */ + sha256_future(const sha256_future& f); +/** + * Assign a future. + */ + sha256_future& operator=(const sha256_future& f); +/** + * Destroy a future. + */ + ~sha256_future(); +private: +/** + * Create a new future. + */ + sha256_future(sha256_hasher& h, unsigned id); +/** + * Resolve a future. + */ + void resolve(unsigned id, const std::string& hash); + void resolve_error(unsigned id, const std::string& err); + + friend class sha256_hasher; + mutable mutex_class mutex; + mutable cv_class condition; + bool is_ready; + unsigned cbid; + std::string value; + std::string error; + sha256_future* prev; + sha256_future* next; + sha256_hasher* hasher; +}; + +/** + * Class performing SHA-256 hashing. + */ +class sha256_hasher +{ +public: +/** + * Create a new SHA-256 hasher. + */ + sha256_hasher(); +/** + * Destroy a SHA-256 hasher. Causes all current jobs to fail. + */ + ~sha256_hasher(); +/** + * Set callback. + */ + void set_callback(std::function cb); +/** + * Compute SHA-256 of file. + */ + sha256_future operator()(const std::string& filename); +/** + * Thread entrypoint. + */ + void entrypoint(); +private: + void link(sha256_future& future); + void unlink(sha256_future& future); + void send_callback(uint64_t this_completed); + void send_idle(); + + friend class sha256_future; + struct queue_job + { + std::string filename; + uint64_t size; + unsigned cbid; + volatile unsigned interested; + }; + sha256_hasher(const sha256_hasher&); + sha256_hasher& operator=(const sha256_hasher&); + thread_class* hash_thread; + mutex_class mutex; + cv_class condition; + std::list queue; + std::list::iterator current_job; + sha256_future* first_future; + sha256_future* last_future; + unsigned next_cbid; + std::function progresscb; + bool quitting; + uint64_t total_work; +}; + +/** + * Some loaded data or indication of no data. + */ +struct loaded_image +{ +/** + * Information about image to load. + */ + struct info + { + enum _type + { + IT_NONE, //Only used in type field of image. + IT_MEMORY, + IT_MARKUP, + IT_FILE + }; + _type type; + unsigned headersize; + }; +/** + * Construct empty image. + * + * throws std::bad_alloc: Not enough memory. + */ + loaded_image() throw(std::bad_alloc); + +/** + * This constructor construct slot by reading data from file. If filename is "", constructs an empty slot. + * + * parameter hasher: Hasher to use. + * parameter filename: The filename to read. If "", empty slot is constructed. + * parameter base: Base filename to interpret the filename against. If "", no base filename is used. + * parameter imginfo: Image information. + * throws std::bad_alloc: Not enough memory. + * throws std::runtime_error: Can't load the data. + */ + loaded_image(sha256_hasher& hasher, const std::string& filename, const std::string& base, + const struct info& imginfo) throw(std::bad_alloc, std::runtime_error); + +/** + * This method patches this slot using specified IPS patch. + * + * parameter patch: The patch to apply + * parameter offset: The amount to add to the offsets in the IPS file. Parts with offsets below zero are not patched. + * throws std::bad_alloc: Not enough memory. + * throws std::runtime_error: Bad IPS patch, or trying to patch file image. + */ + void patch(const std::vector& patch, int32_t offset) throw(std::bad_alloc, std::runtime_error); +/** + * Type. + */ + info::_type type; +/** + * Filename this is loaded from. + */ + std::string filename; +/** + * If this slot is blank, this is set to false, data is undefined and sha256 is "". Otherwise this is set to true, + * data to apporiate data, and sha256 to hash of data. + */ + bool valid; +/** + * The actual data for this slot. + */ + std::vector data; +/** + * SHA-256 for the data in this slot if data is valid. If no valid data, this field is "". + * + * Note, for file images, this takes a bit of time to fill. + */ + sha256_future sha_256; +/** + * Get pointer to loaded data + * + * returns: Pointer to loaded data, or NULL if slot is blank. + */ + operator const char*() const throw() + { + return valid ? reinterpret_cast(&data[0]) : NULL; + } +/** + * Get pointer to loaded data + * + * returns: Pointer to loaded data, or NULL if slot is blank. + */ + operator const uint8_t*() const throw() + { + return valid ? reinterpret_cast(&data[0]) : NULL; + } +/** + * Get size of slot + * + * returns: The number of bytes in slot, or 0 if slot is blank. + */ + operator unsigned() const throw() + { + return valid ? data.size() : 0; + } +}; + +#endif diff --git a/src/core/mainloop.cpp b/src/core/mainloop.cpp index 29580c8c..a9c0251a 100644 --- a/src/core/mainloop.cpp +++ b/src/core/mainloop.cpp @@ -346,8 +346,8 @@ namespace *our_rom = newrom; } for(size_t i = 0; i < sizeof(our_rom->romimg)/sizeof(our_rom->romimg[0]); i++) { - our_movie.romimg_sha256[i] = our_rom->romimg[i].sha_256; - our_movie.romxml_sha256[i] = our_rom->romxml[i].sha_256; + our_movie.romimg_sha256[i] = our_rom->romimg[i].sha_256.read(); + our_movie.romxml_sha256[i] = our_rom->romxml[i].sha_256.read(); } } catch(std::exception& e) { platform::error_message(std::string("Can't load ROM: ") + e.what()); diff --git a/src/core/misc.cpp b/src/core/misc.cpp index f7652875..fb572dbd 100644 --- a/src/core/misc.cpp +++ b/src/core/misc.cpp @@ -153,8 +153,8 @@ struct loaded_rom load_rom_from_commandline(std::vector cmdline) th (r.rtype->get_image_info(i).hname + " ROM"); xmlname = r.rtype->get_image_info(i).hname + " XML"; } - if(r.romimg[i].valid) messages << romname << " hash: " << r.romimg[i].sha_256 << std::endl; - if(r.romxml[i].valid) messages << xmlname << " hash: " << r.romxml[i].sha_256 << std::endl; + if(r.romimg[i].valid) messages << romname << " hash: " << r.romimg[i].sha_256.read() << std::endl; + if(r.romxml[i].valid) messages << xmlname << " hash: " << r.romxml[i].sha_256.read() << std::endl; } return r; } diff --git a/src/core/moviedata.cpp b/src/core/moviedata.cpp index 045bf4fc..3ce1789c 100644 --- a/src/core/moviedata.cpp +++ b/src/core/moviedata.cpp @@ -85,21 +85,21 @@ namespace messages << "Saved core state to " << name << std::endl; }); - bool warn_hash_mismatch(const std::string& mhash, const loaded_slot& slot, + bool warn_hash_mismatch(const std::string& mhash, const loaded_image& slot, const std::string& name, bool fatal) { - if(mhash == slot.sha_256) + if(mhash == slot.sha_256.read()) return true; if(!fatal) { messages << "WARNING: " << name << " hash mismatch!" << std::endl << "\tMovie: " << mhash << std::endl - << "\tOur ROM: " << slot.sha_256 << std::endl; + << "\tOur ROM: " << slot.sha_256.read() << std::endl; return true; } else { platform::error_message("Can't load state because hashes mismatch"); messages << "ERROR: " << name << " hash mismatch!" << std::endl << "\tMovie: " << mhash << std::endl - << "\tOur ROM: " << slot.sha_256 << std::endl; + << "\tOur ROM: " << slot.sha_256.read() << std::endl; return false; } } @@ -211,8 +211,8 @@ void do_save_state(const std::string& filename, int binary) throw(std::bad_alloc our_movie.is_savestate = true; our_movie.sram = our_rom->rtype->save_sram(); for(size_t i = 0; i < sizeof(our_rom->romimg)/sizeof(our_rom->romimg[0]); i++) { - our_movie.romimg_sha256[i] = our_rom->romimg[i].sha_256; - our_movie.romxml_sha256[i] = our_rom->romxml[i].sha_256; + our_movie.romimg_sha256[i] = our_rom->romimg[i].sha_256.read(); + our_movie.romxml_sha256[i] = our_rom->romxml[i].sha_256.read(); } our_movie.savestate = our_rom->save_core_state(); get_framebuffer().save(our_movie.screenshot); diff --git a/src/core/rom.cpp b/src/core/rom.cpp index 6d790ced..c04212f1 100644 --- a/src/core/rom.cpp +++ b/src/core/rom.cpp @@ -190,92 +190,27 @@ namespace return false; } } + + struct loaded_image::info get_xml_info() + { + loaded_image::info i; + i.type = loaded_image::info::IT_MARKUP; + i.headersize = 0; + return i; + } + + struct loaded_image::info xlate_info(core_romimage_info ri) + { + loaded_image::info i; + if(ri.pass_mode == 0) i.type = loaded_image::info::IT_MEMORY; + if(ri.pass_mode == 1) i.type = loaded_image::info::IT_FILE; + i.headersize = ri.headersize; + return i; + } + + sha256_hasher lsnes_image_hasher; } -loaded_slot::loaded_slot() throw(std::bad_alloc) -{ - valid = false; - xml = false; - sha_256 = ""; - filename_flag = false; -} - -loaded_slot::loaded_slot(const std::string& filename, const std::string& base, - const struct core_romimage_info& imginfo, bool xml_flag) throw(std::bad_alloc, std::runtime_error) -{ - unsigned headered = 0; - xml = xml_flag; - if(filename == "") { - valid = false; - sha_256 = ""; - filename_flag = (!xml && imginfo.pass_mode); - return; - } - //XMLs are always loaded, no matter what. - if(!xml && imginfo.pass_mode) { - std::string _filename = filename; - //Translate the passed filename to absolute one. - _filename = resolve_file_relative(_filename, base); - _filename = boost_fs::absolute(boost_fs::path(_filename)).string(); - filename_flag = true; - data.resize(_filename.length()); - std::copy(_filename.begin(), _filename.end(), data.begin()); - //Compute the SHA-256. - std::istream& s = open_file_relative(filename, ""); - sha256 hash; - char buffer[8192]; - size_t block; - while((block = s.readsome(buffer, 8192))) - hash.write(buffer, block); - sha_256 = hash.read(); - delete &s; - valid = true; - return; - } - filename_flag = false; - valid = true; - data = read_file_relative(filename, base); - if(!xml && imginfo.headersize) - headered = ((data.size() % (2 * imginfo.headersize)) == imginfo.headersize) ? imginfo.headersize : 0; - if(headered && !xml) { - if(data.size() >= headered) { - memmove(&data[0], &data[headered], data.size() - headered); - data.resize(data.size() - headered); - } else { - data.resize(0); - } - } - sha_256 = sha256::hash(data); - if(xml) { - size_t osize = data.size(); - data.resize(osize + 1); - data[osize] = 0; - } -} - -void loaded_slot::patch(const std::vector& patch, int32_t offset) throw(std::bad_alloc, std::runtime_error) -{ - if(filename_flag) - throw std::runtime_error("CD images can't be patched on the fly"); - try { - std::vector data2 = data; - if(xml && valid) - data2.resize(data2.size() - 1); - data2 = do_patch_file(data2, patch, offset); - //Mark the slot as valid and update hash. - valid = true; - std::string new_sha256 = sha256::hash(data2); - if(xml) { - size_t osize = data2.size(); - data2.resize(osize + 1); - data2[osize] = 0; - } - data = data2; - sha_256 = new_sha256; - } catch(...) { - throw; - } -} std::pair get_current_rom_info() throw() { @@ -298,13 +233,13 @@ loaded_rom::loaded_rom(const std::string& file, core_type& ctype) throw(std::bad //This thing has a BIOS. romidx = 1; std::string basename = lsnes_vset["firmwarepath"].str() + "/" + bios; - romimg[0] = loaded_slot(basename, "", ctype.get_image_info(0), false); + romimg[0] = loaded_image(lsnes_image_hasher, basename, "", xlate_info(ctype.get_image_info(0))); if(file_exists(basename + ".xml")) - romxml[0] = loaded_slot(basename + ".xml", "", ctype.get_image_info(0), true); + romxml[0] = loaded_image(lsnes_image_hasher, basename + ".xml", "", get_xml_info()); } - romimg[romidx] = loaded_slot(file, "", ctype.get_image_info(romidx), false); + romimg[romidx] = loaded_image(lsnes_image_hasher, file, "", xlate_info(ctype.get_image_info(romidx))); if(file_exists(file + ".xml")) - romxml[romidx] = loaded_slot(file + ".xml", "", ctype.get_image_info(romidx), true); + romxml[romidx] = loaded_image(lsnes_image_hasher, file + ".xml", "", get_xml_info()); load_filename = file; msu1_base = resolve_file_relative(file, ""); return; @@ -333,13 +268,15 @@ loaded_rom::loaded_rom(const std::string& file, const std::string& tmpprefer) th //This thing has a BIOS. romidx = 1; std::string basename = lsnes_vset["firmwarepath"].str() + "/" + bios; - romimg[0] = loaded_slot(basename, "", coretype->get_image_info(0), false); + romimg[0] = loaded_image(lsnes_image_hasher, basename, "", + xlate_info(coretype->get_image_info(0))); if(file_exists(basename + ".xml")) - romxml[0] = loaded_slot(basename + ".xml", "", coretype->get_image_info(0), true); + romxml[0] = loaded_image(lsnes_image_hasher, basename + ".xml", "", get_xml_info()); } - romimg[romidx] = loaded_slot(file, "", coretype->get_image_info(romidx), false); + romimg[romidx] = loaded_image(lsnes_image_hasher, file, "", + xlate_info(coretype->get_image_info(romidx))); if(file_exists(file + ".xml")) - romxml[romidx] = loaded_slot(file + ".xml", "", coretype->get_image_info(romidx), true); + romxml[romidx] = loaded_image(lsnes_image_hasher, file + ".xml", "", get_xml_info()); msu1_base = resolve_file_relative(file, ""); return; } @@ -410,8 +347,8 @@ loaded_rom::loaded_rom(const std::string& file, const std::string& tmpprefer) th //Load ROMs. for(size_t i = 0; i < rtype->get_image_count(); i++) { - romimg[i] = loaded_slot(cromimg[i], file, rtype->get_image_info(i), false); - romxml[i] = loaded_slot(cromxml[i], file, rtype->get_image_info(i), true); + romimg[i] = loaded_image(lsnes_image_hasher, cromimg[i], file, xlate_info(rtype->get_image_info(i))); + romxml[i] = loaded_image(lsnes_image_hasher, cromxml[i], file, get_xml_info()); } //Patch ROMs. @@ -544,7 +481,12 @@ void loaded_rom::load_core_state(const std::vector& buf, bool nochecksum) if(memcmp(tmp, &buf[buf.size() - 32], 32)) throw std::runtime_error("Savestate corrupt"); } - rtype->unserialize(&buf[0], buf.size() - 32);; + rtype->unserialize(&buf[0], buf.size() - 32); +} + +void set_hasher_callback(std::function cb) +{ + lsnes_image_hasher.set_callback(cb); } std::map preferred_core; diff --git a/src/library/fileimage.cpp b/src/library/fileimage.cpp new file mode 100644 index 00000000..a3270158 --- /dev/null +++ b/src/library/fileimage.cpp @@ -0,0 +1,464 @@ +#include "fileimage.hpp" +#include "sha256.hpp" +#include "patch.hpp" +#include "zip.hpp" +#include +#include + +#ifdef BOOST_FILESYSTEM3 +namespace boost_fs = boost::filesystem3; +#else +namespace boost_fs = boost::filesystem; +#endif + +namespace +{ + std::map> cached_entries; + + std::mutex& global_queue_mutex() + { + static std::mutex m; + return m; + } + + void* thread_trampoline(sha256_hasher* h) + { + h->entrypoint(); + return NULL; + } + + std::string lookup_cache(const std::string& filename) + { + std::string cache = filename + ".sha256"; + time_t filetime = boost_fs::last_write_time(boost_fs::path(filename)); + if(cached_entries.count(cache)) { + //Found the cache entry... + if(cached_entries[cache].first == filetime) + return cached_entries[cache].second; + else { + //Stale. + unlink(cache.c_str()); + cached_entries.erase(cache); + return ""; + } + } + + std::string cached_hash; + time_t rfiletime; + + std::ifstream in(cache); + if(!in) + return ""; //Failed. + + std::string tmp; + std::getline(in, tmp); + std::istringstream _in(tmp); + _in >> rfiletime; + std::getline(in, cached_hash); + + if(rfiletime == filetime) { + cached_entries[cache] = std::make_pair(rfiletime, cached_hash); + } else { + //Stale. + unlink(cache.c_str()); + cached_entries.erase(cache); + return ""; + } + return cached_hash; + } + + void store_cache(const std::string& filename, const std::string& value) + { + std::string cache = filename + ".sha256"; + time_t filetime = boost_fs::last_write_time(boost_fs::path(filename)); + std::ofstream out(cache); + cached_entries[cache] = std::make_pair(filetime, value); + if(!out) + return; //Failed! + out << filetime << std::endl; + out << value << std::endl; + out.close(); + } + + uint64_t get_file_size(const std::string& filename) + { + uintmax_t size = boost_fs::file_size(boost_fs::path(filename)); + if(size == static_cast(-1)) + return 0; + return size; + } +} + +sha256_future::sha256_future() +{ + is_ready = false; + cbid = 0; + prev = next = NULL; + hasher = NULL; +} + +sha256_future::sha256_future(const std::string& _value) +{ + is_ready = true; + value = _value; + cbid = 0; + prev = next = NULL; + hasher = NULL; +} + +sha256_future::sha256_future(sha256_hasher& h, unsigned id) +{ + umutex_class h2(global_queue_mutex()); + is_ready = false; + cbid = id; + prev = next = NULL; + hasher = &h; + hasher->link(*this); +} + +sha256_future::~sha256_future() +{ + umutex_class h2(global_queue_mutex()); + umutex_class h(mutex); + if(hasher) + hasher->unlink(*this); +} + +bool sha256_future::ready() const +{ + umutex_class h(mutex); + return is_ready; +} + +std::string sha256_future::read() const +{ + umutex_class h(mutex); + while(!is_ready) + condition.wait(h); + if(error != "") + throw std::runtime_error(error); + return value; +} + +sha256_future::sha256_future(const sha256_future& f) +{ + umutex_class h2(global_queue_mutex()); + umutex_class h(f.mutex); + is_ready = f.is_ready; + cbid = f.cbid; + value = f.value; + error = f.error; + prev = next = NULL; + hasher = f.hasher; + if(!is_ready && hasher) + hasher->link(*this); +} + +sha256_future& sha256_future::operator=(const sha256_future& f) +{ + if(this == &f) + return *this; + umutex_class h2(global_queue_mutex()); + if((size_t)this < (size_t)&f) { + mutex.lock(); + f.mutex.lock(); + } else { + f.mutex.lock(); + mutex.lock(); + } + + if(!is_ready && hasher) + hasher->unlink(*this); + is_ready = f.is_ready; + cbid = f.cbid; + value = f.value; + error = f.error; + prev = next = NULL; + hasher = f.hasher; + if(!is_ready && hasher) + hasher->link(*this); + mutex.unlock(); + f.mutex.unlock(); +} + +void sha256_future::resolve(unsigned id, const std::string& hash) +{ + umutex_class h(mutex); + hasher->unlink(*this); + if(id != cbid) + return; + is_ready = true; + value = hash; + condition.notify_all(); +} + +void sha256_future::resolve_error(unsigned id, const std::string& err) +{ + umutex_class h(mutex); + hasher->unlink(*this); + if(id != cbid) + return; + is_ready = true; + error = err; + condition.notify_all(); +} + +void sha256_hasher::link(sha256_future& future) +{ + //We assume caller holds global queue lock. + { + umutex_class h(mutex); + unsigned cbid = future.cbid; + for(auto& i : queue) + if(i.cbid == cbid) + i.interested--; + } + future.prev = last_future; + future.next = NULL; + if(last_future) + last_future->next = &future; + last_future = &future; + if(!first_future) + first_future = &future; +} + +void sha256_hasher::unlink(sha256_future& future) +{ + //We assume caller holds global queue lock. + { + umutex_class h(mutex); + unsigned cbid = future.cbid; + for(auto& i : queue) + if(i.cbid == cbid) + i.interested++; + } + if(&future == first_future) + first_future = future.next; + if(&future == last_future) + last_future = future.prev; + if(future.prev) + future.prev->next = future.next; + if(future.next) + future.next->prev = future.prev; +} + +sha256_future sha256_hasher::operator()(const std::string& filename) +{ + queue_job j; + j.filename = filename; + j.size = get_file_size(filename); + j.cbid = next_cbid++; + j.interested = 1; + sha256_future future(*this, j.cbid); + queue.push_back(j); + umutex_class h(mutex); + total_work += j.size; + condition.notify_all(); + return future; +} + +void sha256_hasher::set_callback(std::function cb) +{ + umutex_class h(mutex); + progresscb = cb; +} + +sha256_hasher::sha256_hasher() +{ + quitting = false; + first_future = NULL; + last_future = NULL; + next_cbid = 0; + total_work = 0; + progresscb = [](uint64_t x) -> void {}; + hash_thread = new thread_class(thread_trampoline, this); +} + +sha256_hasher::~sha256_hasher() +{ + { + umutex_class h(mutex); + quitting = true; + condition.notify_all(); + } + hash_thread->join(); + delete hash_thread; + umutex_class h2(global_queue_mutex()); + while(first_future) + first_future->resolve_error(first_future->cbid, "Hasher deleted"); +} + +void sha256_hasher::entrypoint() +{ + FILE* fp; + while(true) { + //Wait for work or quit signal. + { + umutex_class h(mutex); + while(!quitting && queue.empty()) { + send_idle(); + condition.wait(h); + } + if(quitting) + return; + //We hawe work. + current_job = queue.begin(); + } + + //Hash this item. + uint64_t progress = 0; + std::string cached_hash; + fp = NULL; + cached_hash = lookup_cache(current_job->filename); + if(cached_hash != "") { + umutex_class h2(global_queue_mutex()); + for(sha256_future* fut = first_future; fut != NULL; fut = fut->next) + fut->resolve(current_job->cbid, cached_hash); + goto finished; + } + fp = fopen(current_job->filename.c_str(), "rb"); + if(!fp) { + umutex_class h2(global_queue_mutex()); + for(sha256_future* fut = first_future; fut != NULL; fut = fut->next) + fut->resolve_error(current_job->cbid, "Can't open file"); + } else { + sha256 hash; + while(!feof(fp) && !ferror(fp)) { + { + umutex_class h(mutex); + if(!current_job->interested) + goto finished; //Aborted. + } + unsigned char buf[16384]; + size_t s = fread(buf, 1, sizeof(buf), fp); + progress += s; + hash.write(buf, s); + send_callback(progress); + } + if(ferror(fp)) { + umutex_class h2(global_queue_mutex()); + for(sha256_future* fut = first_future; fut != NULL; fut = fut->next) + fut->resolve_error(current_job->cbid, "Can't read file"); + } else { + std::string hval = hash.read(); + umutex_class h2(global_queue_mutex()); + for(sha256_future* fut = first_future; fut != NULL; fut = fut->next) + fut->resolve(current_job->cbid, hval); + store_cache(current_job->filename, hval); + } + } +finished: + if(fp) fclose(fp); + //Okay, this work item is complete. + { + umutex_class h(mutex); + total_work -= current_job->size; + queue.erase(current_job); + } + send_callback(0); + } +} + +void sha256_hasher::send_callback(uint64_t this_completed) +{ + uint64_t amount; + { + umutex_class h(mutex); + if(this_completed > total_work) + amount = 0; + else + amount = total_work - this_completed; + } + progresscb(amount); +} + +void sha256_hasher::send_idle() +{ + progresscb(0xFFFFFFFFFFFFFFFFULL); +} + +loaded_image::loaded_image() throw(std::bad_alloc) +{ + type = info::IT_NONE; + valid = false; + sha_256 = sha256_future(""); + filename = ""; +} + +loaded_image::loaded_image(sha256_hasher& h, const std::string& _filename, const std::string& base, + const struct loaded_image::info& info) throw(std::bad_alloc, std::runtime_error) +{ + if(info.type == info::IT_NONE && _filename != "") + throw std::runtime_error("Tried to load NULL image"); + if(_filename == "") { + //NULL. + type = info::IT_NONE; + valid = false; + sha_256 = sha256_future(""); + return; + } + + //Load markups and memory images. + if(info.type == info::IT_MEMORY || info.type == info::IT_MARKUP) { + unsigned headered = 0; + filename = resolve_file_relative(_filename, base); + type = info.type; + data = read_file_relative(_filename, base); + valid = true; + if(info.type == info::IT_MEMORY && info.headersize) + headered = ((data.size() % (2 * info.headersize)) == info.headersize) ? info.headersize : 0; + if(data.size() >= headered) { + if(headered) { + memmove(&data[0], &data[headered], data.size() - headered); + data.resize(data.size() - headered); + } + } else { + data.resize(0); + } + sha_256 = sha256_future(sha256::hash(data)); + if(info.type == info::IT_MARKUP) { + size_t osize = data.size(); + data.resize(osize + 1); + data[osize] = 0; + } + return; + } + + if(info.type == info::IT_FILE) { + filename = resolve_file_relative(_filename, base); + filename = boost_fs::absolute(boost_fs::path(filename)).string(); + type = info::IT_FILE; + data.resize(filename.length()); + std::copy(filename.begin(), filename.end(), data.begin()); + valid = true; + sha_256 = h(filename); + return; + } + throw std::runtime_error("Unknown image type"); +} + +void loaded_image::patch(const std::vector& patch, int32_t offset) throw(std::bad_alloc, std::runtime_error) +{ + if(type == info::IT_NONE) + throw std::runtime_error("Not an image"); + if(type != info::IT_MEMORY && type != info::IT_MARKUP) + throw std::runtime_error("File images can't be patched on the fly"); + try { + std::vector data2 = data; + if(type == info::IT_MARKUP) + data2.resize(data2.size() - 1); + data2 = do_patch_file(data2, patch, offset); + //Mark the slot as valid and update hash. + valid = true; + std::string new_sha256 = sha256::hash(data2); + if(type == info::IT_MARKUP) { + size_t osize = data2.size(); + data2.resize(osize + 1); + data2[osize] = 0; + } + data = data2; + sha_256 = sha256_future(new_sha256); + } catch(...) { + throw; + } +} diff --git a/src/platform/wxwidgets/main.cpp b/src/platform/wxwidgets/main.cpp index c034de47..872d0021 100644 --- a/src/platform/wxwidgets/main.cpp +++ b/src/platform/wxwidgets/main.cpp @@ -478,8 +478,8 @@ bool lsnes_app::OnInit() //Initialize the remainder. mov->rerecords = "0"; for(size_t i = 0; i < sizeof(rom->romimg)/sizeof(rom->romimg[0]); i++) { - mov->romimg_sha256[i] = rom->romimg[i].sha_256; - mov->romxml_sha256[i] = rom->romxml[i].sha_256; + mov->romimg_sha256[i] = rom->romimg[i].sha_256.read(); + mov->romxml_sha256[i] = rom->romxml[i].sha_256.read(); } } mov->gametype = &rom->rtype->combine_region(*rom->region); diff --git a/src/platform/wxwidgets/mainwindow.cpp b/src/platform/wxwidgets/mainwindow.cpp index 83f6ba16..cc18dc7f 100644 --- a/src/platform/wxwidgets/mainwindow.cpp +++ b/src/platform/wxwidgets/mainwindow.cpp @@ -144,8 +144,29 @@ namespace bool old_rotate = false; bool main_window_dirty; bool is_fs = false; + bool hashing_in_progress = false; + uint64_t hashing_left = 0; + int64_t last_update = 0; thread_class* emulation_thread; + void hash_callback(uint64_t left) + { + wxwin_mainwindow* mwin = main_window; + if(left == 0xFFFFFFFFFFFFFFFFULL) { + hashing_in_progress = false; + runuifun([mwin]() { if(mwin) mwin->notify_update_status(); }); + last_update = get_utime() - 2000000; + return; + } + hashing_in_progress = true; + hashing_left = left; + uint64_t this_update = get_utime(); + if(this_update < last_update - 1000000 || this_update > last_update + 1000000) { + runuifun([mwin]() { if(mwin) mwin->notify_update_status(); }); + last_update = this_update; + } + } + std::pair lsplit(std::string l) { for(unsigned i = 0; i < l.length() - 3; i++) @@ -982,6 +1003,7 @@ wxwin_mainwindow::wxwin_mainwindow() }); gpanel->SetDropTarget(new loadfile(this)); spanel->SetDropTarget(new loadfile(this)); + set_hasher_callback(hash_callback); } void wxwin_mainwindow::request_paint() @@ -1034,6 +1056,12 @@ std::u32string read_variable_map(const std::map& va void wxwin_mainwindow::update_statusbar(const std::map& vars) { + if(hashing_in_progress) { + std::ostringstream s; + s << "Hashing ROMs, approximately " << ((hashing_left + 524288) >> 20) << "MB left..."; + statusbar->SetStatusText(towxstring(s.str())); + return; + } if(vars.empty()) return; try { diff --git a/src/platform/wxwidgets/romselect.cpp b/src/platform/wxwidgets/romselect.cpp index d4e4ab4b..582281c5 100644 --- a/src/platform/wxwidgets/romselect.cpp +++ b/src/platform/wxwidgets/romselect.cpp @@ -608,8 +608,8 @@ struct moviefile wxwin_project::make_movie() set_mprefix_for_project(f.projectid, tostdstring(prefix->GetValue())); f.rerecords = "0"; for(size_t i = 0; i < sizeof(our_rom->romimg)/sizeof(our_rom->romimg[0]); i++) { - f.romimg_sha256[i] = our_rom->romimg[i].sha_256; - f.romxml_sha256[i] = our_rom->romxml[i].sha_256; + f.romimg_sha256[i] = our_rom->romimg[i].sha_256.read(); + f.romxml_sha256[i] = our_rom->romxml[i].sha_256.read(); } size_t lines = authors->GetNumberOfLines(); for(size_t i = 0; i < lines; i++) { diff --git a/src/util/lsnes-dumpavi.cpp b/src/util/lsnes-dumpavi.cpp index d9ffad28..d54620ac 100644 --- a/src/util/lsnes-dumpavi.cpp +++ b/src/util/lsnes-dumpavi.cpp @@ -24,6 +24,31 @@ namespace { + bool hashing_in_progress = false; + uint64_t hashing_left = 0; + int64_t last_update = 0; + + void hash_callback(uint64_t left) + { + if(left == 0xFFFFFFFFFFFFFFFFULL) { + hashing_in_progress = false; + std::cout << "Done." << std::endl; + last_update = get_utime() - 2000000; + return; + } + if(!hashing_in_progress) { + std::cout << "Hashing disc images..." << std::flush; + } + hashing_in_progress = true; + hashing_left = left; + uint64_t this_update = get_utime(); + if(this_update < last_update - 1000000 || this_update > last_update + 1000000) { + std::cout << ((hashing_left + 524288) >> 20) << "..." << std::flush; + last_update = this_update; + } + } + + class myavsnoop : public information_dispatch { public: @@ -295,6 +320,8 @@ int main(int argc, char** argv) } } + set_hasher_callback(hash_callback); + messages << "--- Loading ROM ---" << std::endl; struct loaded_rom r; try { @@ -312,8 +339,6 @@ int main(int argc, char** argv) messages << "Detected region: " << r.rtype->combine_region(*r.region).get_name() << std::endl; set_nominal_framerate(r.region->approx_framerate()); - messages << "--- Internal memory mappings ---" << std::endl; - dump_region_map(); messages << "--- End of Startup --- " << std::endl; moviefile movie;