diff --git a/CMakeLists.txt b/CMakeLists.txt index 4161fa2..723a38f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,7 +19,7 @@ set(CMAKE_TOOLCHAIN_FILE "${CMAKE_CURRENT_LIST_DIR}/cmake/toolchain.cmake") project( 573in1 LANGUAGES C CXX ASM - VERSION 1.0.2 + VERSION 1.0.3 DESCRIPTION "Konami System 573 maintenance tool" HOMEPAGE_URL "https://github.com/spicyjpeg/573in1" ) @@ -77,6 +77,7 @@ set( src/common/fs/file.cpp src/common/fs/iso9660.cpp src/common/fs/misc.cpp + src/common/fs/package.cpp src/common/fs/zip.cpp ) set( @@ -204,7 +205,7 @@ function(addPS1Executable name address stackTop) endfunction() # IMPORTANT: these addresses assume the boot executable's size (including code, -# heap and stack allocations as well as the resource archive, but excluding the +# heap and stack allocations as well as the resource package, but excluding the # executable header) is 576 KB (0x90000 bytes) or less, and that each launcher's # size is 12 KB (0x3000 bytes) or less. addPS1Executable(main 800a0000 801dfff0 ${mainSources}) @@ -221,7 +222,7 @@ target_link_libraries(bootStub PUBLIC subExecutableFlags) target_link_libraries(launcher801fd000 PUBLIC launcher) target_link_libraries(launcher803fd000 PUBLIC launcher) -## Boot stubs and resource archives +## Boot stubs and resource packages file(GLOB_RECURSE assetList RELATIVE "${PROJECT_SOURCE_DIR}" assets/*) @@ -233,24 +234,25 @@ function(addBuildVariant name resourceName) add_custom_command( COMMAND "${Python3_EXECUTABLE}" - "${PROJECT_SOURCE_DIR}/tools/buildResourceArchive.py" + "${PROJECT_SOURCE_DIR}/tools/buildResourcePackage.py" + -a 64 ${resourceName}.json - ${resourceName}.zip - OUTPUT ${resourceName}.zip + ${resourceName}.pkg + OUTPUT ${resourceName}.pkg DEPENDS ${resourceName}.json ${assetList} main.psexe launcher801fd000.psexe launcher803fd000.psexe - COMMENT "Building ${name} resource archive" + COMMENT "Building ${name} resource package" VERBATIM ) addPS1Executable(${name} 80010000 0) addBinaryFileWithSize( - ${name} _resourceArchive _resourceArchiveLength - "${PROJECT_BINARY_DIR}/${resourceName}.zip" + ${name} _resourcePackage _resourcePackageLength + "${PROJECT_BINARY_DIR}/${resourceName}.pkg" ) target_link_libraries(${name} PUBLIC bootStub) diff --git a/cmake/options.cmake b/cmake/options.cmake index 3632362..3219abf 100644 --- a/cmake/options.cmake +++ b/cmake/options.cmake @@ -84,7 +84,7 @@ set( set( ENABLE_DUMMY_CART_DRIVER OFF CACHE BOOL "Enable support for simulating a dummy security cartridge (if \ -data/test.573 is present in the resource archive)" +data/dummy.dmp is present in the resource package)" ) set( ENABLE_X76F041_CART_DRIVER ON diff --git a/resources.json b/resources.json index 4a02b4b..bb13c92 100644 --- a/resources.json +++ b/resources.json @@ -3,10 +3,9 @@ "resources": [ { - "type": "binary", - "name": "binaries/main.psexe.lz4", - "source": "${PROJECT_BINARY_DIR}/main.psexe", - "compression": "lz4" + "type": "binary", + "name": "binaries/main.psexe", + "source": "${PROJECT_BINARY_DIR}/main.psexe" }, { "type": "binary", @@ -59,52 +58,52 @@ "source": "${PROJECT_SOURCE_DIR}/assets/textures/font.json" }, { - "type": "binary", - "name": "assets/sounds/startup.vag", - "source": "${PROJECT_SOURCE_DIR}/assets/sounds/startup.vag", - "compression": "none" + "type": "binary", + "name": "assets/sounds/startup.vag", + "source": "${PROJECT_SOURCE_DIR}/assets/sounds/startup.vag", + "compLevel": 0 }, { - "type": "binary", - "name": "assets/sounds/about.vag", - "source": "${PROJECT_SOURCE_DIR}/assets/sounds/about.vag", - "compression": "none" + "type": "binary", + "name": "assets/sounds/about.vag", + "source": "${PROJECT_SOURCE_DIR}/assets/sounds/about.vag", + "compLevel": 0 }, { - "type": "binary", - "name": "assets/sounds/alert.vag", - "source": "${PROJECT_SOURCE_DIR}/assets/sounds/alert.vag", - "compression": "none" + "type": "binary", + "name": "assets/sounds/alert.vag", + "source": "${PROJECT_SOURCE_DIR}/assets/sounds/alert.vag", + "compLevel": 0 }, { - "type": "binary", - "name": "assets/sounds/move.vag", - "source": "${PROJECT_SOURCE_DIR}/assets/sounds/move.vag", - "compression": "none" + "type": "binary", + "name": "assets/sounds/move.vag", + "source": "${PROJECT_SOURCE_DIR}/assets/sounds/move.vag", + "compLevel": 0 }, { - "type": "binary", - "name": "assets/sounds/enter.vag", - "source": "${PROJECT_SOURCE_DIR}/assets/sounds/enter.vag", - "compression": "none" + "type": "binary", + "name": "assets/sounds/enter.vag", + "source": "${PROJECT_SOURCE_DIR}/assets/sounds/enter.vag", + "compLevel": 0 }, { - "type": "binary", - "name": "assets/sounds/exit.vag", - "source": "${PROJECT_SOURCE_DIR}/assets/sounds/exit.vag", - "compression": "none" + "type": "binary", + "name": "assets/sounds/exit.vag", + "source": "${PROJECT_SOURCE_DIR}/assets/sounds/exit.vag", + "compLevel": 0 }, { - "type": "binary", - "name": "assets/sounds/click.vag", - "source": "${PROJECT_SOURCE_DIR}/assets/sounds/click.vag", - "compression": "none" + "type": "binary", + "name": "assets/sounds/click.vag", + "source": "${PROJECT_SOURCE_DIR}/assets/sounds/click.vag", + "compLevel": 0 }, { - "type": "binary", - "name": "assets/sounds/screenshot.vag", - "source": "${PROJECT_SOURCE_DIR}/assets/sounds/screenshot.vag", - "compression": "none" + "type": "binary", + "name": "assets/sounds/screenshot.vag", + "source": "${PROJECT_SOURCE_DIR}/assets/sounds/screenshot.vag", + "compLevel": 0 }, { "type": "text", @@ -112,10 +111,9 @@ "source": "${PROJECT_BINARY_DIR}/about.txt" }, { - "type": "palette", - "name": "assets/palette.dat", - "source": "${PROJECT_SOURCE_DIR}/assets/palette.json", - "compression": "none" + "type": "palette", + "name": "assets/palette.dat", + "source": "${PROJECT_SOURCE_DIR}/assets/palette.json" }, { diff --git a/resourcestiny.json b/resourcestiny.json index 7f7a13b..715f728 100644 --- a/resourcestiny.json +++ b/resourcestiny.json @@ -3,10 +3,9 @@ "resources": [ { - "type": "binary", - "name": "binaries/main.psexe.lz4", - "source": "${PROJECT_BINARY_DIR}/main.psexe", - "compression": "lz4" + "type": "binary", + "name": "binaries/main.psexe", + "source": "${PROJECT_BINARY_DIR}/main.psexe" }, { "type": "binary", @@ -59,46 +58,46 @@ "source": "${PROJECT_SOURCE_DIR}/assets/textures/font.json" }, { - "type": "binary", - "name": "assets/sounds/startup.vag", - "source": "${PROJECT_SOURCE_DIR}/assets/sounds/startup.vag", - "compression": "none" + "type": "binary", + "name": "assets/sounds/startup.vag", + "source": "${PROJECT_SOURCE_DIR}/assets/sounds/startup.vag", + "compLevel": 0 }, { - "type": "binary", - "name": "assets/sounds/alert.vag", - "source": "${PROJECT_SOURCE_DIR}/assets/sounds/alert.vag", - "compression": "none" + "type": "binary", + "name": "assets/sounds/alert.vag", + "source": "${PROJECT_SOURCE_DIR}/assets/sounds/alert.vag", + "compLevel": 0 }, { - "type": "binary", - "name": "assets/sounds/move.vag", - "source": "${PROJECT_SOURCE_DIR}/assets/sounds/move.vag", - "compression": "none" + "type": "binary", + "name": "assets/sounds/move.vag", + "source": "${PROJECT_SOURCE_DIR}/assets/sounds/move.vag", + "compLevel": 0 }, { - "type": "binary", - "name": "assets/sounds/enter.vag", - "source": "${PROJECT_SOURCE_DIR}/assets/sounds/enter.vag", - "compression": "none" + "type": "binary", + "name": "assets/sounds/enter.vag", + "source": "${PROJECT_SOURCE_DIR}/assets/sounds/enter.vag", + "compLevel": 0 }, { - "type": "binary", - "name": "assets/sounds/exit.vag", - "source": "${PROJECT_SOURCE_DIR}/assets/sounds/exit.vag", - "compression": "none" + "type": "binary", + "name": "assets/sounds/exit.vag", + "source": "${PROJECT_SOURCE_DIR}/assets/sounds/exit.vag", + "compLevel": 0 }, { - "type": "binary", - "name": "assets/sounds/click.vag", - "source": "${PROJECT_SOURCE_DIR}/assets/sounds/click.vag", - "compression": "none" + "type": "binary", + "name": "assets/sounds/click.vag", + "source": "${PROJECT_SOURCE_DIR}/assets/sounds/click.vag", + "compLevel": 0 }, { - "type": "binary", - "name": "assets/sounds/screenshot.vag", - "source": "${PROJECT_SOURCE_DIR}/assets/sounds/screenshot.vag", - "compression": "none" + "type": "binary", + "name": "assets/sounds/screenshot.vag", + "source": "${PROJECT_SOURCE_DIR}/assets/sounds/screenshot.vag", + "compLevel": 0 }, { "type": "text", @@ -106,10 +105,9 @@ "source": "${PROJECT_BINARY_DIR}/about.txt" }, { - "type": "palette", - "name": "assets/palette.dat", - "source": "${PROJECT_SOURCE_DIR}/assets/palette.json", - "compression": "none" + "type": "palette", + "name": "assets/palette.dat", + "source": "${PROJECT_SOURCE_DIR}/assets/palette.json" }, { diff --git a/schema/resources.json b/schema/resources.json index 670f371..93ccd53 100644 --- a/schema/resources.json +++ b/schema/resources.json @@ -9,7 +9,7 @@ "properties": { "resources": { - "title": "Resource archive entries", + "title": "Resource package entries", "type": "array", "uniqueItems": true, @@ -39,19 +39,12 @@ }, "name": { "title": "Entry path", - "description": "Full path of the entry within the resource archive.", + "description": "Full path of the entry within the resource package.", "type": "string" }, - "compression": { - "title": "Compression type", - "description": "Must be 'none', 'deflate' or 'lz4'. If unspecified, defaults to 'deflate' unless manually overridden on the command line.", - "type": "string", - - "enum": [ "none", "deflate", "lz4" ] - }, - "compressLevel": { + "compLevel": { "title": "Compression level", - "description": "DEFLATE or LZ4 compression level in 0-9 range, ignored for uncompressed files. If unspecified, defaults to 9 unless manually overridden on the command line.", + "description": "LZ4 compression level in 1-9 range, or 0 to store the file uncompressed. If unspecified, defaults to 9 unless manually overridden on the command line.", "type": "integer", "minimum": 0, @@ -81,10 +74,9 @@ "additionalProperties": false, "properties": { - "type": { "pattern": "^text|binary$" }, - "name": { "type": "string" }, - "compression": {}, - "compressLevel": {}, + "type": { "pattern": "^text|binary$" }, + "name": { "type": "string" }, + "compLevel": {}, "source": { "title": "Path to source file", @@ -100,10 +92,9 @@ "additionalProperties": false, "properties": { - "type": { "const": "tim" }, - "name": { "type": "string" }, - "compression": {}, - "compressLevel": {}, + "type": { "const": "tim" }, + "name": { "type": "string" }, + "compLevel": {}, "source": { "title": "Path to source file", @@ -181,10 +172,9 @@ "additionalProperties": false, "properties": { - "type": { "pattern": "^metrics|palette|strings|db$" }, - "name": { "type": "string" }, - "compression": {}, - "compressLevel": {}, + "type": { "pattern": "^metrics|palette|strings|db$" }, + "name": { "type": "string" }, + "compLevel": {}, "source": { "title": "Path to source file", @@ -200,10 +190,9 @@ "additionalProperties": false, "properties": { - "type": { "const": "metrics" }, - "name": { "type": "string" }, - "compression": {}, - "compressLevel": {}, + "type": { "const": "metrics" }, + "name": { "type": "string" }, + "compLevel": {}, "metrics": { "title": "Font metrics", @@ -217,10 +206,9 @@ "additionalProperties": false, "properties": { - "type": { "const": "palette" }, - "name": { "type": "string" }, - "compression": {}, - "compressLevel": {}, + "type": { "const": "palette" }, + "name": { "type": "string" }, + "compLevel": {}, "palette": { "title": "Color entries", @@ -234,10 +222,9 @@ "additionalProperties": false, "properties": { - "type": { "const": "strings" }, - "name": { "type": "string" }, - "compression": {}, - "compressLevel": {}, + "type": { "const": "strings" }, + "name": { "type": "string" }, + "compLevel": {}, "strings": { "title": "String table", @@ -251,10 +238,9 @@ "additionalProperties": false, "properties": { - "type": { "const": "db" }, - "name": { "type": "string" }, - "compression": {}, - "compressLevel": {}, + "type": { "const": "db" }, + "name": { "type": "string" }, + "compLevel": {}, "strings": { "title": "Game database", diff --git a/src/boot/main.cpp b/src/boot/main.cpp index 597a9b5..370f478 100644 --- a/src/boot/main.cpp +++ b/src/boot/main.cpp @@ -16,78 +16,64 @@ #include #include +#include "common/fs/package.hpp" +#include "common/util/hash.hpp" #include "common/util/misc.hpp" #include "common/util/string.hpp" -#include "common/util/templates.hpp" #include "common/io.hpp" #include "ps1/system.h" -extern "C" const uint8_t _resourceArchive[]; -extern "C" const size_t _resourceArchiveLength; +extern "C" const uint8_t _resourcePackage[]; +extern "C" const size_t _resourcePackageLength; static char _ptrArg[]{ "resource.ptr=xxxxxxxx\0" }; static char _lengthArg[]{ "resource.length=xxxxxxxx\0" }; -class [[gnu::packed]] ZIPFileHeader { -public: - uint32_t magic; - uint16_t version, flags, compType; - uint16_t fileTime, fileDate; - uint32_t crc, compLength, uncompLength; - uint16_t nameLength, extraLength; - - inline bool validateMagic(void) const { - return (magic == util::concat4('P', 'K', 0x03, 0x04)); - } - inline size_t getHeaderLength(void) const { - return sizeof(ZIPFileHeader) + nameLength + extraLength; - } -}; - int main(int argc, const char **argv) { disableInterrupts(); io::init(); - // Parse the header of the archive's first entry manually. This avoids - // pulling in miniz and bloating the binary. - // NOTE: this assumes the main executable is always the first file in the - // archive. - auto zipHeader = reinterpret_cast(_resourceArchive); - auto ptr = &_resourceArchive[zipHeader->getHeaderLength()]; - auto compLength = zipHeader->compLength; - -#if 0 - assert(zipHeader->validateMagic()); - assert(!zipHeader->compType); -#endif + auto header = \ + reinterpret_cast(_resourcePackage); + auto entry = util::getHashTableEntry( + reinterpret_cast(header + 1), + header->numBuckets, + "binaries/main.psexe"_h + ); + auto ptr = &_resourcePackage[entry->offset]; // Decompress only the header to determine where to place the binary in // memory, then rerun the decompressor on the entire executable. util::ExecutableHeader exeHeader; util::decompressLZ4( - reinterpret_cast(&exeHeader), ptr, sizeof(exeHeader), - compLength + reinterpret_cast(&exeHeader), + ptr, + sizeof(exeHeader), + entry->compLength ); auto offset = exeHeader.textOffset - util::EXECUTABLE_BODY_OFFSET; - auto length = exeHeader.textLength + util::EXECUTABLE_BODY_OFFSET; util::decompressLZ4( - reinterpret_cast(offset), ptr, length, compLength + reinterpret_cast(offset), + ptr, + entry->uncompLength, + entry->compLength ); io::clearWatchdog(); util::ExecutableLoader loader( - exeHeader.getEntryPoint(), exeHeader.getInitialGP(), + exeHeader.getEntryPoint(), + exeHeader.getInitialGP(), exeHeader.getStackPtr() ); util::hexValueToString( - &_ptrArg[13], reinterpret_cast(_resourceArchive), 8 + &_ptrArg[13], reinterpret_cast(_resourcePackage), 8 ); loader.addArgument(_ptrArg); - util::hexValueToString(&_lengthArg[16], _resourceArchiveLength, 8); + util::hexValueToString(&_lengthArg[16], _resourcePackageLength, 8); loader.addArgument(_lengthArg); #ifdef ENABLE_ARGV_PARSER diff --git a/src/common/fs/file.cpp b/src/common/fs/file.cpp index 0b313d4..a465ec2 100644 --- a/src/common/fs/file.cpp +++ b/src/common/fs/file.cpp @@ -116,7 +116,7 @@ size_t Provider::loadTIM(gpu::Image &output, const char *path) { if (!loadData(data, path)) return 0; - auto header = data.as(); + auto header = data.as(); if (output.initFromTIMHeader(*header)) { auto image = header->getImage(); @@ -141,7 +141,7 @@ size_t Provider::loadBS( if (!loadData(data, path)) return 0; - size_t bsLength = data.as()->getUncompLength(); + size_t bsLength = data.as()->getUncompLength(); mdec::BSDecompressor decompressor; util::Data buffer; @@ -189,7 +189,7 @@ size_t Provider::loadVAG( if (!loadData(data, path)) return 0; - auto header = data.as(); + auto header = data.as(); if (output.initFromVAGHeader(*header, offset)) loadLength = spu::upload( @@ -285,22 +285,15 @@ size_t Provider::saveVRAMBMP(const gpu::RectWH &rect, const char *path) { static const char _ERROR_STRING[]{ "missingno" }; const char *StringTable::get(util::Hash id) const { - if (!ptr) - return _ERROR_STRING; + auto header = as(); + auto blob = as(); + auto entry = util::getHashTableEntry( + reinterpret_cast(header + 1), + header->numBuckets, + id + ); - auto blob = as(); - auto table = as(); - auto index = id % STRING_TABLE_BUCKET_COUNT; - - do { - auto entry = &table[index]; - index = entry->chained; - - if (entry->hash == id) - return &blob[entry->offset]; - } while (index); - - return _ERROR_STRING; + return entry ? &blob[entry->offset] : _ERROR_STRING; } size_t StringTable::format( diff --git a/src/common/fs/file.hpp b/src/common/fs/file.hpp index 7fe7a23..1f11cdf 100644 --- a/src/common/fs/file.hpp +++ b/src/common/fs/file.hpp @@ -32,16 +32,18 @@ static constexpr size_t MAX_PATH_LENGTH = 256; // The first 4 of these map to the FS_* enum used by FatFs. enum FileSystemType { - NONE = 0, - FAT12 = 1, - FAT16 = 2, - FAT32 = 3, - EXFAT = 4, - ISO9660 = 5, - ZIP_MEMORY = 6, - ZIP_FILE = 7, - HOST = 8, - VFS = 9 + NONE = 0, + FAT12 = 1, + FAT16 = 2, + FAT32 = 3, + EXFAT = 4, + ISO9660 = 5, + PACKAGE_MEMORY = 6, + PACKAGE_FILE = 7, + ZIP_MEMORY = 8, + ZIP_FILE = 9, + HOST = 10, + VFS = 11 }; // These are functionally equivalent to the FA_* flags used by FatFs. @@ -158,12 +160,22 @@ public: /* String table parser */ -static constexpr size_t STRING_TABLE_BUCKET_COUNT = 256; - -struct StringTableEntry { +struct StringTableHeader { public: - uint32_t hash; - uint16_t offset, chained; + uint16_t numBuckets, numEntries; +}; + +class StringTableEntry { +public: + util::Hash id; + uint16_t offset, chained; + + inline util::Hash getHash(void) const { + return id; + } + inline uint16_t getChained(void) const { + return chained; + } }; class StringTable : public util::Data { diff --git a/src/common/fs/package.cpp b/src/common/fs/package.cpp new file mode 100644 index 0000000..b69b6a9 --- /dev/null +++ b/src/common/fs/package.cpp @@ -0,0 +1,256 @@ +/* + * 573in1 - Copyright (C) 2022-2024 spicyjpeg + * + * 573in1 is free software: you can redistribute it and/or modify it under the + * terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * 573in1 is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * 573in1. If not, see . + */ + +#include +#include +#include "common/fs/file.hpp" +#include "common/fs/package.hpp" +#include "common/util/hash.hpp" +#include "common/util/log.hpp" +#include "common/util/string.hpp" + +namespace fs { + +/* Package filesystem provider */ + +const PackageIndexEntry *PackageProvider::_getEntry(const char *path) const { + // Any leading path separators must be stripped manually. + while ((*path == '/') || (*path == '\\')) + path++; + + auto header = _index.as(); + + return util::getHashTableEntry( + reinterpret_cast(header + 1), + header->numBuckets, + util::hash(path) + ); +} + +bool PackageProvider::init(File *file) { + if (type) + return false; + + _file = file; + + // Parse the package's header to obtain the size of the index, then reread + // the entire index section. + PackageIndexHeader header; + + if (file->read(&header, sizeof(header)) < sizeof(header)) + return false; + + size_t indexLength = header.indexLength; + + if (!_index.allocate(indexLength)) + return false; + + if ( + file->seek(0) || + (file->read(_index.ptr, indexLength) != indexLength) + ) { + _index.destroy(); + return false; + } + + type = PACKAGE_FILE; + capacity = file->size - indexLength; + + LOG_FS("mounted package file"); + return true; +} + +bool PackageProvider::init(const void *packageData, size_t length) { + if (type) + return false; + + auto header = reinterpret_cast(packageData); + + // TODO: set a flag to prevent output deallocation + _file = nullptr; + _index.ptr = reinterpret_cast(uintptr_t(packageData)); + _index.length = header->indexLength; + + type = PACKAGE_MEMORY; + capacity = 0; + + LOG_FS("mounted package: 0x%08x", packageData); + return true; +} + +void PackageProvider::close(void) { + if (!type) + return; + + _index.destroy(); + +#if 0 + if (_file) { + _file->close(); + delete _file; + } +#endif + + type = NONE; + capacity = 0; +} + +bool PackageProvider::getFileInfo(FileInfo &output, const char *path) { + auto blob = _index.as(); + auto entry = _getEntry(path); + + if (!entry) + return false; + +#if 0 + const char *ptr = __builtin_strrchr(&blob[entry->nameOffset], '/'); + + if (ptr) + ptr++; + else + ptr = &blob[entry->nameOffset]; +#else + auto ptr = &blob[entry->nameOffset]; +#endif + + __builtin_strncpy(output.name, ptr, sizeof(output.name)); + output.size = entry->uncompLength; + output.attributes = READ_ONLY | ARCHIVE; + return true; +} + +size_t PackageProvider::loadData(util::Data &output, const char *path) { + auto blob = _index.as(); + auto entry = _getEntry(path); + + if (!entry) + return 0; + + auto offset = entry->offset; + size_t compLength = entry->compLength; + size_t uncompLength = entry->uncompLength; + + if (_file) { + if (compLength) { + // Package on disk, file compressed + auto margin = util::getLZ4InPlaceMargin(compLength); + + if (!output.allocate(uncompLength + margin)) + return 0; + + auto compPtr = &output.as()[margin]; + + if ( + (_file->seek(offset) != offset) || + (_file->read(compPtr, compLength) < compLength) + ) { + output.destroy(); + return 0; + } + + util::decompressLZ4( + output.as(), + compPtr, + uncompLength, + compLength + ); + } else { + // Package on disk, file not compressed + if (!output.allocate(uncompLength)) + return 0; + + if (_file->seek(offset) != offset) { + output.destroy(); + return 0; + } + + return _file->read(output.ptr, uncompLength); + } + } else { + if (compLength) { + // Package in RAM, file compressed + if (!output.allocate(uncompLength)) + return 0; + + util::decompressLZ4( + output.as(), + &blob[offset], + uncompLength, + compLength + ); + } else { + // Package in RAM, file not compressed (return in-place pointer) + // TODO: set a flag to prevent output deallocation + output.ptr = &blob[offset]; + output.length = uncompLength; + } + } + + return uncompLength; +} + +size_t PackageProvider::loadData(void *output, size_t length, const char *path) { + auto blob = _index.as(); + auto entry = _getEntry(path); + + if (!entry) + return 0; + + auto offset = entry->offset; + size_t compLength = entry->compLength; + size_t uncompLength = util::min(length, size_t(entry->uncompLength)); + + if (_file) { + if (_file->seek(offset) != offset) + return 0; + + if (compLength) { + // Package on disk, file compressed + auto margin = util::getLZ4InPlaceMargin(compLength); + auto compPtr = &reinterpret_cast(output)[margin]; + + if (_file->read(compPtr, compLength) < compLength) + return 0; + + util::decompressLZ4( + reinterpret_cast(output), + compPtr, + uncompLength, + compLength + ); + } else { + // Package on disk, file not compressed + if (_file->read(output, uncompLength) < uncompLength) + return 0; + } + } else { + if (compLength) + // Package in RAM, file compressed + util::decompressLZ4( + reinterpret_cast(output), + &blob[offset], + uncompLength, + compLength + ); + else + // Package in RAM, file not compressed + __builtin_memcpy(output, &blob[offset], uncompLength); + } + + return uncompLength; +} + +} diff --git a/src/common/fs/package.hpp b/src/common/fs/package.hpp new file mode 100644 index 0000000..312a45e --- /dev/null +++ b/src/common/fs/package.hpp @@ -0,0 +1,71 @@ +/* + * 573in1 - Copyright (C) 2022-2024 spicyjpeg + * + * 573in1 is free software: you can redistribute it and/or modify it under the + * terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * 573in1 is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * 573in1. If not, see . + */ + +#pragma once + +#include +#include +#include "common/fs/file.hpp" +#include "common/util/hash.hpp" +#include "common/util/templates.hpp" + +namespace fs { + +/* Package index parser */ + +struct PackageIndexHeader { +public: + uint32_t indexLength; + uint16_t numBuckets, numEntries; +}; + +class PackageIndexEntry { +public: + util::Hash id; + uint16_t nameOffset, chained; + uint64_t offset; + uint32_t compLength, uncompLength; + + inline util::Hash getHash(void) const { + return id; + } + inline uint16_t getChained(void) const { + return chained; + } +}; + +/* Package filesystem provider */ + +// The current implementation only supports loading an entire file at once. +class PackageProvider : public Provider { +private: + util::Data _index; + File *_file; + + const PackageIndexEntry *_getEntry(const char *path) const; + +public: + bool init(File *file); + bool init(const void *packageData, size_t length); + void close(void); + + bool getFileInfo(FileInfo &output, const char *path); + + size_t loadData(util::Data &output, const char *path); + size_t loadData(void *output, size_t length, const char *path); +}; + +} diff --git a/src/common/gpufont.cpp b/src/common/gpufont.cpp index 92a3fa6..36d080e 100644 --- a/src/common/gpufont.cpp +++ b/src/common/gpufont.cpp @@ -15,6 +15,7 @@ */ #include +#include "common/util/hash.hpp" #include "common/util/string.hpp" #include "common/gpu.hpp" #include "common/gpufont.hpp" @@ -28,18 +29,19 @@ CharacterSize FontMetrics::get(util::UTF8CodePoint id) const { if (!ptr) return 0; - auto table = reinterpret_cast(getHeader() + 1); - auto index = id % METRICS_BUCKET_COUNT; + auto header = as(); + auto entry = util::getHashTableEntry( + reinterpret_cast(header + 1), + header->numBuckets, + id + ); - do { - auto entry = &table[index]; - index = entry->getChained(); - - if (entry->getCodePoint() == id) - return entry->size; - } while (index); - - return (id == FONT_INVALID_CHAR) ? 0 : get(FONT_INVALID_CHAR); + if (entry) + return entry->size; + else if (id != FONT_INVALID_CHAR) + return get(FONT_INVALID_CHAR); + else + return 0; } /* Font class */ @@ -53,7 +55,7 @@ void Font::draw( ctx.setTexturePage(image.texpage); - auto header = metrics.getHeader(); + auto header = metrics.as(); int x = rect.x1; int clipX1 = clipRect.x1; @@ -153,7 +155,7 @@ void Font::draw( } int Font::getCharacterWidth(util::UTF8CodePoint ch) const { - auto header = metrics.getHeader(); + auto header = metrics.as(); switch (ch) { case 0: @@ -180,7 +182,7 @@ void Font::getStringBounds( if (!str || !metrics.ptr) return; - auto header = metrics.getHeader(); + auto header = metrics.as(); int x = rect.x1, maxX = rect.x1, y = rect.y1; @@ -259,7 +261,7 @@ int Font::getStringWidth(const char *str, bool breakOnSpace) const { if (!str || !metrics.ptr) return 0; - auto header = metrics.getHeader(); + auto header = metrics.as(); int width = 0, maxWidth = 0; diff --git a/src/common/gpufont.hpp b/src/common/gpufont.hpp index a1bb55b..4b0548d 100644 --- a/src/common/gpufont.hpp +++ b/src/common/gpufont.hpp @@ -18,6 +18,7 @@ #include #include +#include "common/util/hash.hpp" #include "common/util/string.hpp" #include "common/util/templates.hpp" #include "common/gpu.hpp" @@ -26,7 +27,6 @@ namespace gpu { /* Font metrics class */ -static constexpr size_t METRICS_BUCKET_COUNT = 256; static constexpr size_t METRICS_CODE_POINT_BITS = 21; static constexpr util::UTF8CodePoint FONT_INVALID_CHAR = 0xfffd; @@ -35,8 +35,9 @@ using CharacterSize = uint32_t; struct FontMetricsHeader { public: - uint8_t spaceWidth, tabWidth, lineHeight; - int8_t baselineOffset; + uint8_t spaceWidth, tabWidth, lineHeight; + int8_t baselineOffset; + uint16_t numBuckets, numEntries; }; class FontMetricsEntry { @@ -44,7 +45,7 @@ public: uint32_t codePoint; CharacterSize size; - inline util::UTF8CodePoint getCodePoint(void) const { + inline util::Hash getHash(void) const { return codePoint & ((1 << METRICS_CODE_POINT_BITS) - 1); } inline uint32_t getChained(void) const { @@ -54,9 +55,6 @@ public: class FontMetrics : public util::Data { public: - inline const FontMetricsHeader *getHeader(void) const { - return as(); - } inline CharacterSize operator[](util::UTF8CodePoint id) const { return get(id); } @@ -75,13 +73,13 @@ public: if (!metrics.ptr) return 0; - return metrics.getHeader()->spaceWidth; + return metrics.as()->spaceWidth; } inline int getLineHeight(void) const { if (!metrics.ptr) return 0; - return metrics.getHeader()->lineHeight; + return metrics.as()->lineHeight; } void draw( diff --git a/src/common/util/hash.hpp b/src/common/util/hash.hpp index 3c85ec6..06a2ea3 100644 --- a/src/common/util/hash.hpp +++ b/src/common/util/hash.hpp @@ -40,6 +40,28 @@ template static constexpr inline Hash hash( Hash hash(const char *str, char terminator = 0); Hash hash(const uint8_t *data, size_t length); +/* Hash table parser */ + +template static inline const T *getHashTableEntry( + const T *table, size_t numBuckets, Hash id +) { +#if 0 + auto index = id % NB; +#else + auto index = id & (numBuckets - 1); +#endif + + do { + auto entry = &table[index]; + index = entry->getChained(); + + if (entry->getHash() == id) + return entry; + } while (index); + + return nullptr; +} + /* CRC calculation */ uint8_t dsCRC8(const uint8_t *data, size_t length); diff --git a/src/main/app/app.cpp b/src/main/app/app.cpp index e73734c..e40294b 100644 --- a/src/main/app/app.cpp +++ b/src/main/app/app.cpp @@ -22,7 +22,7 @@ #include "common/fs/file.hpp" #include "common/fs/iso9660.hpp" #include "common/fs/misc.hpp" -#include "common/fs/zip.hpp" +#include "common/fs/package.hpp" #include "common/storage/device.hpp" #include "common/util/log.hpp" #include "common/util/misc.hpp" @@ -179,7 +179,7 @@ bool FileIOManager::loadResourceFile(const char *path) { if (path) _resourceFile = vfs.openFile(path, fs::READ); - // Fall back to the default in-memory resource archive in case of failure. + // Fall back to the default in-memory resource package in case of failure. if (_resourceFile) { if (resource.init(_resourceFile)) return true; diff --git a/src/main/app/app.hpp b/src/main/app/app.hpp index 071ae88..06112db 100644 --- a/src/main/app/app.hpp +++ b/src/main/app/app.hpp @@ -19,7 +19,7 @@ #include #include "common/fs/file.hpp" #include "common/fs/misc.hpp" -#include "common/fs/zip.hpp" +#include "common/fs/package.hpp" #include "common/storage/device.hpp" #include "common/util/log.hpp" #include "common/util/templates.hpp" @@ -76,11 +76,11 @@ public: const void *resourcePtr; size_t resourceLength; - fs::ZIPProvider resource; + fs::PackageProvider resource; #ifdef ENABLE_PCDRV - fs::HostProvider host; + fs::HostProvider host; #endif - fs::VFSProvider vfs; + fs::VFSProvider vfs; storage::Device *ideDevices[2]; fs::Provider *ideProviders[2]; diff --git a/src/main/workers/cartworkers.cpp b/src/main/workers/cartworkers.cpp index 0b78314..7cbd380 100644 --- a/src/main/workers/cartworkers.cpp +++ b/src/main/workers/cartworkers.cpp @@ -45,7 +45,7 @@ bool cartDetectWorker(App &app) { #ifdef ENABLE_DUMMY_CART_DRIVER if (!cart::dummyDriverDump.chipType) - app._fileIO.resource.loadStruct(cart::dummyDriverDump, "data/test.573"); + app._fileIO.resource.loadStruct(cart::dummyDriverDump, "data/dummy.dmp"); if (cart::dummyDriverDump.chipType) { LOG_APP("using dummy cart driver"); @@ -214,11 +214,11 @@ bool cartDumpWorker(App &app) { app._cartParser->getRegion(region) ) { snprintf( - path, sizeof(path), EXTERNAL_DATA_DIR "/%s%s.573", code, region + path, sizeof(path), EXTERNAL_DATA_DIR "/%s%s.dmp", code, region ); } else { if (!app._getNumberedPath( - path, sizeof(path), EXTERNAL_DATA_DIR "/cart%04d.573" + path, sizeof(path), EXTERNAL_DATA_DIR "/cart%04d.dmp" )) goto _error; } diff --git a/src/main/workers/miscworkers.cpp b/src/main/workers/miscworkers.cpp index b313dc3..b80e3e6 100644 --- a/src/main/workers/miscworkers.cpp +++ b/src/main/workers/miscworkers.cpp @@ -99,7 +99,7 @@ bool fileInitWorker(App &app) { app._fileIO.mountIDE(); app._workerStatus.update(2, 3, WSTR("App.fileInitWorker.loadResources")); - if (app._fileIO.loadResourceFile(EXTERNAL_DATA_DIR "/resource.zip")) + if (app._fileIO.loadResourceFile(EXTERNAL_DATA_DIR "/resource.pkg")) app._loadResources(); return true; diff --git a/tools/buildResourceArchive.py b/tools/buildResourceArchive.py deleted file mode 100755 index c981261..0000000 --- a/tools/buildResourceArchive.py +++ /dev/null @@ -1,209 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# 573in1 - Copyright (C) 2022-2024 spicyjpeg -# -# 573in1 is free software: you can redistribute it and/or modify it under the -# terms of the GNU General Public License as published by the Free Software -# Foundation, either version 3 of the License, or (at your option) any later -# version. -# -# 573in1 is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along with -# 573in1. If not, see . - -__version__ = "0.4.6" -__author__ = "spicyjpeg" - -import json -from argparse import ArgumentParser, FileType, Namespace -from pathlib import Path -from typing import Any, ByteString -from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile - -import lz4.block -from common.assets import * -from PIL import Image - -## Main - -def createParser() -> ArgumentParser: - parser = ArgumentParser( - description = \ - "Parses a JSON file containing a list of resources to convert, " - "generates the respective files and packs them into a ZIP archive.", - add_help = False - ) - - group = parser.add_argument_group("Tool options") - group.add_argument( - "-h", "--help", - action = "help", - help = "Show this help message and exit" - ) - - group = parser.add_argument_group("Compression options") - group.add_argument( - "-c", "--compression", - type = str, - choices = ( "none", "deflate", "lz4" ), - default = "deflate", - help = "Set default compression algorithm (default DEFLATE)" - ) - group.add_argument( - "-l", "--compress-level", - type = int, - default = 9, - help = "Set default DEFLATE and LZ4 compression level (default 9)", - metavar = "0-9" - ) - - group = parser.add_argument_group("File paths") - group.add_argument( - "-s", "--source-dir", - type = Path, - help = \ - "Set path to directory containing source files (same directory as " - "resource list by default)", - metavar = "dir" - ) - group.add_argument( - "configFile", - type = FileType("rt", encoding = "utf-8"), - help = "Path to JSON configuration file", - ) - group.add_argument( - "output", - type = Path, - help = "Path to ZIP file to generate" - ) - - return parser - -def main(): - parser: ArgumentParser = createParser() - args: Namespace = parser.parse_args() - - with args.configFile as file: - configFile: dict[str, Any] = json.load(file) - sourceDir: Path = \ - args.source_dir or Path(file.name).parent - - assetList: list[dict[str, Any]] = configFile["resources"] - - with ZipFile(args.output, "w", allowZip64 = False) as _zip: - for asset in assetList: - match asset.get("type", "file").strip(): - case "empty": - data: ByteString = bytes(int(asset.get("size", 0))) - - case "text": - with open( - sourceDir / asset["source"], "rt", encoding = "utf-8" - ) as file: - data: ByteString = file.read().encode("ascii") - - case "binary": - with open(sourceDir / asset["source"], "rb") as file: - data: ByteString = file.read() - - case "tim": - ix: int = int(asset["imagePos"]["x"]) - iy: int = int(asset["imagePos"]["y"]) - cx: int = int(asset["clutPos"]["x"]) - cy: int = int(asset["clutPos"]["y"]) - - image: Image.Image = Image.open(sourceDir / asset["source"]) - image.load() - - if image.mode != "P": - image = image.quantize( - int(asset.get("quantize", 16)), dither = Image.NONE - ) - - data: ByteString = generateIndexedTIM(image, ix, iy, cx, cy) - - case "metrics": - if "metrics" in asset: - metrics: dict = asset["metrics"] - else: - with open( - sourceDir / asset["source"], "rt", - encoding = "utf-8" - ) as file: - metrics: dict = json.load(file) - - data: ByteString = generateFontMetrics(metrics) - - case "palette": - if "palette" in asset: - palette: dict = asset["palette"] - else: - with open( - sourceDir / asset["source"], "rt", - encoding = "utf-8" - ) as file: - palette: dict = json.load(file) - - data: ByteString = generateColorPalette(palette) - - case "strings": - if "strings" in asset: - strings: dict = asset["strings"] - else: - with open( - sourceDir / asset["source"], "rt", - encoding = "utf-8" - ) as file: - strings: dict = json.load(file) - - data: ByteString = generateStringTable(strings) - - case "db": - if "db" in asset: - db: dict = asset["db"] - else: - with open( - sourceDir / asset["source"], "rt", - encoding = "utf-8" - ) as file: - db: dict = json.load(file) - - # TODO: implement - data: ByteString = b"" - - case _type: - raise KeyError(f"unsupported asset type '{_type}'") - - compressLevel: int | None = \ - asset.get("compressLevel", args.compress_level) - - match asset.get("compression", args.compression).strip(): - case "none" | None: - _zip.writestr(asset["name"], data, ZIP_STORED) - - case "deflate": - _zip.writestr( - asset["name"], data, ZIP_DEFLATED, compressLevel - ) - - case "lz4": - # ZIP archives do not "officially" support LZ4 compression, - # so the entry is stored as an uncompressed file. - compressed: bytes = lz4.block.compress( - data, - mode = "high_compression", - compression = compressLevel, - store_size = False - ) - - _zip.writestr(asset["name"], compressed, ZIP_STORED) - - case _type: - raise KeyError(f"unsupported compression type '{_type}'") - -if __name__ == "__main__": - main() diff --git a/tools/buildResourcePackage.py b/tools/buildResourcePackage.py new file mode 100644 index 0000000..f6c88c1 --- /dev/null +++ b/tools/buildResourcePackage.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# 573in1 - Copyright (C) 2022-2024 spicyjpeg +# +# 573in1 is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# 573in1 is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# 573in1. If not, see . + +__version__ = "1.0.3" +__author__ = "spicyjpeg" + +import json +from argparse import ArgumentParser, FileType, Namespace +from pathlib import Path +from typing import Any, ByteString, Mapping + +import lz4.block +from common.assets import * +from PIL import Image + +## Asset conversion + +def processAsset(asset: Mapping[str, Any], sourceDir: Path) -> ByteString: + match asset.get("type", "file").strip(): + case "empty": + return bytes(int(asset.get("size", 0))) + + case "text" | "binary": + with open(sourceDir / asset["source"], "rb") as file: + data: ByteString = file.read() + + return data + + case "tim": + ix: int = int(asset["imagePos"]["x"]) + iy: int = int(asset["imagePos"]["y"]) + cx: int = int(asset["clutPos"]["x"]) + cy: int = int(asset["clutPos"]["y"]) + + image: Image.Image = Image.open(sourceDir / asset["source"]) + image.load() + + if image.mode != "P": + image = image.quantize( + int(asset.get("quantize", 16)), dither = Image.NONE + ) + + return generateIndexedTIM(image, ix, iy, cx, cy) + + case "metrics": + if "metrics" in asset: + metrics: dict = asset["metrics"] + else: + with open( + sourceDir / asset["source"], "rt", + encoding = "utf-8" + ) as file: + metrics: dict = json.load(file) + + return generateFontMetrics(metrics) + + case "palette": + if "palette" in asset: + palette: dict = asset["palette"] + else: + with open( + sourceDir / asset["source"], "rt", + encoding = "utf-8" + ) as file: + palette: dict = json.load(file) + + return generateColorPalette(palette) + + case "strings": + if "strings" in asset: + strings: dict = asset["strings"] + else: + with open( + sourceDir / asset["source"], "rt", + encoding = "utf-8" + ) as file: + strings: dict = json.load(file) + + return generateStringTable(strings) + + case "db": + if "db" in asset: + db: dict = asset["db"] + else: + with open( + sourceDir / asset["source"], "rt", + encoding = "utf-8" + ) as file: + db: dict = json.load(file) + + # TODO: implement + return b"" + + case _type: + raise KeyError(f"unsupported asset type '{_type}'") + +## Main + +def createParser() -> ArgumentParser: + parser = ArgumentParser( + description = \ + "Parses a JSON file containing a list of resources to convert, " + "generates the respective files and packs them into a 573in1 " + "resource package (.pkg file).", + add_help = False + ) + + group = parser.add_argument_group("Tool options") + group.add_argument( + "-h", "--help", + action = "help", + help = "Show this help message and exit" + ) + + group = parser.add_argument_group("Package options") + group.add_argument( + "-a", "--align", + type = int, + default = 2048, + help = \ + "Ensure all files in the package are aligned to specified sector " + "size (default 2048)", + metavar = "length" + + ) + group.add_argument( + "-c", "--compress-level", + type = int, + default = 9, + help = \ + "Set default LZ4 compression level (0 to disable compression, " + "default 9)", + metavar = "0-9" + ) + + group = parser.add_argument_group("File paths") + group.add_argument( + "-s", "--source-dir", + type = Path, + help = \ + "Set path to directory containing source files (same directory as " + "resource list by default)", + metavar = "dir" + ) + group.add_argument( + "configFile", + type = FileType("rt", encoding = "utf-8"), + help = "Path to JSON configuration file", + ) + group.add_argument( + "output", + type = FileType("wb"), + help = "Path to package file to generate" + ) + + return parser + +def main(): + parser: ArgumentParser = createParser() + args: Namespace = parser.parse_args() + + with args.configFile as file: + configFile: dict[str, Any] = json.load(file) + sourceDir: Path = \ + args.source_dir or Path(file.name).parent + + entries: dict[str, PackageIndexEntry] = {} + fileData: bytearray = bytearray() + + for asset in configFile["resources"]: + data: ByteString = processAsset(asset, sourceDir) + entry: PackageIndexEntry = \ + PackageIndexEntry(len(fileData), 0, len(data)) + + compLevel: int | None = asset.get("compLevel", args.compress_level) + + if data and compLevel: + data = lz4.block.compress( + data, + mode = "high_compression", + compression = compLevel, + store_size = False + ) + entry.compLength = len(data) + + entries[asset["name"]] = entry + fileData += data + + while len(fileData) % args.align: + fileData.append(0) + + indexData: bytearray = generatePackageIndex(entries, args.align) + + while len(indexData) % args.align: + indexData.append(0) + + with args.output as file: + file.write(indexData) + file.write(fileData) + +if __name__ == "__main__": + main() diff --git a/tools/common/assets.py b/tools/common/assets.py index fdf1f6e..089eda0 100644 --- a/tools/common/assets.py +++ b/tools/common/assets.py @@ -14,14 +14,16 @@ # You should have received a copy of the GNU General Public License along with # 573in1. If not, see . -from itertools import chain -from struct import Struct -from typing import Any, Generator, Mapping, Sequence +from dataclasses import dataclass +from struct import Struct +from typing import Any, Generator, Mapping, Sequence import numpy from numpy import ndarray from PIL import Image -from .util import colorFromString, generateHashTable, hashData +from .util import \ + HashTableBuilder, StringBlobBuilder, colorFromString, hashData, \ + roundUpToMultiple ## .TIM image converter @@ -130,17 +132,18 @@ def generateIndexedTIM( ## Font metrics generator -_METRICS_HEADER_STRUCT: Struct = Struct("< 3B b") +_METRICS_HEADER_STRUCT: Struct = Struct("< 3B b 2H") _METRICS_ENTRY_STRUCT: Struct = Struct("< 2I") -_METRICS_BUCKET_COUNT: int = 256 -def generateFontMetrics(metrics: Mapping[str, Any]) -> bytearray: +def generateFontMetrics( + metrics: Mapping[str, Any], numBuckets: int = 256 +) -> bytearray: spaceWidth: int = int(metrics["spaceWidth"]) tabWidth: int = int(metrics["tabWidth"]) lineHeight: int = int(metrics["lineHeight"]) baselineOffset: int = int(metrics["baselineOffset"]) - entries: dict[int, int] = {} + hashTable: HashTableBuilder = HashTableBuilder(numBuckets) for ch, entry in metrics["characterSizes"].items(): x: int = int(entry["x"]) @@ -156,7 +159,7 @@ def generateFontMetrics(metrics: Mapping[str, Any]) -> bytearray: if h > lineHeight: raise ValueError("character height exceeds line height") - entries[ord(ch)] = (0 + hashTable.addEntry(ord(ch), 0 | (x << 0) | (y << 8) | (w << 16) @@ -164,30 +167,26 @@ def generateFontMetrics(metrics: Mapping[str, Any]) -> bytearray: | (i << 30) ) - buckets, chained = generateHashTable(entries, _METRICS_BUCKET_COUNT) - table: bytearray = bytearray() - - if (len(buckets) + len(chained)) > 2048: - raise RuntimeError("font hash table must have <=2048 entries") - - table += _METRICS_HEADER_STRUCT.pack( + metrics: bytearray = bytearray() + metrics += _METRICS_HEADER_STRUCT.pack( spaceWidth, tabWidth, lineHeight, - baselineOffset + baselineOffset, + numBuckets, + len(hashTable.entries) ) - for entry in chain(buckets, chained): + for entry in hashTable.entries: if entry is None: - table += _METRICS_ENTRY_STRUCT.pack(0, 0) - continue + metrics += bytes(_METRICS_ENTRY_STRUCT.size) + else: + metrics += _METRICS_ENTRY_STRUCT.pack( + entry.fullHash | (entry.chainIndex << 21), + entry.data + ) - table += _METRICS_ENTRY_STRUCT.pack( - entry.fullHash | (entry.chainIndex << 21), - entry.data - ) - - return table + return metrics ## Color palette generator @@ -235,65 +234,108 @@ def generateColorPalette( ## String table generator -_STRING_TABLE_ENTRY_STRUCT: Struct = Struct("< I 2H") -_STRING_TABLE_BUCKET_COUNT: int = 256 -_STRING_TABLE_ALIGNMENT: int = 4 +_STRING_TABLE_HEADER_STRUCT: Struct = Struct("< 2H") +_STRING_TABLE_ENTRY_STRUCT: Struct = Struct("< I 2H") +_STRING_TABLE_ALIGNMENT: int = 4 def _walkStringTree( strings: Mapping[str, Any], prefix: str = "" -) -> Generator[tuple[int, bytes | None], None, None]: +) -> Generator[tuple[int, bytes], None, None]: for key, value in strings.items(): fullKey: str = prefix + key keyHash: int = hashData(fullKey.encode("ascii")) - if value is None: - yield keyHash, None - elif isinstance(value, str): - yield keyHash, value.encode("utf-8") + if isinstance(value, str): + yield keyHash, value.encode("utf-8") + b"\0" else: yield from _walkStringTree(value, f"{fullKey}.") -def generateStringTable(strings: Mapping[str, Any]) -> bytearray: - offsets: dict[bytes, int] = {} - entries: dict[int, int] = {} - blob: bytearray = bytearray() +def generateStringTable( + strings: Mapping[str, Any], numBuckets: int = 256 +) -> bytearray: + hashTable: HashTableBuilder = HashTableBuilder(numBuckets) + blob: StringBlobBuilder = StringBlobBuilder(_STRING_TABLE_ALIGNMENT) for keyHash, string in _walkStringTree(strings): - if string is None: - entries[keyHash] = 0 - continue + hashTable.addEntry(keyHash, blob.addString(string)) - # Identical strings associated to multiple keys are deduplicated. - offset: int | None = offsets.get(string, None) + tableLength: int = 0 \ + + _STRING_TABLE_HEADER_STRUCT.size \ + + _STRING_TABLE_ENTRY_STRUCT.size * len(hashTable.entries) - if offset is None: - offset = len(blob) - offsets[string] = offset + tableData: bytearray = bytearray() + tableData += _STRING_TABLE_HEADER_STRUCT.pack( + numBuckets, len(hashTable.entries) + ) - blob += string - blob.append(0) - - while len(blob) % _STRING_TABLE_ALIGNMENT: - blob.append(0) - - entries[keyHash] = offset - - buckets, chained = generateHashTable(entries, _STRING_TABLE_BUCKET_COUNT) - table: bytearray = bytearray() - - # Relocate the offsets and serialize the table. - blobOffset: int = \ - (len(buckets) + len(chained)) * _STRING_TABLE_ENTRY_STRUCT.size - - for entry in chain(buckets, chained): + for entry in hashTable.entries: if entry is None: - table += _STRING_TABLE_ENTRY_STRUCT.pack(0, 0, 0) - continue + tableData += bytes(_STRING_TABLE_ENTRY_STRUCT.size) + else: + tableData += _STRING_TABLE_ENTRY_STRUCT.pack( + entry.fullHash, + tableLength + entry.data, + entry.chainIndex + ) - table += _STRING_TABLE_ENTRY_STRUCT.pack( - entry.fullHash, - 0 if (entry.data is None) else (blobOffset + entry.data), - entry.chainIndex + return tableData + blob.data + +## Package header generator + +_PACKAGE_INDEX_HEADER_STRUCT: Struct = Struct("< I 2H") +_PACKAGE_INDEX_ENTRY_STRUCT: Struct = Struct("< I 2H Q 2I") +_PACKAGE_STRING_ALIGNMENT: int = 4 + +@dataclass +class PackageIndexEntry: + offset: int + compLength: int + uncompLength: int + nameOffset: int = 0 + +def generatePackageIndex( + files: Mapping[str, PackageIndexEntry], alignment: int = 2048, + numBuckets: int = 256 +) -> bytearray: + hashTable: HashTableBuilder = HashTableBuilder(numBuckets) + blob: StringBlobBuilder = StringBlobBuilder(_PACKAGE_STRING_ALIGNMENT) + + for name, entry in files.items(): + nameString: bytes = name.encode("ascii") + data: PackageIndexEntry = PackageIndexEntry( + entry.offset, + entry.compLength, + entry.uncompLength, + blob.addString(nameString + b"\0") ) - return table + blob + hashTable.addEntry(hashData(nameString), data) + + tableLength: int = 0 \ + + _PACKAGE_INDEX_HEADER_STRUCT.size \ + + _PACKAGE_INDEX_ENTRY_STRUCT.size * len(hashTable.entries) + indexLength: int = tableLength + len(blob.data) + + tableData: bytearray = bytearray() + tableData += _PACKAGE_INDEX_HEADER_STRUCT.pack( + indexLength, + numBuckets, + len(hashTable.entries) + ) + + fileDataOffset: int = roundUpToMultiple(indexLength, alignment) + + for entry in hashTable.entries: + if entry is None: + tableData += bytes(_PACKAGE_INDEX_ENTRY_STRUCT.size) + else: + tableData += _PACKAGE_INDEX_ENTRY_STRUCT.pack( + entry.fullHash, + tableLength + entry.data.nameOffset, + entry.chainIndex, + fileDataOffset + entry.data.offset, + entry.data.compLength, + entry.data.uncompLength + ) + + return tableData + blob.data diff --git a/tools/common/util.py b/tools/common/util.py index b2ba0c7..d5fc161 100644 --- a/tools/common/util.py +++ b/tools/common/util.py @@ -15,18 +15,23 @@ # 573in1. If not, see . import logging, re -from collections import defaultdict from dataclasses import dataclass from hashlib import md5 from io import SEEK_END, SEEK_SET from typing import \ - Any, BinaryIO, ByteString, Generator, Iterable, Iterator, Mapping, \ - Sequence, TextIO + Any, BinaryIO, ByteString, Iterable, Iterator, Sequence, TextIO ## Value manipulation +def roundUpToMultiple(value: int, length: int) -> int: + diff: int = value % length + + return (value - diff + length) if diff else value + def encodeSigned(value: int, bitLength: int) -> int: - return value & (1 << bitLength) + valueMask: int = (1 << bitLength) - 1 + + return value & valueMask def decodeSigned(value: int, bitLength: int) -> int: signMask: int = 1 << (bitLength - 1) @@ -153,35 +158,60 @@ class HashTableEntry: chainIndex: int data: Any -def generateHashTable( - entries: Mapping[int, Any], numBuckets: int -) -> tuple[list[HashTableEntry | None], list[HashTableEntry]]: - chains: defaultdict[int, list[HashTableEntry]] = defaultdict(list) +class HashTableBuilder: + def __init__(self, numBuckets: int = 256): + self._numBuckets: int = numBuckets - for fullHash, data in entries.items(): - entry: HashTableEntry = HashTableEntry(fullHash, 0, data) + self.entries: list[HashTableEntry | None] = [ None ] * numBuckets - chains[fullHash % numBuckets].append(entry) + def addEntry(self, fullHash: int, data: Any) -> int: + index: int = fullHash % self._numBuckets - buckets: list[HashTableEntry | None] = [] - chained: list[HashTableEntry] = [] + entry: HashTableEntry = HashTableEntry(fullHash, 0, data) + bucket: HashTableEntry | None = self.entries[index] - for shortHash in range(numBuckets): - entries: list[HashTableEntry] = chains[shortHash] + # If no bucket exists for the entry's index, create one. + if bucket is None: + self.entries[index] = entry + return index + if bucket.fullHash == fullHash: + raise KeyError(f"collision detected, hash={fullHash:#08x}") - if not len(entries): # Empty bucket - buckets.append(None) - continue + # Otherwise, follow the buckets's chain, find the last chained item and + # link the new entry to it. + while bucket.chainIndex: + bucket = self.entries[bucket.chainIndex] - for index, entry in enumerate(entries): - entry.chainIndex = numBuckets + len(chained) + index + if bucket.fullHash == fullHash: + raise KeyError(f"collision detected, hash={fullHash:#08x}") - entries[-1].chainIndex = 0 # Terminate chain + bucket.chainIndex = len(self.entries) + self.entries.append(entry) - buckets.append(entries[0]) - chained += entries[1:] + return bucket.chainIndex - return buckets, chained +class StringBlobBuilder: + def __init__(self, alignment: int = 1): + self._alignment: int = alignment + self._offsets: dict[ByteString, int] = {} + + self.data: bytearray = bytearray() + + def addString(self, string: ByteString) -> int: + # If the same string is already in the blob, return its offset without + # adding new data. + offset: int | None = self._offsets.get(string, None) + + if offset is None: + offset = len(self.data) + + self._offsets[string] = offset + self.data += string + + while len(self.data) % self._alignment: + self.data.append(0) + + return offset ## Odd/even interleaved file reader