From 21488cc3c002c14e79f312e9d8b6bdea363b66f0 Mon Sep 17 00:00:00 2001 From: Will Toohey Date: Tue, 9 Jan 2018 17:22:36 +1000 Subject: [PATCH] Much wider compatibility, PyPi release --- README.md | 3 ++- ifstools/handlers/AfpFolder.py | 8 ++++++-- ifstools/handlers/GenericFolder.py | 14 +++++++++++--- ifstools/handlers/MD5Folder.py | 10 +++++++--- ifstools/handlers/TexFolder.py | 13 ++++++++----- ifstools/ifs.py | 26 ++++---------------------- ifstools/ifstools.py | 4 ++++ requirements.txt | 2 +- setup.cfg | 2 ++ setup.py | 7 ++++++- 10 files changed, 51 insertions(+), 38 deletions(-) create mode 100644 setup.cfg diff --git a/README.md b/README.md index 373d766..9833965 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,8 @@ Extractor for Konmai IFS files. - Dumps the ifs manifest so you can explore the format ## Install -`pip install git+https://github.com/mon/kbinxml/ git+https://github.com/mon/ifstools/` +Install Python, then: +`pip install ifstools` ## Usage ``` diff --git a/ifstools/handlers/AfpFolder.py b/ifstools/handlers/AfpFolder.py index 9343398..1bf06f7 100644 --- a/ifstools/handlers/AfpFolder.py +++ b/ifstools/handlers/AfpFolder.py @@ -4,6 +4,8 @@ class AfpFolder(MD5Folder): def tree_complete(self): MD5Folder.tree_complete(self) + if not self.info_kbin: + return # findall needs xpath or it'll only search direct children names = [] @@ -15,5 +17,7 @@ class AfpFolder(MD5Folder): for shape in self._split_ints(geo.text): geo_names.append('{}_shape{}'.format(name, shape)) - self._apply_md5_folder(names, self.folders['bsi']) - self._apply_md5_folder(geo_names, self.parent.folders['geo']) + if 'bsi' in self.folders: + self._apply_md5_folder(names, self.folders['bsi']) + if 'geo' in self.parent.folders: + self._apply_md5_folder(geo_names, self.parent.folders['geo']) diff --git a/ifstools/handlers/GenericFolder.py b/ifstools/handlers/GenericFolder.py index eed7012..af0c441 100644 --- a/ifstools/handlers/GenericFolder.py +++ b/ifstools/handlers/GenericFolder.py @@ -8,13 +8,14 @@ from .Node import Node class GenericFolder(Node): - def __init__(self, ifs_data, obj, parent = None, path = '', name = ''): + def __init__(self, ifs_data, obj, parent = None, path = '', name = '', has_super = False): # circular dependencies mean we import here from . import AfpFolder, TexFolder self.folder_handlers = { 'afp' : AfpFolder, 'tex' : TexFolder, } + self.has_super = has_super Node.__init__(self, ifs_data, obj, parent, path, name) file_handler = GenericFile @@ -29,9 +30,16 @@ class GenericFolder(Node): filename = Node.fix_name(child.tag) if filename == '_info_': # metadata continue - elif list(child): # folder + elif filename == '_super_': + self.has_super = True + # folder: has children or timestamp only + elif list(child) or len(child.text.split(' ')) == 1: + # note: files with 'super' references have 'i' tags as backrefs + # We just ignore these + if self.has_super and child[0].tag == 'i': + continue handler = self.folder_handlers.get(filename, GenericFolder) - self.folders[filename] = handler(self.ifs_data, child, self, self.full_path, filename) + self.folders[filename] = handler(self.ifs_data, child, self, self.full_path, filename, self.has_super) else: # file self.files[filename] = self.file_handler(self.ifs_data, child, self, self.full_path, filename) diff --git a/ifstools/handlers/MD5Folder.py b/ifstools/handlers/MD5Folder.py index 1e8e42e..b4a0d6d 100644 --- a/ifstools/handlers/MD5Folder.py +++ b/ifstools/handlers/MD5Folder.py @@ -6,20 +6,24 @@ from . import GenericFolder class MD5Folder(GenericFolder): - def __init__(self, ifs_data, parent, obj, path = '', name = '', md5_tag = None, extension = None): - GenericFolder.__init__(self, ifs_data, parent, obj, path, name) + def __init__(self, ifs_data, parent, obj, path = '', name = '', has_super = False, md5_tag = None, extension = None): + GenericFolder.__init__(self, ifs_data, parent, obj, path, name, has_super) self.md5_tag = md5_tag if md5_tag else self.name self.extension = extension def tree_complete(self): GenericFolder.tree_complete(self) + self.info_kbin = None + self.info_file = None for filename, file in self.files.items(): if filename.endswith('.xml'): self.info_file = file break if not self.info_file: - raise KeyError('MD5 folder contents have no mapping xml') + #raise KeyError('MD5 folder contents have no mapping xml') + # _super_ references to info XML breaks things - just extract what we can + return self.info_kbin = KBinXML(self.info_file.load(convert_kbin = False)) self._apply_md5() diff --git a/ifstools/handlers/TexFolder.py b/ifstools/handlers/TexFolder.py index cc156c1..f87fe27 100644 --- a/ifstools/handlers/TexFolder.py +++ b/ifstools/handlers/TexFolder.py @@ -13,17 +13,19 @@ class TextureList(GenericFile): return k.to_binary() class TexFolder(MD5Folder): - def __init__(self, ifs_data, obj, parent = None, path = '', name = ''): - MD5Folder.__init__(self, ifs_data, obj, parent, path, name, 'image', '.png') + def __init__(self, ifs_data, obj, parent = None, path = '', name = '', has_super = False): + MD5Folder.__init__(self, ifs_data, obj, parent, path, name, has_super, 'image', '.png') def tree_complete(self): MD5Folder.tree_complete(self) + if '_cache' in self.folders: + self.folders.pop('_cache') + if not self.info_kbin: + return self.compress = self.info_kbin.xml_doc.attrib.get('compress') self.info_file.__class__ = TextureList - if '_cache' in self.folders: - self.folders.pop('_cache') self._create_images() def _create_images(self): @@ -35,6 +37,7 @@ class TexFolder(MD5Folder): continue elif indiv.tag == 'image': name = indiv.attrib['name'] + '.png' - ImageFile.upgrade_generic(self.files[name], indiv, fmt, self.compress) + if name in self.files: + ImageFile.upgrade_generic(self.files[name], indiv, fmt, self.compress) else: print('Unknown texturelist.xml element {}'.format(indiv.tag)) diff --git a/ifstools/ifs.py b/ifstools/ifs.py index 1fd73d7..fe9953e 100644 --- a/ifstools/ifs.py +++ b/ifstools/ifs.py @@ -67,9 +67,8 @@ class IFS: self.manifest = KBinXML(file.data[HEADER_SIZE:]) self.tree = GenericFolder(self.data_blob, self.manifest.xml_doc) - if ifs_tree_size != self.tree_size: - print('Expected tree size {} but got {}. Repacking may fail!' - .format(self.tree_size, ifs_tree_size)) + # IFS files repacked with other tools usually have wrong values - don't validate this + #assert ifs_tree_size == self.manifest.mem_size def load_dir(self, path): self.is_file = False @@ -142,7 +141,7 @@ class IFS: if recurse and f.name.endswith('.ifs'): rpath = join(path, f.full_path) i = IFS(rpath) - i.extract(progress, use_cache, recurse, rpath.replace('.ifs','_ifs')) + i.extract(progress, use_cache, recurse, tex_only, rpath.replace('.ifs','_ifs')) ''' If you can get shared memory for IFS.data_blob working, this will be a lot faster. As it is, it gets pickled for every file, and @@ -188,7 +187,7 @@ class IFS: head.append_u16(self.file_version) head.append_u16(self.file_version ^ 0xFFFF) head.append_u32(int(unixtime())) - head.append_u32(self.tree_size) + head.append_u32(self.manifest.mem_size) head.append_u32(manifest_end) ifs_file.write(head.data) @@ -227,20 +226,3 @@ class IFS: self.tree.repack(self.manifest.xml_doc, self.data_blob, tqdm_progress) return self.data_blob.getvalue() - - # suspected to be the in-memory representation - @property - def tree_size(self): - BASE_SIZE = 856 - return BASE_SIZE + self._tree_size_recurse(self.tree) - - def _tree_size_recurse(self, tree, depth = 0): - FILE = 64 - FOLDER = 56 - DEPTH_MULTIPLIER = 16 - - size = len(tree.files) * FILE - size += len(tree.folders) * (FOLDER - depth*DEPTH_MULTIPLIER) - for name, folder in tree.folders.items(): - size += self._tree_size_recurse(folder, depth+1) - return size diff --git a/ifstools/ifstools.py b/ifstools/ifstools.py index f07b8be..0620024 100644 --- a/ifstools/ifstools.py +++ b/ifstools/ifstools.py @@ -52,8 +52,12 @@ def main(): continue if i.is_file: + if args.progress: + print('Extracting...') i.extract(args.progress, args.use_cache, args.recurse, args.tex_only, path) else: + if args.progress: + print('Repacking...') i.repack(args.progress, args.use_cache, path) diff --git a/requirements.txt b/requirements.txt index 75660a2..bd8862b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ lxml tqdm pillow future -git+https://github.com/mon/kbinxml.git \ No newline at end of file +kbinxml>=1.2 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..224a779 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +description-file = README.md \ No newline at end of file diff --git a/setup.py b/setup.py index b9eecb2..324e68f 100644 --- a/setup.py +++ b/setup.py @@ -6,18 +6,23 @@ requires = [ 'lxml', 'tqdm', 'pillow', + 'kbinxml>=1.2', ] if sys.version_info < (3,0): requires.append('future') +version = '1.2' setup( name='ifstools', - version='1.1', + description='Extractor/repacker for Konmai IFS files', + long_description='See Github for up to date documentation', + version=version, entry_points = { 'console_scripts': ['ifstools=ifstools:main'], }, packages=['ifstools', 'ifstools.handlers'], url='https://github.com/mon/ifstools/', + download_url = 'https://github.com/mon/ifstools/archive/{}.tar.gz'.format(version), author='mon', author_email='me@mon.im', install_requires=requires,