diff --git a/README.md b/README.md index 9833965..5b2a8fa 100644 --- a/README.md +++ b/README.md @@ -17,24 +17,32 @@ Install Python, then: ## Usage ``` -usage: ifstools [-h] [-y] [-o OUT_DIR] [--tex-only] [--nocache] [-s] [-r] - file.ifs|folder_ifs [file.ifs|folder_ifs ...] +usage: ifstools [-h] [-e] [-y] [-o OUT_DIR] [--tex-only] [--nocache] [-m] [-s] + [-r] + file_to_unpack.ifs|folder_to_repack_ifs + [file_to_unpack.ifs|folder_to_repack_ifs ...] Unpack/pack IFS files and textures positional arguments: - file.ifs|folder_ifs files/folders to process. Files will be unpacked, - folders will be repacked + file_to_unpack.ifs|folder_to_repack_ifs + files/folders to process. Files will be unpacked, + folders will be repacked optional arguments: - -h, --help show this help message and exit - -y don't prompt for file/folder overwrite - -o OUT_DIR output directory - --tex-only only extract textures - --nocache ignore texture cache, recompress all - -s, --silent don't display files as they are processed - -r, --norecurse if file contains another IFS, don't extract its - contents + -h, --help show this help message and exit + -e, --extract-folders + do not repack folders, instead unpack any IFS files + inside them + -y don't prompt for file/folder overwrite + -o OUT_DIR output directory + --tex-only only extract textures + --nocache ignore texture cache, recompress all + -m, --extract-manifest + extract the IFS manifest for inspection + -s, --silent don't display files as they are processed + -r, --norecurse if file contains another IFS, don't extract its + contents ``` Notes: diff --git a/ifstools/handlers/GenericFile.py b/ifstools/handlers/GenericFile.py index 904e9d6..a14b783 100644 --- a/ifstools/handlers/GenericFile.py +++ b/ifstools/handlers/GenericFile.py @@ -60,6 +60,10 @@ class GenericFile(Node): # offset, size, timestamp elem.text = '{} {} {}'.format(len(data_blob.getvalue()), len(data), self.time) data_blob.write(data) + # 16 byte alignment + align = len(data) % 16 + if align: + data_blob.write(b'\0' * (16-align)) @property def disk_path(self): diff --git a/ifstools/handlers/GenericFolder.py b/ifstools/handlers/GenericFolder.py index af0c441..f563bf2 100644 --- a/ifstools/handlers/GenericFolder.py +++ b/ifstools/handlers/GenericFolder.py @@ -1,5 +1,6 @@ from itertools import chain from os.path import getmtime, basename, join +from collections import OrderedDict import lxml.etree as etree @@ -24,7 +25,7 @@ class GenericFolder(Node): if element.text: self.time = int(element.text) - self.files = {} + self.files = OrderedDict() self.folders = {} for child in element.iterchildren(tag=etree.Element): filename = Node.fix_name(child.tag) diff --git a/ifstools/handlers/ImageFile.py b/ifstools/handlers/ImageFile.py index 7f6daa9..7e0f3bc 100644 --- a/ifstools/handlers/ImageFile.py +++ b/ifstools/handlers/ImageFile.py @@ -103,6 +103,10 @@ class ImageFile(GenericFile): elem.attrib['__type'] = '3s32' elem.text = '{} {} {}'.format(len(data_blob.getvalue()), len(data), self.time) data_blob.write(data) + # 16 byte alignment + align = len(data) % 16 + if align: + data_blob.write(b'\0' * (16-align)) def _load_im(self): data = self.load() diff --git a/ifstools/ifs.py b/ifstools/ifs.py index fe9953e..2f56758 100644 --- a/ifstools/ifs.py +++ b/ifstools/ifs.py @@ -15,7 +15,6 @@ from .handlers import GenericFolder, MD5Folder, ImageFile from . import utils SIGNATURE = 0x6CAD8F89 -HEADER_SIZE = 36 FILE_VERSION = 3 @@ -62,9 +61,12 @@ class IFS: ifs_tree_size = file.get_u32() manifest_end = file.get_u32() self.data_blob = bytes(file.data[manifest_end:]) - # 16 bytes for manifest md5, unchecked - self.manifest = KBinXML(file.data[HEADER_SIZE:]) + if self.file_version > 1: + # md5 of manifest, unchecked + file.offset += 16 + + self.manifest = KBinXML(file.data[file.offset:]) self.tree = GenericFolder(self.data_blob, self.manifest.xml_doc) # IFS files repacked with other tools usually have wrong values - don't validate this @@ -111,7 +113,8 @@ class IFS: def __str__(self): return str(self.tree) - def extract(self, progress = True, use_cache = True, recurse = True, tex_only = False, path = None): + def extract(self, progress = True, use_cache = True, recurse = True, + tex_only = False, extract_manifest = False, path = None): if path is None: path = self.folder_out if tex_only and 'tex' not in self.tree.folders: @@ -119,7 +122,7 @@ class IFS: utils.mkdir_silent(path) utime(path, (self.time, self.time)) - if self.manifest and not tex_only: + if extract_manifest and self.manifest and not tex_only: with open(join(path, 'ifs_manifest.xml'), 'wb') as f: f.write(self.manifest.to_text().encode('utf8')) @@ -141,7 +144,8 @@ class IFS: if recurse and f.name.endswith('.ifs'): rpath = join(path, f.full_path) i = IFS(rpath) - i.extract(progress, use_cache, recurse, tex_only, rpath.replace('.ifs','_ifs')) + i.extract(progress=progress, use_cache=use_cache, recurse=recurse, + tex_only=tex_only, extract_manifest=extract_manifest, path=rpath.replace('.ifs','_ifs')) ''' If you can get shared memory for IFS.data_blob working, this will be a lot faster. As it is, it gets pickled for every file, and @@ -179,7 +183,6 @@ class IFS: data_size.text = str(len(data)) manifest_bin = self.manifest.to_binary() - manifest_end = HEADER_SIZE + len(manifest_bin) manifest_hash = hashlib.md5(manifest_bin).digest() head = ByteBuffer() @@ -188,10 +191,17 @@ class IFS: head.append_u16(self.file_version ^ 0xFFFF) head.append_u32(int(unixtime())) head.append_u32(self.manifest.mem_size) + + manifest_end = len(manifest_bin) + head.offset + 4 + if self.file_version > 1: + manifest_end += 16 + head.append_u32(manifest_end) + if self.file_version > 1: + head.append_bytes(manifest_hash) + ifs_file.write(head.data) - ifs_file.write(manifest_hash) ifs_file.write(manifest_bin) ifs_file.write(data) diff --git a/ifstools/ifstools.py b/ifstools/ifstools.py index 0620024..bc809f2 100644 --- a/ifstools/ifstools.py +++ b/ifstools/ifstools.py @@ -21,14 +21,28 @@ def get_choice(prompt): else: print('Please answer y/n') +def extract(i, args, path): + if args.progress: + print('Extracting...') + i.extract(progress = args.progress, use_cache = args.use_cache, + recurse = args.recurse, tex_only = args.tex_only, path = path, + extract_manifest = args.extract_manifest) + +def repack(i, args, path): + if args.progress: + print('Repacking...') + i.repack(progress = args.progress, use_cache = args.use_cache, path = path) + def main(): parser = argparse.ArgumentParser(description='Unpack/pack IFS files and textures') - parser.add_argument('files', metavar='file.ifs|folder_ifs', type=str, nargs='+', + parser.add_argument('files', metavar='file_to_unpack.ifs|folder_to_repack_ifs', type=str, nargs='+', help='files/folders to process. Files will be unpacked, folders will be repacked') + parser.add_argument('-e', '--extract-folders', action='store_true', help='do not repack folders, instead unpack any IFS files inside them', dest='extract_folders') parser.add_argument('-y', action='store_true', help='don\'t prompt for file/folder overwrite', dest='overwrite') parser.add_argument('-o', default='.', help='output directory', dest='out_dir') parser.add_argument('--tex-only', action='store_true', help='only extract textures', dest='tex_only') parser.add_argument('--nocache', action='store_false', help='ignore texture cache, recompress all', dest='use_cache') + parser.add_argument('-m', '--extract-manifest', action='store_true', help='extract the IFS manifest for inspection', dest='extract_manifest') parser.add_argument('-s', '--silent', action='store_false', dest='progress', help='don\'t display files as they are processed') parser.add_argument('-r', '--norecurse', action='store_false', dest='recurse', @@ -36,6 +50,14 @@ def main(): args = parser.parse_args() + if args.extract_folders: + dirs = [f for f in args.files if os.path.isdir(f)] + # prune + args.files = [f for f in args.files if not os.path.isdir(f)] + # add the extras + for d in dirs: + args.files.extend((os.path.join(d,f) for f in os.listdir(d) if f.lower().endswith('.ifs'))) + for f in args.files: if args.progress: print(f) @@ -52,13 +74,9 @@ def main(): continue if i.is_file: - if args.progress: - print('Extracting...') - i.extract(args.progress, args.use_cache, args.recurse, args.tex_only, path) + extract(i, args, path) else: - if args.progress: - print('Repacking...') - i.repack(args.progress, args.use_cache, path) + repack(i, args, path) if __name__ == '__main__': diff --git a/setup.py b/setup.py index 324e68f..18d9c7b 100644 --- a/setup.py +++ b/setup.py @@ -6,12 +6,12 @@ requires = [ 'lxml', 'tqdm', 'pillow', - 'kbinxml>=1.2', + 'kbinxml>=1.4', ] if sys.version_info < (3,0): requires.append('future') -version = '1.2' +version = '1.3' setup( name='ifstools', description='Extractor/repacker for Konmai IFS files',