Much wider compatibility, PyPi release

README.md

@@ -12,7 +12,8 @@ Extractor for Konmai IFS files.
 - Dumps the ifs manifest so you can explore the format
 
 ## Install
-`pip install git+https://github.com/mon/kbinxml/ git+https://github.com/mon/ifstools/`
+Install Python, then:
+`pip install ifstools`
 
 ## Usage
 ```

ifstools/handlers/AfpFolder.py

@@ -4,6 +4,8 @@ class AfpFolder(MD5Folder):
 
     def tree_complete(self):
         MD5Folder.tree_complete(self)
+        if not self.info_kbin:
+            return
 
         # findall needs xpath or it'll only search direct children
         names = []
@@ -15,5 +17,7 @@ class AfpFolder(MD5Folder):
                 for shape in self._split_ints(geo.text):
                     geo_names.append('{}_shape{}'.format(name, shape))
 
-        self._apply_md5_folder(names, self.folders['bsi'])
+        if 'bsi' in self.folders:
-        self._apply_md5_folder(geo_names, self.parent.folders['geo'])
+            self._apply_md5_folder(names, self.folders['bsi'])
+        if 'geo' in self.parent.folders:
+            self._apply_md5_folder(geo_names, self.parent.folders['geo'])

ifstools/handlers/GenericFolder.py

@@ -8,13 +8,14 @@ from .Node import Node
 
 class GenericFolder(Node):
 
-    def __init__(self, ifs_data, obj, parent = None, path = '', name = ''):
+    def __init__(self, ifs_data, obj, parent = None, path = '', name = '', has_super = False):
         # circular dependencies mean we import here
         from . import AfpFolder, TexFolder
         self.folder_handlers = {
             'afp' : AfpFolder,
             'tex' : TexFolder,
         }
+        self.has_super = has_super
         Node.__init__(self, ifs_data, obj, parent, path, name)
 
     file_handler = GenericFile
@@ -29,9 +30,16 @@ class GenericFolder(Node):
             filename = Node.fix_name(child.tag)
             if filename == '_info_': # metadata
                 continue
-            elif list(child): # folder
+            elif filename == '_super_':
+                self.has_super = True
+            # folder: has children or timestamp only
+            elif list(child) or len(child.text.split(' ')) == 1:
+                # note: files with 'super' references have 'i' tags as backrefs
+                # We just ignore these
+                if self.has_super and child[0].tag == 'i':
+                    continue
                 handler = self.folder_handlers.get(filename, GenericFolder)
-                self.folders[filename] = handler(self.ifs_data, child, self, self.full_path, filename)
+                self.folders[filename] = handler(self.ifs_data, child, self, self.full_path, filename, self.has_super)
             else: # file
                 self.files[filename] = self.file_handler(self.ifs_data, child, self, self.full_path, filename)
 

ifstools/handlers/MD5Folder.py

@@ -6,20 +6,24 @@ from . import GenericFolder
 
 class MD5Folder(GenericFolder):
 
-    def __init__(self, ifs_data, parent, obj, path = '', name = '', md5_tag = None, extension = None):
+    def __init__(self, ifs_data, parent, obj, path = '', name = '', has_super = False, md5_tag = None, extension = None):
-        GenericFolder.__init__(self, ifs_data, parent, obj, path, name)
+        GenericFolder.__init__(self, ifs_data, parent, obj, path, name, has_super)
         self.md5_tag = md5_tag if md5_tag else self.name
         self.extension = extension
 
     def tree_complete(self):
         GenericFolder.tree_complete(self)
 
+        self.info_kbin = None
+        self.info_file = None
         for filename, file in self.files.items():
             if filename.endswith('.xml'):
                 self.info_file = file
                 break
         if not self.info_file:
-            raise KeyError('MD5 folder contents have no mapping xml')
+            #raise KeyError('MD5 folder contents have no mapping xml')
+            # _super_ references to info XML breaks things - just extract what we can
+            return
 
         self.info_kbin = KBinXML(self.info_file.load(convert_kbin = False))
         self._apply_md5()

ifstools/handlers/TexFolder.py

@@ -13,17 +13,19 @@ class TextureList(GenericFile):
         return k.to_binary()
 
 class TexFolder(MD5Folder):
-    def __init__(self, ifs_data, obj, parent = None, path = '', name = ''):
+    def __init__(self, ifs_data, obj, parent = None, path = '', name = '', has_super = False):
-        MD5Folder.__init__(self, ifs_data, obj, parent, path, name, 'image', '.png')
+        MD5Folder.__init__(self, ifs_data, obj, parent, path, name, has_super, 'image', '.png')
 
     def tree_complete(self):
         MD5Folder.tree_complete(self)
+        if '_cache' in self.folders:
+            self.folders.pop('_cache')
+        if not self.info_kbin:
+            return
 
         self.compress = self.info_kbin.xml_doc.attrib.get('compress')
         self.info_file.__class__ = TextureList
 
-        if '_cache' in self.folders:
-            self.folders.pop('_cache')
         self._create_images()
 
     def _create_images(self):
@@ -35,6 +37,7 @@ class TexFolder(MD5Folder):
                     continue
                 elif indiv.tag == 'image':
                     name = indiv.attrib['name'] + '.png'
-                    ImageFile.upgrade_generic(self.files[name], indiv, fmt, self.compress)
+                    if name in self.files:
+                        ImageFile.upgrade_generic(self.files[name], indiv, fmt, self.compress)
                 else:
                     print('Unknown texturelist.xml element {}'.format(indiv.tag))

ifstools/ifs.py

@@ -67,9 +67,8 @@ class IFS:
         self.manifest = KBinXML(file.data[HEADER_SIZE:])
         self.tree = GenericFolder(self.data_blob, self.manifest.xml_doc)
 
-        if ifs_tree_size != self.tree_size:
+        # IFS files repacked with other tools usually have wrong values - don't validate this
-            print('Expected tree size {} but got {}. Repacking may fail!'
+        #assert ifs_tree_size == self.manifest.mem_size
-                .format(self.tree_size, ifs_tree_size))
 
     def load_dir(self, path):
         self.is_file = False
@@ -142,7 +141,7 @@ class IFS:
             if recurse and f.name.endswith('.ifs'):
                 rpath = join(path, f.full_path)
                 i = IFS(rpath)
-                i.extract(progress, use_cache, recurse, rpath.replace('.ifs','_ifs'))
+                i.extract(progress, use_cache, recurse, tex_only, rpath.replace('.ifs','_ifs'))
 
         ''' If you can get shared memory for IFS.data_blob working, this will
             be a lot faster. As it is, it gets pickled for every file, and
@@ -188,7 +187,7 @@ class IFS:
         head.append_u16(self.file_version)
         head.append_u16(self.file_version ^ 0xFFFF)
         head.append_u32(int(unixtime()))
-        head.append_u32(self.tree_size)
+        head.append_u32(self.manifest.mem_size)
         head.append_u32(manifest_end)
 
         ifs_file.write(head.data)
@@ -227,20 +226,3 @@ class IFS:
         self.tree.repack(self.manifest.xml_doc, self.data_blob, tqdm_progress)
 
         return self.data_blob.getvalue()
-
-    # suspected to be the in-memory representation
-    @property
-    def tree_size(self):
-        BASE_SIZE = 856
-        return BASE_SIZE + self._tree_size_recurse(self.tree)
-
-    def _tree_size_recurse(self, tree, depth = 0):
-        FILE = 64
-        FOLDER = 56
-        DEPTH_MULTIPLIER = 16
-
-        size = len(tree.files) * FILE
-        size += len(tree.folders) * (FOLDER - depth*DEPTH_MULTIPLIER)
-        for name, folder in tree.folders.items():
-            size += self._tree_size_recurse(folder, depth+1)
-        return size

ifstools/ifstools.py

@@ -52,8 +52,12 @@ def main():
                 continue
 
         if i.is_file:
+            if args.progress:
+                print('Extracting...')
             i.extract(args.progress, args.use_cache, args.recurse, args.tex_only, path)
         else:
+            if args.progress:
+                print('Repacking...')
             i.repack(args.progress, args.use_cache, path)
 
 

requirements.txt

@@ -2,4 +2,4 @@ lxml
 tqdm
 pillow
 future
-git+https://github.com/mon/kbinxml.git
+kbinxml>=1.2

setup.cfg

@@ -0,0 +1,2 @@
+[metadata]
+description-file = README.md

setup.py

@@ -6,18 +6,23 @@ requires = [
         'lxml',
         'tqdm',
         'pillow',
+        'kbinxml>=1.2',
 ]
 if sys.version_info < (3,0):
     requires.append('future')
 
+version = '1.2'
 setup(
     name='ifstools',
-    version='1.1',
+    description='Extractor/repacker for Konmai IFS files',
+    long_description='See Github for up to date documentation',
+    version=version,
     entry_points = {
         'console_scripts': ['ifstools=ifstools:main'],
     },
     packages=['ifstools', 'ifstools.handlers'],
     url='https://github.com/mon/ifstools/',
+    download_url = 'https://github.com/mon/ifstools/archive/{}.tar.gz'.format(version),
     author='mon',
     author_email='me@mon.im',
     install_requires=requires,