1
0
mirror of https://github.com/mon/ifstools.git synced 2024-11-24 01:50:10 +01:00

Repacking and better compression tools

This commit is contained in:
Will Toohey 2017-12-18 19:37:16 +10:00
parent da489545c8
commit e0125caf09
9 changed files with 430 additions and 139 deletions

View File

@ -1,15 +1,19 @@
# ifstools # ifstools
Extractor for Konmai IFS files. Extractor for Konmai IFS files.
Requires [kbinxml](https://github.com/mon/kbinxml/).
Features: Features:
- Converts all textures to png without requiring a second program - Converts all textures to png without requiring a second program
- Repacks without ingame display issues
- Works on eacloud music ifs files - Works on eacloud music ifs files
- Correctly names AFP files - Correctly names AFP files
- Converts version.xml, afplist.xml, texturelist.xml to plaintext, to facilitate further experimentation. - Converts version.xml, afplist.xml, texturelist.xml to plaintext, to facilitate further experimentation.
- Dumps the ifs manifest so you can explore the format
Todo: Todo:
- Repacking - DXT5 repacking support (current workaround: edit texturelist to use argb8888rev)
- Cache compressed textures (compression is very slow)
Requires [kbinxml](https://github.com/mon/kbinxml/). - Recursive repacking for ifs inside ifs
I hope the rest is self explanatory. Confused? Create a new issue and tell me what docs to add. I hope the rest is self explanatory. Confused? Create a new issue and tell me what docs to add.

View File

@ -1,21 +1,77 @@
from os.path import getmtime
from kbinxml import KBinXML from kbinxml import KBinXML
import lxml.etree as etree
from . import escapes
class GenericFile(object): class GenericFile(object):
def __init__(self, ifs, elem, name): def __init__(self, ifs, path, name, time, start = -1, size = -1):
self.ifs = ifs self.ifs = ifs
self.path = path
self.name = name self.name = name
self.elem = elem self._packed_name = name
self.start, self.size, self.time = self._split_ints(elem.text) self.time = time
self.start = start
self.size = size
def _split_ints(self, text, delim = ' '): @classmethod
def from_xml(cls, ifs, elem, name):
start, size, time = cls._split_ints(elem.text)
self = cls(ifs, None, name, time, start, size)
return self
@classmethod
def from_filesystem(cls, ifs, path, name):
time = int(getmtime(path))
start = size = -1
self = cls(ifs, path, name, time, start, size)
return self
@staticmethod
def _split_ints(text, delim = ' '):
return list(map(int, text.split(delim))) return list(map(int, text.split(delim)))
def tostring(self, indent = 0): def tostring(self, indent = 0):
return '{}{}\n'.format(' ' * indent, self.name) return '{}{}\n'.format(' ' * indent, self.name)
def load(self, raw = False): def load(self, convert_kbin = True):
if self.path:
return self._load_from_filesystem(convert_kbin)
else:
return self._load_from_ifs(convert_kbin)
def _load_from_ifs(self, convert_kbin = True):
data = self.ifs.load_file(self.start, self.size) data = self.ifs.load_file(self.start, self.size)
if not raw: if convert_kbin and self.name.endswith('.xml') and KBinXML.is_binary_xml(data):
if self.name.endswith('.xml') and KBinXML.is_binary_xml(data): data = KBinXML(data).to_text().encode('utf8')
data = KBinXML(data).to_text().encode('utf8')
return data return data
def _load_from_filesystem(self, convert_kbin = True):
with open(self.path, 'rb') as f:
ret = f.read()
self.size = len(ret)
return ret
def repack(self, manifest, data_blob, progress):
if progress:
print(self.name)
elem = etree.SubElement(manifest, self.packed_name)
elem.attrib['__type'] = '3s32'
data = self.load(convert_kbin = False)
if self.name.endswith('.xml') and not KBinXML.is_binary_xml(data):
data = KBinXML(data).to_binary()
# offset, size, timestamp
elem.text = '{} {} {}'.format(len(data_blob.getvalue()), len(data), self.time)
data_blob.write(data)
@property
def packed_name(self):
return self.sanitize_name(self._packed_name)
def sanitize_name(self, n):
for e in escapes[::-1]:
n = n.replace(e[1], e[0])
if n[0].isdigit():
n = '_' + n
return n

View File

@ -1,34 +1,68 @@
from itertools import chain from itertools import chain
from os.path import getmtime, basename, join
import lxml.etree as etree import lxml.etree as etree
from . import get_folder_handlers from . import get_folder_handlers, escapes
from .GenericFile import GenericFile from .GenericFile import GenericFile
escapes = [
('_E', '.'),
('__', '_'),
]
class GenericFolder(): class GenericFolder():
def __init__(self, ifs, element, name = ''):
self.ifs = ifs
self.info_elem = None
self.name = name
self.elem = element
self.time = element.text
self.files = {} def __init__(self, ifs, name, time, files, folders):
self.folders = {} self.ifs = ifs
self.name = name
# xml sanitisation performed by the public property
self._packed_name = name
self.time = time
self.files = files
self.folders = folders
@classmethod
def from_xml(cls, ifs, element, name = ''):
time = int(element.text) if element.text else None
files = {}
folders = {}
for child in element.iterchildren(tag=etree.Element): for child in element.iterchildren(tag=etree.Element):
name = self.fix_name(child.tag) filename = cls.fix_name(child.tag)
if name == '_info_': # metadata if filename == '_info_': # metadata
self.info_elem = child info_elem = child
elif list(child): # folder elif list(child): # folder
handler = get_folder_handlers().get(name, GenericFolder) handler = get_folder_handlers().get(filename, GenericFolder)
self.folders[name] = handler(self.ifs, child, name) folders[filename] = handler.from_xml(ifs, child, filename)
else: # file else: # file
self.files[name] = GenericFile(self.ifs, child, name) files[filename] = GenericFile.from_xml(ifs, child, filename)
return cls(ifs, name, time, files, folders)
@classmethod
def from_filesystem(cls, ifs, tree, name = ''):
time = int(getmtime(tree['path']))
files = {}
folders = {}
for folder in tree['folders']:
base = basename(folder['path'])
handler = get_folder_handlers().get(base, GenericFolder)
folders[base] = handler.from_filesystem(ifs, folder, base)
for filename in tree['files']:
path = join(tree['path'], filename)
files[filename] = GenericFile.from_filesystem(ifs, path, filename)
return cls(ifs, name, time, files, folders)
def repack(self, manifest, data_blob, progress):
if self.name:
manifest = etree.SubElement(manifest, self.packed_name)
manifest.attrib['__type'] = 's32'
manifest.text = str(self.time)
if progress:
print(self.name)
for name, entry in chain(self.folders.items(), self.files.items()):
entry.repack(manifest, data_blob, progress)
def tostring(self, indent = 0): def tostring(self, indent = 0):
ret = '' ret = ''
@ -39,7 +73,19 @@ class GenericFolder():
ret += entry.tostring(indent) ret += entry.tostring(indent)
return ret return ret
def fix_name(self, n): @property
def packed_name(self):
return self.sanitize_name(self._packed_name)
def sanitize_name(self, n):
for e in escapes[::-1]:
n = n.replace(e[1], e[0])
if n[0].isdigit():
n = '_' + n
return n
@staticmethod
def fix_name(n):
for e in escapes: for e in escapes:
n = n.replace(*e) n = n.replace(*e)
if n[0] == '_' and n[1].isdigit(): if n[0] == '_' and n[1].isdigit():

View File

@ -2,6 +2,7 @@ from io import BytesIO
from struct import unpack, pack from struct import unpack, pack
from PIL import Image from PIL import Image
import lxml.etree as etree
from kbinxml import KBinXML from kbinxml import KBinXML
from . import GenericFile from . import GenericFile
@ -20,8 +21,10 @@ dxt5_end = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + \
class ImageFile(GenericFile): class ImageFile(GenericFile):
def __init__(self, gen_file, image_elem, fmt, compress): def __init__(self, gen_file, image_elem, fmt, compress):
super().__init__(gen_file.ifs, gen_file.elem, gen_file.name + '.png') super().__init__(gen_file.ifs, gen_file.path,
gen_file.name + '.png', gen_file.time,
gen_file.start, gen_file.size)
self._packed_name = gen_file._packed_name
self.image_elem = image_elem self.image_elem = image_elem
self.format = fmt self.format = fmt
self.compress = compress self.compress = compress
@ -33,8 +36,15 @@ class ImageFile(GenericFile):
(self.imgrect[3]-self.imgrect[2])//2 (self.imgrect[3]-self.imgrect[2])//2
) )
def load(self): @classmethod
data = super().load() def from_xml(cls, ifs, elem, name):
raise Exception('ImageFile must be instantiated from existing element')
@classmethod
def from_filesystem(cls, ifs, tree, name):
raise Exception('ImageFile must be instantiated from existing element')
def _load_from_ifs(self, convert_kbin = False):
data = super()._load_from_ifs()
if self.compress == 'avslz': if self.compress == 'avslz':
uncompressed_size = unpack('>I', data[:4])[0] uncompressed_size = unpack('>I', data[:4])[0]
@ -72,3 +82,29 @@ class ImageFile(GenericFile):
b = BytesIO() b = BytesIO()
im.save(b, format = 'PNG') im.save(b, format = 'PNG')
return b.getvalue() return b.getvalue()
def repack(self, manifest, data_blob, progress):
if progress:
print(self.name)
data = self.load()
im = Image.open(BytesIO(data))
if self.format == 'argb8888rev':
data = im.tobytes('raw', 'BGRA')
else:
raise NotImplementedError('Unknown format {}'.format(self.format))
if self.compress == 'avslz':
o = data
uncompressed_size = len(data)
data = lz77.compress(data)
compressed_size = len(data)
data = pack('>I', uncompressed_size) + pack('>I', compressed_size) + data
# offset, size, timestamp
elem = etree.SubElement(manifest, self.packed_name)
elem.attrib['__type'] = '3s32'
elem.text = '{} {} {}'.format(len(data_blob.getvalue()), len(data), self.time)
data_blob.write(data)

View File

@ -5,32 +5,53 @@ from kbinxml import KBinXML
from . import GenericFolder from . import GenericFolder
class MD5Folder(GenericFolder): class MD5Folder(GenericFolder):
def __init__(self, ifs, element, name, md5_tag = None):
super().__init__(ifs, element, name) def __init__(self, ifs, name, time, files, folders):
super().__init__(ifs, name, time, files, folders)
for filename, file in self.files.items(): for filename, file in self.files.items():
if filename.endswith('.xml'): if filename.endswith('.xml'):
self.info_kbin = file self.info_kbin = file
break break
if not self.info_kbin: if not self.info_kbin:
raise KeyError('MD5 folder expected but no mapping xml') raise KeyError('MD5 folder contents have no mapping xml')
self.info_kbin = KBinXML(self.info_kbin.load(True)) self.info_kbin = KBinXML(self.info_kbin.load(convert_kbin = False))
@classmethod
def from_xml(cls, ifs, element, name = '', md5_tag = None, extension = None):
self = super().from_xml(ifs, element, name)
self._apply_md5(md5_tag, extension)
return self
@classmethod
def from_filesystem(cls, ifs, tree, name = '', md5_tag = None, extension = None):
self = super().from_filesystem(ifs, tree, name)
self._apply_md5(md5_tag, extension)
return self
def _apply_md5(self, md5_tag, extension):
if not md5_tag: if not md5_tag:
md5_tag = name md5_tag = self.name
# findall needs xpath or it'll only search children # findall needs xpath or it'll only search direct children
for tag in self.info_kbin.xml_doc.findall('.//' + md5_tag): for tag in self.info_kbin.xml_doc.findall('.//' + md5_tag):
filename = tag.attrib['name'] filename = tag.attrib['name']
hash = md5(filename.encode(self.info_kbin.encoding)).hexdigest() hash = md5(filename.encode(self.info_kbin.encoding)).hexdigest()
# handles subfolders like afp/bsi/ # handles subfolders like afp/bsi/
self.rename_recurse(self, hash, filename) self.rename_recurse(self, hash, filename, extension)
def rename_recurse(self, entry, original, replacement): def rename_recurse(self, entry, original, replacement, extension):
# handles renamed files (eg tex->png)
if extension and (replacement + extension in entry.files):
entry.files[replacement] = entry.files.pop(replacement + extension)
entry.files[replacement].name = replacement
# handles deobfuscated filesystems
if replacement in entry.files:
entry.files[replacement]._packed_name = original
if original in entry.files: if original in entry.files:
orig = entry.files.pop(original) orig = entry.files.pop(original)
orig.name = replacement orig.name = replacement
entry.files[replacement] = orig entry.files[replacement] = orig
for name, folder in entry.folders.items(): for name, folder in entry.folders.items():
self.rename_recurse(folder, original, replacement) self.rename_recurse(folder, original, replacement, extension)

View File

@ -1,11 +1,23 @@
from . import MD5Folder, GenericFile, ImageFile from . import MD5Folder, GenericFile, ImageFile
class TexFolder(MD5Folder): class TexFolder(MD5Folder):
def __init__(self, ifs, element, name): def __init__(self, ifs, name, time, files, folders):
super().__init__(ifs, element, name, 'image') super().__init__(ifs, name, time, files, folders)
self.compress = self.info_kbin.xml_doc.attrib.get('compress') self.compress = self.info_kbin.xml_doc.attrib.get('compress')
@classmethod
def from_xml(cls, ifs, element, name = ''):
self = super().from_xml(ifs, element, name, 'image', '.png')
self._create_images()
return self
@classmethod
def from_filesystem(cls, ifs, tree, name = ''):
self = super().from_filesystem(ifs, tree, name, 'image', '.png')
self._create_images()
return self
def _create_images(self):
for tex in self.info_kbin.xml_doc.iterchildren(): for tex in self.info_kbin.xml_doc.iterchildren():
folder = tex.attrib['name'] folder = tex.attrib['name']
fmt = tex.attrib['format'] fmt = tex.attrib['format']

View File

@ -5,6 +5,11 @@ def get_folder_handlers():
'tex' : TexFolder 'tex' : TexFolder
} }
escapes = [
('_E', '.'),
('__', '_'),
]
from .GenericFile import GenericFile from .GenericFile import GenericFile
from .ImageFile import ImageFile from .ImageFile import ImageFile

View File

@ -1,5 +1,7 @@
# consistency with py 2/3 # consistency with py 2/3
from builtins import bytes from builtins import bytes
from struct import unpack, pack
from io import BytesIO
WINDOW_SIZE = 0x1000 WINDOW_SIZE = 0x1000
WINDOW_MASK = WINDOW_SIZE - 1 WINDOW_MASK = WINDOW_SIZE - 1
@ -7,113 +9,103 @@ THRESHOLD = 3
INPLACE_THRESHOLD = 0xA INPLACE_THRESHOLD = 0xA
LOOK_RANGE = 0x200 LOOK_RANGE = 0x200
MAX_LEN = 0xF + THRESHOLD MAX_LEN = 0xF + THRESHOLD
MAX_BUFFER = 0x10 + 1
def decompress(input): def decompress(input):
input = bytes(input) input = BytesIO(input)
decompressed = bytearray() decompressed = bytearray()
cur_byte = 0
input_length = len(input)
window = [0] * WINDOW_SIZE
window_cursor = 0
while cur_byte < input_length: while True:
flag = input[cur_byte] flag = input.read(1)[0]
cur_byte += 1
for i in range(8): for i in range(8):
if (flag >> i) & 1 == 1: if (flag >> i) & 1 == 1:
decompressed.append(input[cur_byte]) decompressed.append(input.read(1)[0])
window[window_cursor] = input[cur_byte]
window_cursor = (window_cursor + 1) & WINDOW_MASK
cur_byte += 1
else: else:
w = input[cur_byte] << 8 | input[cur_byte + 1] w = unpack('>H', input.read(2))[0]
cur_byte += 2 position = (w >> 4)
if (w >> 4) == 0: length = (w & 0x0F) + THRESHOLD
if position == 0:
return bytes(decompressed) return bytes(decompressed)
position = ((window_cursor - (w >> 4)) & WINDOW_MASK) if position > len(decompressed):
length = (w & 0x0F) + THRESHOLD diff = position - len(decompressed)
diff = min(diff, length)
decompressed.extend([0]*diff)
length -= diff
# optimise
if -position+length < 0:
decompressed.extend(decompressed[-position:-position+length])
else:
for loop in range(length):
decompressed.append(decompressed[-position])
for loop in range(length): def match_window(in_data, offset):
b = window[position & WINDOW_MASK] '''Find the longest match for the string starting at offset in the preceeding data
decompressed.append(b) '''
window[window_cursor] = b window_start = max(offset - WINDOW_MASK, 0)
window_cursor = (window_cursor + 1) & WINDOW_MASK
position = position + 1
return bytes(decompressed)
for n in range(MAX_LEN, THRESHOLD-1, -1):
window_end = min(offset + n, len(in_data))
# we've not got enough data left for a meaningful result
if window_end - offset < THRESHOLD:
return None
str_to_find = in_data[offset:window_end]
idx = in_data.rfind(str_to_find, window_start, window_end-n)
if idx != -1:
code_offset = offset - idx # - 1
code_len = len(str_to_find)
return (code_offset, code_len)
def match_current(window, pos, max_len, data, dpos): return None
length = 0
data_len = len(data)
while dpos + length < data_len and length < max_len and \
window[(pos + length) & WINDOW_MASK] == data[dpos + length] and length < MAX_LEN:
length += 1
return length
def match_window(window, pos, data, dpos):
max_pos = 0;
max_len = 0;
for i in range(THRESHOLD, LOOK_RANGE):
length = match_current(window, (pos - i) & WINDOW_MASK, i, data, dpos)
if length >= INPLACE_THRESHOLD:
return (i, length)
if length >= THRESHOLD:
max_pos = i
max_len = length
if max_len >= THRESHOLD:
return (max_pos, max_len)
else:
return None
def compress(input): def compress(input):
compressed = bytearray() compressed = bytearray()
input = bytes(input) input = bytes([0]*WINDOW_SIZE) + bytes(input)
input_size = len(input) input_size = len(input)
window = [0] * WINDOW_SIZE current_pos = WINDOW_SIZE
current_pos = 0
current_window = 0
bit = 0 bit = 0
buf = [0] * 0x11
while current_pos < input_size: while current_pos < input_size:
flag_byte = 0; flag_byte = 0;
current_buffer = 0; buf = bytearray()
for _ in range(8): for _ in range(8):
if current_pos >= input_size: if current_pos >= input_size:
buf[current_buffer] = 0;
window[current_window] = 0;
current_buffer += 1;
current_pos += 1;
current_window += 1;
bit = 0; bit = 0;
else: else:
match = match_window(window, current_window, input, current_pos) match = match_window(input, current_pos)
if match: if match:
pos, length = match pos, length = match
byte1 = (pos >> 4) info = (pos << 4) | ((length - THRESHOLD) & 0x0F)
byte2 = (((pos & 0x0F) << 4) | ((length - THRESHOLD) & 0x0F)) buf.extend(pack('>H', info))
buf[current_buffer] = byte1
buf[current_buffer + 1] = byte2
current_buffer += 2
bit = 0 bit = 0
for _ in range(length): current_pos += length
window[current_window & WINDOW_MASK] = input[current_pos]
current_pos += 1
current_window += 1
else: else:
buf[current_buffer] = input[current_pos] buf.append(input[current_pos])
window[current_window] = input[current_pos]
current_pos += 1 current_pos += 1
current_window += 1
current_buffer += 1
bit = 1 bit = 1
flag_byte = (flag_byte >> 1) | ((bit & 1) << 7) flag_byte = (flag_byte >> 1) | ((bit & 1) << 7)
current_window = current_window & WINDOW_MASK
compressed.append(flag_byte) compressed.append(flag_byte)
for i in range(current_buffer): compressed.extend(buf)
compressed.append(buf[i])
compressed.append(0) compressed.append(0)
compressed.append(0) compressed.append(0)
compressed.append(0) compressed.append(0)
return bytes(compressed)
def compress_dummy(input):
input_length = len(input)
compressed = bytearray()
extra_bytes = input_length % 8
for i in range(0, input_length-extra_bytes, 8):
compressed.append(0xFF)
compressed.extend(input[i:i+8])
if extra_bytes > 0:
compressed.append(0xFF >> (8 - extra_bytes))
compressed.extend(input[-extra_bytes:])
compressed.append(0)
compressed.append(0)
compressed.append(0)
return bytes(compressed) return bytes(compressed)

View File

@ -1,39 +1,90 @@
from os.path import basename, dirname, splitext, join from os.path import basename, dirname, splitext, join, isdir, isfile, getmtime
from os import mkdir, utime from os import mkdir, utime, walk
from io import BytesIO
import hashlib import hashlib
import lxml.etree as etree import lxml.etree as etree
from struct import unpack from time import time as unixtime
from kbinxml.kbinxml import KBinXML from kbinxml.kbinxml import KBinXML
from kbinxml.bytebuffer import ByteBuffer from kbinxml.bytebuffer import ByteBuffer
from handlers import GenericFolder from handlers import GenericFolder
SIGNATURE = 0x6CAD8F89
KBIN_OFFSET = 36 KBIN_OFFSET = 36
FILE_VERSION = 3
class IFS: class IFS:
def __init__(self, path): def __init__(self, path):
if isfile(path):
self._load_ifs(path)
self.is_file = True
elif isdir(path):
self._load_dir(path)
self.is_file = False
else:
raise IOError('Input path does not exist')
def _load_ifs(self, path):
out = splitext(basename(path))[0] + '_ifs' out = splitext(basename(path))[0] + '_ifs'
self.default_out = join(dirname(path), out) self.default_out = join(dirname(path), out)
self.ifs_out = path
with open(path, 'rb') as f: with open(path, 'rb') as f:
self.file = f.read() self.file = f.read()
b = ByteBuffer(self.file) b = ByteBuffer(self.file)
self.signature = b.get_u32() signature = b.get_u32()
self.ifs_size = b.get_u32() if signature != SIGNATURE:
self.unk1 = b.get_u32() raise IOError('Given file was not an IFS file!')
self.unk2 = b.get_u32() self.file_version = b.get_u16()
# next u16 is just NOT(version)
assert b.get_u16() ^ self.file_version == 0xFFFF
self.time = b.get_u32()
self.tree_size = b.get_u32()
self.header_end = b.get_u32() self.header_end = b.get_u32()
# 16 bytes more, unsure # 16 bytes for manifest md5, unchecked
self.manifest = KBinXML(self.file[KBIN_OFFSET:]) self.manifest = KBinXML(self.file[KBIN_OFFSET:])
#with open('debug_manifest.xml', 'wb') as f:
# f.write(self.manifest.to_text().encode('utf8'))
self._parse_manifest() self._parse_manifest()
assert self.tree_size == self._tree_size()
def _load_dir(self, path):
self.default_out = path
self.ifs_out = basename(path).replace('_ifs', '.ifs')
self.file_version = FILE_VERSION
self.time = int(getmtime(path))
self.tree_size = -1
self.header_end = -1
self.manifest = None
os_tree = self._create_dir_tree(path)
self.tree = GenericFolder.from_filesystem(self, os_tree)
def _create_dir_tree(self, path):
tree = self._create_dir_tree_recurse(walk(path))
if 'ifs_manifest.xml' in tree['files']:
tree['files'].remove('ifs_manifest.xml')
return tree
def _create_dir_tree_recurse(self, walker):
tree = {}
root, dirs, files = next(walker)
tree['path'] = root
tree['files'] = files
tree['folders'] = []
for dir in dirs:
tree['folders'].append(self._create_dir_tree_recurse(walker))
return tree
def _parse_manifest(self): def _parse_manifest(self):
self.tree = GenericFolder(self, self.manifest.xml_doc) self.tree = GenericFolder.from_xml(self, self.manifest.xml_doc)
def tostring(self): def tostring(self):
return self.tree.tostring() return self.tree.tostring()
@ -41,10 +92,67 @@ class IFS:
def extract_all(self, progress = True, recurse = True, path = None): def extract_all(self, progress = True, recurse = True, path = None):
self.out = path if path else self.default_out self.out = path if path else self.default_out
self._mkdir(self.out) self._mkdir(self.out)
with open(join(self.out, 'ifs_manifest.xml'), 'wb') as f: if self.manifest:
f.write(self.manifest.to_text().encode('utf8')) with open(join(self.out, 'ifs_manifest.xml'), 'wb') as f:
f.write(self.manifest.to_text().encode('utf8'))
self._extract_tree(self.tree, progress, recurse) self._extract_tree(self.tree, progress, recurse)
def repack(self, progress = True, path = None):
if path is None:
path = self.ifs_out
data_blob = BytesIO()
self.manifest = KBinXML(etree.Element('imgfs'))
manifest_info = etree.SubElement(self.manifest.xml_doc, '_info_')
# the important bit
self.tree.repack(self.manifest.xml_doc, data_blob, progress)
data = data_blob.getvalue()
data_md5 = etree.SubElement(manifest_info, 'md5')
data_md5.attrib['__type'] = 'bin'
data_md5.attrib['__size'] = '16'
data_md5.text = hashlib.md5(data).hexdigest()
data_size = etree.SubElement(manifest_info, 'size')
data_size.attrib['__type'] = 'u32'
data_size.text = str(len(data))
manifest_bin = self.manifest.to_binary()
self.header_end = 36 + len(manifest_bin)
self.ifs_size = self.header_end + len(data)
self.tree_size = self._tree_size()
manifest_hash = hashlib.md5(manifest_bin).digest()
head = ByteBuffer()
head.append_u32(SIGNATURE)
head.append_u16(self.file_version)
head.append_u16(self.file_version ^ 0xFFFF)
head.append_u32(int(unixtime()))
head.append_u32(self.tree_size)
head.append_u32(self.header_end)
with open(path, 'wb') as ifs_file:
ifs_file.write(head.data)
ifs_file.write(manifest_hash)
ifs_file.write(manifest_bin)
ifs_file.write(data)
# suspected to be the in-memory representation
def _tree_size(self):
BASE_SIZE = 856
return BASE_SIZE + self._tree_size_recurse(self.tree)
def _tree_size_recurse(self, tree, depth = 0):
FILE = 64
FOLDER = 56
DEPTH_MULTIPLIER = 16
size = len(tree.files) * FILE
size += len(tree.folders) * (FOLDER - depth*DEPTH_MULTIPLIER)
for name, folder in tree.folders.items():
size += self._tree_size_recurse(folder, depth+1)
return size
def _extract_tree(self, tree, progress = True, recurse = True, dir = ''): def _extract_tree(self, tree, progress = True, recurse = True, dir = ''):
outdir = join(self.out, dir) outdir = join(self.out, dir)
if progress: if progress:
@ -64,6 +172,10 @@ class IFS:
for name, f in tree.folders.items(): for name, f in tree.folders.items():
self._extract_tree(f, progress, recurse, join(dir, f.name)) self._extract_tree(f, progress, recurse, join(dir, f.name))
# fallback to file timestamp
timestamp = tree.time if tree.time else self.time
utime(outdir, (timestamp, timestamp))
def _mkdir(self, dir): def _mkdir(self, dir):
try: try:
@ -72,7 +184,11 @@ class IFS:
pass pass
def load_file(self, start, size): def load_file(self, start, size):
return self.file[self.header_end+start:self.header_end+start+size] start = self.header_end+start
end = start + size
assert start <= len(self.file) and end <= len(self.file)
return self.file[start:end]
def _save_with_time(self, filename, data, time): def _save_with_time(self, filename, data, time):
with open(filename, 'wb') as f: with open(filename, 'wb') as f:
@ -82,7 +198,10 @@ class IFS:
if __name__ == '__main__': if __name__ == '__main__':
import sys import sys
if len(sys.argv) < 2: if len(sys.argv) < 2:
print('ifstools filename.ifs') print('ifstools filename.ifs OR folder_ifs')
exit() exit()
i = IFS(sys.argv[1]) i = IFS(sys.argv[1])
i.extract_all() if i.is_file:
i.extract_all()
else:
i.repack()