1
0
mirror of synced 2025-01-31 04:13:51 +01:00

Some fixes.

Fix zlib compression corrupting the file.
Fix sarc and msbt format descriptions.
Adjust when a file  in an IArchiveFile gets saved.
Porgress on MTA 0X50 compression type. not finished atm.
Start on ICompressionFormat interface. Will soon be the way compression formats are all handled.
This commit is contained in:
KillzXGaming 2019-07-01 15:44:19 -04:00
parent c5e6c4ed04
commit 42f6b670e0
25 changed files with 464 additions and 71 deletions

Binary file not shown.

View File

@ -315,7 +315,6 @@ namespace FirstPlugin
public uint unkown;
public uint CompressionType;
private long DataOffset;
public IFileFormat FileHandler;
public uint CompressedFileSize;
public uint padding;
@ -367,10 +366,7 @@ namespace FirstPlugin
byte[] CompressedData;
public void Write(FileWriter writer)
{
if (FileHandler != null && FileHandler.CanSave)
{
FileData = FileHandler.Save();
}
this.SaveFileFormat();
CompressedData = Compress(FileData, CompressionType);

View File

@ -142,19 +142,15 @@ namespace FirstPlugin
{
Offsets[i] = (uint)(writer.Position - DataStart);
files[i].SaveFileFormat();
writer.Write(files[i].FileData);
Align(writer, (int)Alignment);
}
writer.Seek(12, System.IO.SeekOrigin.Begin);
writer.Write(Offsets);
}
return mem.ToArray();
}

View File

@ -119,14 +119,9 @@ namespace FirstPlugin
{
reader.Seek(4, System.IO.SeekOrigin.Begin);
uint decompSize = reader.ReadUInt32();
uint compSize = (uint)reader.BaseStream.Length - 8;
byte[] filedata = reader.getSection(8, (int)compSize);
// data = STLibraryCompression.Type_LZ4.Decompress(filedata, 0, (int)compSize, (int)decompSize);
// data = STLibraryCompression.GZIP.Decompress(filedata);
return data;
return STLibraryCompression.MTA_CUSTOM.Decompress(data, decompSize);
}
else if (compType == 0x30)
{

View File

@ -16,7 +16,7 @@ namespace FirstPlugin.New
public FileType FileType { get; set; } = FileType.Archive;
public bool CanSave { get; set; }
public string[] Description { get; set; } = new string[] { "SARC", "SARC", "SARC", "SARC", "SARC", "SARC" };
public string[] Description { get; set; } = new string[] { "Sorted ARChive" };
public string[] Extension { get; set; } = new string[] { "*.pack", "*.sarc", "*.bgenv", "*.sblarc", "*.sbactorpack", ".arc" };
public string FileName { get; set; }
public string FilePath { get; set; }
@ -164,6 +164,7 @@ namespace FirstPlugin.New
sarcData.Files.Clear();
foreach (var file in files)
{
file.SaveFileFormat();
sarcData.Files.Add(file.FileName, file.FileData);
}
@ -173,9 +174,6 @@ namespace FirstPlugin.New
return sarc.Item2;
}
public class SarcEntry : ArchiveFileInfo
{
public SARC sarc; //Sarc file the entry is located in

View File

@ -16,7 +16,7 @@ namespace FirstPlugin
public FileType FileType { get; set; } = FileType.Archive;
public bool CanSave { get; set; }
public string[] Description { get; set; } = new string[] { "SARC", "SARC", "SARC", "SARC", "SARC", "SARC" };
public string[] Description { get; set; } = new string[] { "Sorted ARChive" };
public string[] Extension { get; set; } = new string[] { "*.pack", "*.sarc", "*.bgenv", "*.sblarc", "*.sbactorpack", ".arc" };
public string FileName { get; set; }
public string FilePath { get; set; }

View File

@ -118,6 +118,8 @@ namespace FirstPlugin
{
SetAlignment(writer, files[i].FileName);
writer.WriteUint32Offset(files[i]._posHeader + 4);
files[i].SaveFileFormat();
writer.Write(files[i].FileData);
}
writer.Close();

View File

@ -16,7 +16,7 @@ namespace FirstPlugin
public FileType FileType { get; set; } = FileType.Message;
public bool CanSave { get; set; }
public string[] Description { get; set; } = new string[] { "Message Binary Text" };
public string[] Description { get; set; } = new string[] { "Message Studio Binary Table" };
public string[] Extension { get; set; } = new string[] { "*.msbt" };
public string FileName { get; set; }
public string FilePath { get; set; }

View File

@ -68,6 +68,8 @@ namespace FirstPlugin
List<Type> types = new List<Type>();
foreach (Type T in LoadFileFormats())
types.Add(T);
foreach (Type T in LoadCompressionFormats())
types.Add(T);
foreach (Type T in LoadMenus())
types.Add(T);
@ -262,6 +264,13 @@ namespace FirstPlugin
MenuItems.Add(typeof(MenuExt));
return MenuItems.ToArray();
}
private Type[] LoadCompressionFormats()
{
List<Type> Formats = new List<Type>();
return Formats.ToArray();
}
private Type[] LoadFileFormats()
{
List<Type> Formats = new List<Type>();

View File

@ -0,0 +1,59 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.IO;
using System.IO.Compression;
using System.Threading.Tasks;
using Switch_Toolbox.Library.IO;
using K4os.Compression.LZ4.Streams;
namespace Switch_Toolbox.Library
{
public class LZ4F : ICompressionFormat
{
public string[] Description { get; set; } = new string[] { "LZ4F Compression" };
public string[] Extension { get; set; } = new string[] { "*.cmp", "*.lz4f" };
public bool Identify(Stream stream)
{
using (var reader = new FileReader(stream, true))
{
uint DecompressedSize = reader.ReadUInt32();
uint magicCheck = reader.ReadUInt32();
bool LZ4FDefault = magicCheck == 0x184D2204;
return LZ4FDefault;
}
}
public bool CanCompress { get; } = true;
public Stream Decompress(Stream stream)
{
using (MemoryStream mem = new MemoryStream())
{
using (var source = LZ4Stream.Decode(stream))
{
source.CopyTo(mem);
}
return mem;
}
}
public Stream Compress(Stream stream)
{
var mem = new MemoryStream();
using (var writer = new FileWriter(mem))
{
var data = stream.ToArray();
writer.Write(data.Length);
byte[] buffer = LZ4.Frame.LZ4Frame.Compress(new MemoryStream(data),
LZ4.Frame.LZ4MaxBlockSize.MB1, true, true, false, true, false);
writer.Write(buffer, 0, buffer.Length);
}
return stream;
}
}
}

View File

@ -7,6 +7,7 @@ using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using System.Collections.Generic;
using System.Linq;
using System;
using System.Runtime.InteropServices;
namespace Switch_Toolbox.Library.IO
{
@ -96,15 +97,14 @@ namespace Switch_Toolbox.Library.IO
var output = new MemoryStream();
output.Write(new byte[] { 0x78, 0xDA }, 0, 2);
using (var decompressedStream = new MemoryStream(output.ToArray()))
{
using (var zipStream = new DeflateStream(output, CompressionMode.Compress))
{
zipStream.Write(b, 0, b.Length);
using (var zipStream = new DeflateStream(output, CompressionMode.Compress, true))
zipStream.Write(b, 0, b.Length);
return output.ToArray();
}
}
//Add this as it weirdly prevents the data getting corrupted
//From https://github.com/IcySon55/Kuriimu/blob/f670c2719affc1eaef8b4c40e40985881247acc7/src/Kontract/Compression/ZLib.cs
var adler = b.Aggregate(Tuple.Create(1, 0), (x, n) => Tuple.Create((x.Item1 + n) % 65521, (x.Item1 + x.Item2 + n) % 65521));
output.Write(new[] { (byte)(adler.Item2 >> 8), (byte)adler.Item2, (byte)(adler.Item1 >> 8), (byte)adler.Item1 }, 0, 4);
return output.ToArray();
}
public static void CopyStream(System.IO.Stream input, System.IO.Stream output)
@ -128,23 +128,219 @@ namespace Switch_Toolbox.Library.IO
(X << 8) & 0xff0000 | (X << 24) & 0xff000000);
}
public byte[] Decompress(byte[] data, uint decompressedLength)
public static unsafe byte[] Decompress(byte[] input, uint decompressedLength)
{
uint dest = 0;
uint source = 0;
fixed (byte* outputPtr = new byte[decompressedLength])
{
fixed (byte* inputPtr = input)
{
Decompress(outputPtr, inputPtr, decompressedLength);
}
byte[] decomp = new byte[decompressedLength];
Marshal.Copy((IntPtr)outputPtr, decomp, 0, decomp.Length);
return decomp;
}
}
//Thanks Simon. Code ported from
//https://github.com/simontime/MarioTennisAces0x50Decompressor/blob/master/decompress.c
public static unsafe void Decompress(byte* output, byte* input, uint decompressedLength)
{
uint pos = 8;
byte* end = input + decompressedLength;
byte* data = input + pos;
byte[] Output = new byte[decompressedLength];
if (pos > decompressedLength)
{
uint flag;
while (true)
{
flag = 0xFF000000 * data[0];
if (flag != 0)
break;
data++;
for (int i = 0; i < 8; i++)
*output++ = *data++;
//EndOperation
CheckFinished(data, end);
}
flag |= 0x800000;
data++;
//IterateFlag
while ((flag & 0x80000000) == 0)
{
IterateFlag(flag, data, output);
}
while (true)
{
flag <<= 1;
if (flag == 0)
CheckFinished(data, end);
int op_ofs = (data[0] >> 4) | (data[1] << 4);
int op_len = data[0] & 0xF;
if (op_ofs == 0)
return;
byte* chunk = output -op_ofs;
if (op_len > 1)
data += 2;
else
{
int op_len_ext = data[2] + (op_len | 0x10);
if (op_len == 1)
{
int add_len = (data[3] << 8) | 0xFF;
data += 4;
op_len = op_len_ext + add_len;
if (op_ofs >= 2)
Loop1(flag, op_len, chunk, data, output);
}
else
{
data += 3;
op_len = op_len_ext;
if (op_ofs >= 2)
{
Loop1(flag, op_len, chunk, data, output);
}
}
}
Loop2(flag, op_len, data, output, chunk);
}
}
return Output;
EndOperation(data, end);
}
}
private static unsafe void Loop1(uint flag, int op_len, byte* chunk, byte* data, byte* output)
{
if (((*chunk ^ *output) & 1) == 0)
{
if ((*chunk & 1) != 0)
{
*output++ = *chunk++;
op_len--;
}
int op_len_sub = op_len - 2;
if (op_len >= 2)
{
int masked_len = ((op_len_sub >> 1) + 1) & 7;
byte* out_ptr = output;
byte* chunk_ptr = chunk;
if (masked_len != 0)
{
while (masked_len-- != 0)
{
*out_ptr++ = *chunk_ptr++;
*out_ptr++ = *chunk_ptr++;
op_len -= 2;
}
}
uint masked_ext_len = (uint)op_len_sub & 0xFFFFFFFE;
if (op_len_sub >= 0xE)
{
do
{
for (int i = 0; i < 0x10; i++)
*out_ptr++ = *chunk_ptr++;
op_len -= 0x10;
}
while (op_len > 1);
}
output += masked_ext_len + 2;
op_len = op_len_sub - (int)masked_ext_len;
chunk += masked_ext_len + 2;
}
if (op_len == 0)
CheckFlag(flag, data, output);
}
Loop2(flag, op_len, data, output, chunk);
}
private static unsafe void Loop2(uint flag, int op_len, byte* data, byte* output, byte* chunk)
{
int masked_len = op_len & 7;
byte* out_ptr = output;
byte* chunk_ptr = chunk;
if (masked_len != 0)
{
while (masked_len-- != 0)
*out_ptr++ = *chunk_ptr++;
}
if (op_len - 1 >= 7)
{
do
{
for (int i = 0; i < 8; i++)
*out_ptr++ = *chunk_ptr++;
}
while (chunk_ptr != chunk + op_len);
}
output += op_len;
CheckFlag(flag, data, output);
}
private static unsafe void CheckFlag(uint flag, byte* data, byte* output)
{
if ((flag & 0x80000000) == 0)
IterateFlag(flag, data, output);
}
private static unsafe void IterateFlag(uint flag, byte* data, byte* output)
{
flag <<= 1;
*output++ = *data++;
}
private static unsafe void CheckFinished(byte* data, byte* end)
{
if (data >= end)
EndOperation(data, end);
}
private static unsafe void EndOperation(byte* data, byte* end)
{
byte* ext = end + 0x20;
if (data < ext)
*end-- = *--ext;
while (data < ext) ;
}
public class LZSS
{
//From https://github.com/IcySon55/Kuriimu/blob/f670c2719affc1eaef8b4c40e40985881247acc7/src/Kontract/Compression/LZSS.cs

View File

@ -0,0 +1,49 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.IO;
using System.IO.Compression;
using System.Threading.Tasks;
using Switch_Toolbox.Library.IO;
namespace Switch_Toolbox.Library
{
public class ZCMP : ICompressionFormat
{
public string[] Description { get; set; } = new string[] { "ZLIB Compression (ZCMP)" };
public string[] Extension { get; set; } = new string[] { "*.cmp" };
public bool Identify(Stream stream)
{
using (var reader = new FileReader(stream, true))
{
return reader.CheckSignature(4, "ZCMP");
}
}
public bool CanCompress { get; } = true;
public Stream Decompress(Stream stream)
{
using (var br = new FileReader(stream, true))
{
var ms = new System.IO.MemoryStream();
br.BaseStream.Position = 130;
using (var ds = new DeflateStream(new MemoryStream(br.ReadBytes((int)br.BaseStream.Length - 80)), CompressionMode.Decompress))
ds.CopyTo(ms);
return ms;
}
}
public Stream Compress(Stream stream)
{
var mem = new MemoryStream();
mem.Write(new byte[] { 0x78, 0xDA }, 0, 2);
using (var zipStream = new DeflateStream(mem, CompressionMode.Compress))
{
zipStream.CopyTo(stream);
return mem;
}
}
}
}

View File

@ -12,6 +12,7 @@ namespace Switch_Toolbox.Library
{
}
public static IFileMenuExtension[] GetMenuExtensions()
{
//Add plugin and main application menu extensions
@ -83,33 +84,68 @@ namespace Switch_Toolbox.Library
return types.ToArray();
}
public static IFileFormat[] GetFileFormats()
public static ICompressionFormat[] GetCompressionFormats()
{
//Add plugin and main application file formats
List<IFileFormat> types = new List<IFileFormat>();
types.Add((IFileFormat)Activator.CreateInstance(typeof(DDS)));
types.Add((IFileFormat)Activator.CreateInstance(typeof(ASTC)));
types.Add((IFileFormat)Activator.CreateInstance(typeof(TGA)));
List<ICompressionFormat> types = new List<ICompressionFormat>();
LoadCompressionFormats(FormatList.GetCompressionFormats(), types);
if (GenericPluginLoader._Plugins == null)
GenericPluginLoader.LoadPlugin();
foreach (var plugin in GenericPluginLoader._Plugins)
{
foreach (Type type in plugin.Value.Types)
{
Type[] interfaces_array = type.GetInterfaces();
for (int i = 0; i < interfaces_array.Length; i++)
{
if (interfaces_array[i] == typeof(IFileFormat))
{
types.Add((IFileFormat)Activator.CreateInstance(type));
}
}
}
LoadCompressionFormats(plugin.Value.Types, types);
}
return types.ToArray();
}
private static void LoadCompressionFormats(Type[] Types, List<ICompressionFormat> Formats)
{
foreach (Type type in Types)
{
Type[] interfaces_array = type.GetInterfaces();
for (int i = 0; i < interfaces_array.Length; i++)
{
if (interfaces_array[i] == typeof(ICompressionFormat))
{
Formats.Add((ICompressionFormat)Activator.CreateInstance(type));
}
}
}
}
public static IFileFormat[] GetFileFormats()
{
//Add plugin and main application file formats
List<IFileFormat> types = new List<IFileFormat>();
LoadFileFormats(FormatList.GetFileFormats(), types);
if (GenericPluginLoader._Plugins == null)
GenericPluginLoader.LoadPlugin();
foreach (var plugin in GenericPluginLoader._Plugins)
{
LoadFileFormats(plugin.Value.Types, types);
}
return types.ToArray();
}
private static void LoadFileFormats(Type[] Types, List<IFileFormat> Formats)
{
foreach (Type type in Types)
{
Type[] interfaces_array = type.GetInterfaces();
for (int i = 0; i < interfaces_array.Length; i++)
{
if (interfaces_array[i] == typeof(IFileFormat))
{
Formats.Add((IFileFormat)Activator.CreateInstance(type));
}
}
}
}
}
}

View File

@ -0,0 +1,29 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Switch_Toolbox.Library
{
public class FormatList
{
//File formats accessable from this library
public static Type[] GetFileFormats()
{
List<Type> Formats = new List<Type>();
Formats.Add(typeof(DDS));
Formats.Add(typeof(ASTC));
Formats.Add(typeof(TGA));
return Formats.ToArray();
}
//Compression formats accessable from this library
public static Type[] GetCompressionFormats()
{
List<Type> Formats = new List<Type>();
return Formats.ToArray();
}
}
}

View File

@ -165,6 +165,20 @@ namespace Switch_Toolbox.Library.IO
fileReader.Position = 0;
ushort MagicHex2 = fileReader.ReadUInt16();
Stream stream;
if (data != null)
stream = new MemoryStream(data);
else
stream = File.OpenRead(FileName);
foreach (ICompressionFormat compressionFormat in FileManager.GetCompressionFormats())
{
if (compressionFormat.Identify(stream))
{
compressionFormat.Decompress(stream);
}
}
if (Magic == "Yaz0")
{
if (data != null)
@ -255,12 +269,6 @@ namespace Switch_Toolbox.Library.IO
fileReader.Close();
Stream stream;
if (data != null)
stream = new MemoryStream(data);
else
stream = File.OpenRead(FileName);
foreach (IFileFormat fileFormat in FileManager.GetFileFormats())
{
//Set the file name so we can check it's extension in the identifier.

View File

@ -206,11 +206,6 @@ namespace Switch_Toolbox.Library.IO
FileIsCompressed = true;
CompressionType = CompressionType.Yaz0;
}
if (extension == ".cmp")
{
FileIsCompressed = true;
CompressionType = CompressionType.Lz4f;
}
if (EnableDialog && FileIsCompressed)
{

View File

@ -141,19 +141,19 @@ namespace Switch_Toolbox.Library
}
}
public void SaveFileFormat()
{
if (FileFormat != null && FileFormat.CanSave)
_fileData = FileFormat.Save();
}
[Browsable(false)]
public string Name { get; set; } = string.Empty; //File Name (No Path)
[Browsable(false)]
public virtual byte[] FileData
{
get
{
if (FileFormat != null && FileFormat.CanSave)
return FileFormat.Save();
else
return _fileData;
}
get { return _fileData; }
set { _fileData = value; }
}
@ -681,7 +681,7 @@ namespace Switch_Toolbox.Library
{
if (fileFormat.CanSave)
{
ArchiveFileInfo.FileData = fileFormat.Save();
ArchiveFileInfo.SaveFileFormat();
UpdateEditor();
}
}

View File

@ -0,0 +1,21 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.IO;
namespace Switch_Toolbox.Library
{
public interface ICompressionFormat
{
string[] Description { get; set; }
string[] Extension { get; set; }
bool Identify(Stream stream);
bool CanCompress { get; }
Stream Decompress(Stream stream);
Stream Compress(Stream stream);
}
}

View File

@ -209,9 +209,11 @@
<Compile Include="Audio\AudioFileRipper.cs" />
<Compile Include="Audio\VGAudioFile.cs" />
<Compile Include="Collections\SortExtensions.cs" />
<Compile Include="Compression\LZ4F.cs" />
<Compile Include="Compression\STLibraryCompression.cs" />
<Compile Include="Compression\ZCMP.cs" />
<Compile Include="Config.cs" />
<Compile Include="Cryptography\crc32.cs" />
<Compile Include="Security\Cryptography\crc32.cs" />
<Compile Include="DrawableContainer.cs" />
<Compile Include="FileFormats\3DS\ETC1.cs" />
<Compile Include="FileFormats\Animation\SMD.cs" />
@ -231,6 +233,7 @@
<Compile Include="FileFormats\SizeTables\TPFileSizeTable.cs" />
<Compile Include="FileFormats\TGA.cs" />
<Compile Include="FileFormats\ZIP.cs" />
<Compile Include="Format Managers\FormatList.cs" />
<Compile Include="Forms\Archive\ArchiveFilePanel.cs">
<SubType>UserControl</SubType>
</Compile>
@ -566,6 +569,7 @@
<Compile Include="Generics\GenericPolygonGroup.cs" />
<Compile Include="Generics\GenericTexture.cs" />
<Compile Include="Generics\OpenGLTexture.cs" />
<Compile Include="Interfaces\ICompressionFormat.cs" />
<Compile Include="Interfaces\IDirectoryNode.cs" />
<Compile Include="Interfaces\INode.cs" />
<Compile Include="IO\Extensions\ByteArrayExtensions.cs" />
@ -685,7 +689,7 @@
<Compile Include="Interfaces\IFileFormat.cs" />
<Compile Include="IO\FileIO.cs" />
<Compile Include="Interfaces\IPlugin.cs" />
<Compile Include="Plugin\FileManager.cs" />
<Compile Include="Format Managers\FileManager.cs" />
<Compile Include="Plugin\PluginLoader.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Properties\Resources.Designer.cs">