Add UTF-8 support for UTF tables

Source/CsbEditor/Program.cs

@@ -75,6 +75,8 @@ namespace CsbEditor
                                         {
                                             CriCpkEntry cpkEntry = cpkArchive.GetByPath(sdlName);
 
+                                            if (cpkEntry != null)
+                                            {
                                                 using (Stream cpkSource = File.OpenRead(cpkPath))
                                                 using (Stream aaxSource = cpkEntry.Open(cpkSource))
                                                 {
@@ -91,6 +93,7 @@ namespace CsbEditor
                                                     }
                                                 }
                                             }
+                                        }
 
                                         else
                                         {

Source/SonicAudioLib/Archive/CriCpkArchive.cs

@@ -564,7 +564,20 @@ namespace SonicAudioLib.Archive
 
         public CriCpkEntry GetByPath(string path)
         {
-            return entries.FirstOrDefault(entry => ((entry.DirectoryName + "/" + entry.Name) == path));
+            string correctedPath = path.Replace('\\', '/');
+
+            return entries.FirstOrDefault(entry =>
+            {
+                string search = string.Empty;
+                if (!string.IsNullOrEmpty(entry.DirectoryName))
+                {
+                    search += $"{entry.DirectoryName.Replace('\\', '/')}/";
+                }
+
+                search += entry.Name;
+
+                return search == correctedPath;
+            });
         }
 
         private DateTime DateTimeFromCpkDateTime(ulong dateTime)

Source/SonicAudioLib/CriMw/CriTable.Internal.cs

@@ -4,10 +4,13 @@ namespace SonicAudioLib.CriMw
 {
     struct CriTableHeader
     {
-        public static readonly string Signature = "@UTF";
+        public const string Signature = "@UTF";
+        public const byte EncodingTypeShiftJis = 0;
+        public const byte EncodingTypeUtf8 = 1;
+
         public uint Length { get; set; }
-        public bool FirstBoolean { get; set; }
+        public byte UnknownByte { get; set; }
-        public bool SecondBoolean { get; set; }
+        public byte EncodingType { get; set; }
         public ushort RowsPosition { get; set; }
         public uint StringPoolPosition { get; set; }
         public uint DataPoolPosition { get; set; }
@@ -18,7 +21,7 @@ namespace SonicAudioLib.CriMw
     }
 
     [Flags]
-    enum CriFieldFlag
+    enum CriFieldFlag : byte
     {
         Name = 16,
         DefaultValue = 32,

Source/SonicAudioLib/CriMw/CriTableReader.cs

@@ -12,7 +12,8 @@ namespace SonicAudioLib.CriMw
         private List<CriTableField> fields;
         private Stream source;
         private CriTableHeader header;
-        private int rowIndex = -1;
+        private Encoding encoding;
+        private long rowIndex = -1;
         private uint headerPosition;
         private bool leaveOpen;
 
@@ -56,7 +57,7 @@ namespace SonicAudioLib.CriMw
             }
         }
 
-        public int CurrentRow
+        public long CurrentRow
         {
             get
             {
@@ -72,6 +73,14 @@ namespace SonicAudioLib.CriMw
             }
         }
 
+        public Encoding EncodingType
+        {
+            get
+            {
+                return encoding;
+            }
+        }
+
         private void ReadTable()
         {
             headerPosition = (uint)source.Position;
@@ -102,8 +111,29 @@ namespace SonicAudioLib.CriMw
             }
 
             header.Length = ReadUInt32() + 0x8;
-            header.FirstBoolean = ReadBoolean();
+            header.UnknownByte = ReadByte();
-            header.SecondBoolean = ReadBoolean();
+            header.EncodingType = ReadByte();
+
+            if (header.UnknownByte != 0)
+            {
+                throw new Exception($"Invalid byte ({header.UnknownByte}. Please report the error with the file.");
+            }
+			
+			// This actually might be a boolean telling the reader that it's using UTF-8 encoding, but idk
+            switch (header.EncodingType)
+            {
+                case CriTableHeader.EncodingTypeShiftJis:
+                    encoding = Encoding.GetEncoding("shift-jis");
+                    break;
+
+                case CriTableHeader.EncodingTypeUtf8:
+                    encoding = Encoding.UTF8;
+                    break;
+
+                default:
+                    throw new Exception($"Unknown encoding type ({header.EncodingType}). Please report the error with the file.");
+            }
+
             header.RowsPosition = (ushort)(ReadUInt16() + 0x8);
             header.StringPoolPosition = ReadUInt32() + 0x8;
             header.DataPoolPosition = ReadUInt32() + 0x8;
@@ -112,11 +142,6 @@ namespace SonicAudioLib.CriMw
             header.RowLength = ReadUInt16();
             header.NumberOfRows = ReadUInt32();
 
-            if (header.FirstBoolean)
-            {
-                throw new Exception($"Invalid boolean ({header.FirstBoolean}. Please report the error with the file.");
-            }
-
             for (ushort i = 0; i < header.NumberOfFields; i++)
             {
                 CriTableField field = new CriTableField();
@@ -271,7 +296,7 @@ namespace SonicAudioLib.CriMw
             return true;
         }
 
-        public bool MoveToRow(int rowIndex)
+        public bool MoveToRow(long rowIndex)
         {
             if (rowIndex >= header.NumberOfRows)
             {
@@ -617,7 +642,7 @@ namespace SonicAudioLib.CriMw
             long previousPosition = source.Position;
 
             source.Position = headerPosition + header.StringPoolPosition + stringPosition;
-            string strResult = EndianStream.ReadCString(source, Encoding.GetEncoding("shift-jis"));
+            string strResult = EndianStream.ReadCString(source, encoding);
             source.Position = previousPosition;
 
             if (strResult == "<NULL>" ||
@@ -675,12 +700,12 @@ namespace SonicAudioLib.CriMw
 
                         ReadData(out vldPosition, out vldLength);
 
-                        // SecondBoolean being true, check if utf table
+                        // Some ACB files have the length info set to zero for UTF table fields, so find the correct length
                         if (vldPosition > 0 && vldLength == 0)
                         {
                             source.Position = headerPosition + header.DataPoolPosition + vldPosition;
 
-                            if (Encoding.ASCII.GetString(ReadBytes(4)) == "@UTF")
+                            if (EndianStream.ReadCString(source, 4) == "@UTF")
                             {
                                 vldLength = ReadUInt32() + 8;
                             }

Source/SonicAudioLib/CriMw/CriTableWriter.cs

@@ -71,8 +71,8 @@ namespace SonicAudioLib.CriMw
 
             EndianStream.WriteCString(destination, CriTableHeader.Signature, 4);
             WriteUInt32(uint.MinValue);
-            WriteBoolean(false);
+            WriteByte(byte.MinValue);
-            WriteBoolean(false);
+            WriteByte(byte.MinValue);
             WriteUInt16(ushort.MinValue);
             WriteUInt32(uint.MinValue);
             WriteUInt32(uint.MinValue);
@@ -112,13 +112,20 @@ namespace SonicAudioLib.CriMw
 
             header.Length = (uint)destination.Position - headerPosition;
 
-            header.FirstBoolean = false;
+            if (settings.EncodingType == Encoding.GetEncoding("shift-jis"))
-            header.SecondBoolean = false;
+            {
+                header.EncodingType = CriTableHeader.EncodingTypeShiftJis;
+            }
+
+            else if (settings.EncodingType == Encoding.UTF8)
+            {
+                header.EncodingType = CriTableHeader.EncodingTypeUtf8;
+            }
 
             destination.Position = headerPosition + 4;
             WriteUInt32(header.Length - 8);
-            WriteBoolean(header.FirstBoolean);
+            WriteByte(header.UnknownByte);
-            WriteBoolean(header.SecondBoolean);
+            WriteByte(header.EncodingType);
             WriteUInt16((ushort)(header.RowsPosition - 8));
             WriteUInt32(header.StringPoolPosition - 8);
             WriteUInt32(header.DataPoolPosition - 8);
@@ -670,6 +677,11 @@ namespace SonicAudioLib.CriMw
 
             set
             {
+                if (value != Encoding.UTF8 || value != Encoding.GetEncoding("shift-jis"))
+                {
+                    return;
+                }
+
                 encodingType = value;
             }
         }

Source/SonicAudioLib/IO/EndianStream.cs

@@ -43,6 +43,16 @@ namespace SonicAudioLib.IO
             return buffer;
         }
 
+        public static byte[] ReadBytesAt(Stream source, int length, long position)
+        {
+            long oldPosition = source.Position;
+            source.Position = position;
+            FillBuffer(source, length);
+            source.Position = oldPosition;
+
+            return buffer;
+        }
+
         public static void WriteBytes(Stream destination, byte[] value)
         {
             destination.Write(value, 0, value.Length);