Add .awc XMA [Red Dead Redemption (XMA)]

src/libvgmstream.vcproj

@@ -216,10 +216,18 @@
                     RelativePath=".\meta\aix_streamfile.h"
                     >
                 </File>
+                <File
+                    RelativePath=".\meta\awc_xma_streamfile.h"
+                    >
+                </File>
                 <File
                     RelativePath=".\meta\bar_streamfile.h"
                     >
                 </File>
+                <File
+                    RelativePath=".\meta\ea_eaac_eatrax_streamfile.h"
+                    >
+                </File>
                 <File
                     RelativePath=".\meta\sqex_scd_streamfile.h"
                     >

src/libvgmstream.vcxproj

@@ -96,7 +96,9 @@
     <ClInclude Include="meta\adx_keys.h" />
     <ClInclude Include="meta\aax_utf.h" />
     <ClInclude Include="meta\aix_streamfile.h" />
+    <ClInclude Include="meta\awc_xma_streamfile.h" />
     <ClInclude Include="meta\bar_streamfile.h" />
+    <ClInclude Include="meta\ea_eaac_eatrax_streamfile.h" />
     <ClInclude Include="meta\sqex_scd_streamfile.h" />
     <ClInclude Include="meta\meta.h" />
     <ClInclude Include="meta\hca_keys.h" />

src/libvgmstream.vcxproj.filters

@@ -68,9 +68,15 @@
     <ClInclude Include="meta\aix_streamfile.h">
       <Filter>meta\Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="meta\awc_xma_streamfile.h">
+      <Filter>meta\Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="meta\bar_streamfile.h">
       <Filter>meta\Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="meta\ea_eaac_eatrax_streamfile.h">
+      <Filter>meta\Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="meta\sqex_scd_streamfile.h">
       <Filter>meta\Header Files</Filter>
     </ClInclude>

src/meta/awc.c

@@ -1,6 +1,7 @@
 #include "meta.h"
 #include "../coding/coding.h"
 #include "../layout/layout.h"
+#include "awc_xma_streamfile.h"
 
 typedef struct {
     int big_endian;
@@ -64,6 +65,76 @@ VGMSTREAM * init_vgmstream_awc(STREAMFILE *streamFile) {
             vgmstream->codec_endian = awc.big_endian;
             break;
 
+#ifdef VGM_USE_FFMPEG
+        case 0x05: {    /* XMA2 (X360) */
+            uint8_t buf[0x100];
+            size_t bytes, block_size, block_count, substream_size;
+            off_t substream_offset;
+
+            if (awc.is_music) {
+                /* 1ch XMAs in blocks, we'll use layered layout + custom IO to get multi-FFmpegs working */
+                int i;
+                layered_layout_data * data = NULL;
+
+                /* init layout */
+                data = init_layout_layered(awc.channel_count);
+                if (!data) goto fail;
+                vgmstream->layout_data = data;
+                vgmstream->layout_type = layout_layered;
+                vgmstream->coding_type = coding_FFmpeg;
+
+                /* open each layer subfile */
+                for (i = 0; i < awc.channel_count; i++) {
+                    STREAMFILE* temp_streamFile;
+                    int layer_channels = 1;
+
+                    /* build the layer VGMSTREAM */
+                    data->layers[i] = allocate_vgmstream(layer_channels, 0);
+                    if (!data->layers[i]) goto fail;
+
+                    data->layers[i]->sample_rate = awc.sample_rate;
+                    data->layers[i]->meta_type = meta_AWC;
+                    data->layers[i]->coding_type = coding_FFmpeg;
+                    data->layers[i]->layout_type = layout_none;
+                    data->layers[i]->num_samples = awc.num_samples;
+
+                    /* setup custom IO streamfile, pass to FFmpeg and hope it's fooled */
+                    temp_streamFile = setup_awc_xma_streamfile(streamFile, awc.stream_offset, awc.stream_size, awc.block_chunk, awc.channel_count, i);
+                    if (!temp_streamFile) goto fail;
+
+                    substream_offset = 0; /* where FFmpeg thinks data starts, which our custom streamFile will clamp */
+                    substream_size = get_streamfile_size(temp_streamFile); /* data of one XMA substream without blocks */
+                    block_size = 0x8000; /* no idea */
+                    block_count = substream_size / block_size; /* not accurate but not needed */
+
+                    bytes = ffmpeg_make_riff_xma2(buf, 0x100, awc.num_samples, substream_size, layer_channels, awc.sample_rate, block_count, block_size);
+                    data->layers[i]->codec_data = init_ffmpeg_header_offset(temp_streamFile, buf,bytes, substream_offset,substream_size);
+
+                    close_streamfile(temp_streamFile);
+                    if (!data->layers[i]->codec_data) goto fail;
+                }
+
+                /* setup layered VGMSTREAMs */
+                if (!setup_layout_layered(data))
+                    goto fail;
+            }
+            else {
+                /* regular XMA for sfx */
+                block_size = 0x8000; /* no idea */
+                block_count = awc.stream_size / block_size; /* not accurate but not needed */
+
+                bytes = ffmpeg_make_riff_xma2(buf, 0x100, awc.num_samples, awc.stream_size, awc.channel_count, awc.sample_rate, block_count, block_size);
+                vgmstream->codec_data = init_ffmpeg_header_offset(streamFile, buf,bytes, awc.stream_offset,awc.stream_size);
+                if (!vgmstream->codec_data) goto fail;
+                vgmstream->coding_type = coding_FFmpeg;
+                vgmstream->layout_type = layout_none;
+            }
+
+            break;
+        }
+
+
+#endif
 #ifdef VGM_USE_MPEG
         case 0x07: {    /* MPEG (PS3) */
             mpeg_custom_config cfg;
@@ -80,7 +151,6 @@ VGMSTREAM * init_vgmstream_awc(STREAMFILE *streamFile) {
         }
 #endif
 
-        case 0x05:      /* XMA2 (X360) */
         default:
             VGM_LOG("AWC: unknown codec 0x%02x\n", awc.codec);
             goto fail;

src/meta/awc_xma_streamfile.h

@@ -0,0 +1,257 @@
+#ifndef _AWC_XMA_STREAMFILE_H_
+#define _AWC_XMA_STREAMFILE_H_
+#include "../streamfile.h"
+
+
+typedef struct {
+    /* config */
+    off_t stream_offset;
+    size_t stream_size;
+    int channel_count;
+    int channel;
+    size_t chunk_size;
+
+    /* state */
+    off_t logical_offset; /* offset that corresponds to physical_offset */
+    off_t physical_offset; /* actual file offset */
+    off_t next_block_offset; /* physical offset of the next block start */
+    off_t last_offset; /* physical offset of where the last block ended */
+    size_t current_data_size;
+    size_t current_consumed_size;
+
+    size_t total_size; /* size of the resulting XMA data */
+} awc_xma_io_data;
+
+
+static size_t get_block_header_size(STREAMFILE *streamFile, off_t offset, awc_xma_io_data *data);
+static size_t get_repeated_data_size(STREAMFILE *streamFile, off_t new_offset, off_t last_offset);
+static size_t get_block_skip_count(STREAMFILE *streamFile, off_t offset, int channel);
+
+/* Reads plain XMA data of a single stream. Each block has a header and channels have different num_samples/frames.
+ * Channel data is separate within the block (first all frames of ch0, then ch1, etc), padded, and sometimes
+ * the last few frames of a channel are repeated in the new block (marked with the "discard samples" field). */
+static size_t awc_xma_io_read(STREAMFILE *streamfile, uint8_t *dest, off_t offset, size_t length, awc_xma_io_data* data) {
+    size_t total_read = 0;
+    size_t frame_size = 0x800;
+
+    /* ignore bad reads */
+    if (offset < 0 || offset > data->total_size) {
+        return 0;
+    }
+
+    /* previous offset: re-start as we can't map logical<>physical offsets
+     * (kinda slow as it trashes buffers, but shouldn't happen often) */
+    if (offset < data->logical_offset) {
+        data->logical_offset = 0x00;
+        data->physical_offset = data->stream_offset;
+        data->next_block_offset = 0;
+        data->last_offset = 0;
+        data->current_data_size = 0;
+        data->current_consumed_size = 0;
+    }
+
+    /* read from block, moving to next when all data is consumed */
+    while (length > 0) {
+        size_t to_read, bytes_read;
+
+        /* new block */
+        if (data->current_data_size == 0) {
+            size_t header_size    = get_block_header_size(streamfile, data->physical_offset, data);
+            /* header table entries = frames... I hope */
+            size_t skip_size      = get_block_skip_count(streamfile, data->physical_offset, data->channel) * frame_size;
+          //size_t skip_size      = read_32bitBE(data->physical_offset + 0x10*data->channel + 0x00, streamfile) * frame_size;
+            size_t data_size      = read_32bitBE(data->physical_offset + 0x10*data->channel + 0x04, streamfile) * frame_size;
+            size_t repeat_samples = read_32bitBE(data->physical_offset + 0x10*data->channel + 0x08, streamfile);
+            size_t repeat_size    = 0;
+
+            /* if there are repeat samples current block repeats some frames from last block, find out size */
+            if (repeat_samples && data->last_offset) {
+                off_t data_offset = data->physical_offset + header_size + skip_size;
+                repeat_size = get_repeated_data_size(streamfile, data_offset, data->last_offset);
+            }
+
+            data->next_block_offset = data->physical_offset + data->chunk_size;
+            data->physical_offset += header_size + skip_size + repeat_size; /* data start */
+            data->current_data_size = data_size - repeat_size; /* readable max in this block */
+            data->current_consumed_size = 0;
+            continue;
+        }
+
+        /* block end, go next */
+        if (data->current_consumed_size == data->current_data_size) {
+            data->last_offset = data->physical_offset; /* where last block ended */
+            data->physical_offset = data->next_block_offset;
+            data->current_data_size = 0;
+            continue;
+        }
+
+        /* requested offset is further along, pretend we consumed data and try again */
+        if (offset > data->logical_offset) {
+            size_t to_consume = offset - data->logical_offset;
+            if (to_consume > data->current_data_size - data->current_consumed_size)
+                to_consume = data->current_data_size - data->current_consumed_size;
+
+            data->physical_offset += to_consume;
+            data->logical_offset += to_consume;
+            data->current_consumed_size += to_consume;
+            continue;
+        }
+
+        /* clamp reads up to this block's end */
+        to_read = (data->current_data_size - data->current_consumed_size);
+        if (to_read > length)
+            to_read = length;
+        if (to_read == 0)
+            return total_read; /* should never happen... */
+
+        /* finally read and move buffer/offsets */
+        bytes_read = read_streamfile(dest, data->physical_offset, to_read, streamfile);
+        total_read += bytes_read;
+        if (bytes_read != to_read)
+            return total_read; /* couldn't read fully */
+
+        dest += bytes_read;
+        offset += bytes_read;
+        length -= bytes_read;
+
+        data->physical_offset += bytes_read;
+        data->logical_offset += bytes_read;
+        data->current_consumed_size += bytes_read;
+    }
+
+    return total_read;
+}
+
+static size_t awc_xma_io_size(STREAMFILE *streamfile, awc_xma_io_data* data) {
+    off_t physical_offset, max_physical_offset, last_offset;
+    size_t frame_size = 0x800;
+    size_t total_size = 0;
+
+    if (data->total_size)
+        return data->total_size;
+
+    physical_offset = data->stream_offset;
+    max_physical_offset = data->stream_offset + data->stream_size;
+    last_offset = 0;
+
+    /* read blocks and sum final size */
+    while (physical_offset < max_physical_offset) {
+        size_t header_size    = get_block_header_size(streamfile, physical_offset, data);
+        /* header table entries = frames... I hope */
+        size_t skip_size      = get_block_skip_count(streamfile, physical_offset, data->channel) * frame_size;
+      //size_t skip_size      = read_32bitBE(physical_offset + 0x10*data->channel + 0x00, streamfile) * frame_size;
+        size_t data_size      = read_32bitBE(physical_offset + 0x10*data->channel + 0x04, streamfile) * frame_size;
+        size_t repeat_samples = read_32bitBE(physical_offset + 0x10*data->channel + 0x08, streamfile);
+        size_t repeat_size    = 0;
+
+        /* if there are repeat samples current block repeats some frames from last block, find out size */
+        if (repeat_samples && last_offset) {
+            off_t data_offset = physical_offset + header_size + skip_size;
+            repeat_size = get_repeated_data_size(streamfile, data_offset, last_offset);
+        }
+
+        last_offset = physical_offset + header_size + skip_size + data_size;
+        total_size += data_size - repeat_size;
+        physical_offset += data->chunk_size;
+    }
+
+    data->total_size = total_size;
+    return data->total_size;
+}
+
+
+/* Prepares custom IO for AWC XMA, which is interleaved XMA in AWC blocks */
+static STREAMFILE* setup_awc_xma_streamfile(STREAMFILE *streamFile, off_t stream_offset, size_t stream_size, size_t chunk_size, int channel_count, int channel) {
+    STREAMFILE *temp_streamFile = NULL, *new_streamFile = NULL;
+    awc_xma_io_data io_data = {0};
+    size_t io_data_size = sizeof(awc_xma_io_data);
+
+    io_data.stream_offset = stream_offset;
+    io_data.stream_size = stream_size;
+    io_data.chunk_size = chunk_size;
+    io_data.channel_count = channel_count;
+    io_data.channel = channel;
+    io_data.physical_offset = stream_offset;
+
+    /* setup subfile */
+    new_streamFile = open_wrap_streamfile(streamFile);
+    if (!new_streamFile) goto fail;
+    temp_streamFile = new_streamFile;
+
+    new_streamFile = open_io_streamfile(temp_streamFile, &io_data,io_data_size, awc_xma_io_read,awc_xma_io_size);
+    if (!new_streamFile) goto fail;
+    temp_streamFile = new_streamFile;
+
+    //todo maybe should force to read filesize once
+    return temp_streamFile;
+
+fail:
+    close_streamfile(temp_streamFile);
+    return NULL;
+}
+
+/* block header size, aligned/padded to 0x800 */
+static size_t get_block_header_size(STREAMFILE *streamFile, off_t offset, awc_xma_io_data *data) {
+    size_t header_size = 0;
+    int i;
+    int entries = data->channel_count;
+
+    for (i = 0; i < entries; i++) {
+        header_size += 0x10;
+        header_size += read_32bitBE(offset + 0x10*i + 0x04, streamFile) * 0x04; /* entries in the table */
+    }
+
+    if (header_size % 0x800) /* padded */
+        header_size +=  0x800 - (header_size % 0x800);
+
+    return header_size;
+}
+
+
+/* find data that repeats in the beginning of a new block at the end of last block */
+static size_t get_repeated_data_size(STREAMFILE *streamFile, off_t new_offset, off_t last_offset) {
+    uint8_t new_frame[0x800];/* buffer to avoid fseek back and forth */
+    size_t frame_size = 0x800;
+    off_t offset;
+    int i;
+
+    /* read block first frame */
+    if (read_streamfile(new_frame,new_offset, frame_size,streamFile) != frame_size)
+        goto fail;
+
+    /* find the frame in last bytes of prev block */
+    offset = last_offset - 0x4000; /* typical max is 1 frame of ~0x800, no way to know exact size */
+    while (offset < last_offset) {
+        /* compare frame vs prev block data */
+        for (i = 0; i < frame_size; i++) {
+            if ((uint8_t)read_8bit(offset+i,streamFile) != new_frame[i])
+                break;
+        }
+
+        /* frame fully compared? */
+        if (i == frame_size)
+            return last_offset - offset;
+        else
+            offset += i+1;
+    }
+
+fail:
+    VGM_LOG("AWC: can't find repeat size, new=0x%08lx, last=0x%08lx\n", new_offset, last_offset);
+    return 0; /* keep on truckin' */
+}
+
+/* header has a skip value, but somehow it's sometimes bigger than expected (WHY!?!?) so just sum all */
+static size_t get_block_skip_count(STREAMFILE *streamFile, off_t offset, int channel) {
+    size_t skip_count = 0;
+    int i;
+
+    //skip_size = read_32bitBE(offset + 0x10*channel + 0x00, streamFile); /* wrong! */
+    for (i = 0; i < channel; i++) {
+        skip_count += read_32bitBE(offset + 0x10*i + 0x04, streamFile); /* number of frames of this channel */
+    }
+
+    return skip_count;
+}
+
+
+#endif /* _AWC_XMA_STREAMFILE_H_ */

src/vgmstream.c

@@ -466,7 +466,7 @@ static VGMSTREAM * init_vgmstream_internal(STREAMFILE *streamFile) {
         /* check FFmpeg streams here, for lack of a better place */
         if (vgmstream->coding_type == coding_FFmpeg) {
             ffmpeg_codec_data *data = (ffmpeg_codec_data *) vgmstream->codec_data;
-            if (data->streamCount && !vgmstream->num_streams) {
+            if (data && data->streamCount && !vgmstream->num_streams) {
                 vgmstream->num_streams = data->streamCount;
             }
         }