Merge pull request #1050 from bnnm/stdio-mc

- Improve STDIO for TXTP that open many small files
- Add encrypted .mus from Minecraft v1.6.1<
- Add .wav with MP3 [Bear's Imagine That! (PC)]
- sgxd/rxws: improve companion file handling

cli/vgmstream_cli.c

@@ -694,6 +694,8 @@ static int convert_file(cli_config* cfg) {
         if (!valid) goto fail;
     }
 
+    vgmstream_set_log_stdout(VGM_LOG_LEVEL_ALL);
+
     /* open streamfile and pass subsong */
     {
         STREAMFILE* sf = open_stdio_streamfile(cfg->infilename);
@@ -702,8 +704,6 @@ static int convert_file(cli_config* cfg) {
             goto fail;
         }
 
-        vgmstream_set_log_stdout(VGM_LOG_LEVEL_ALL);
-
         sf->stream_index = cfg->subsong_index;
         vgmstream = init_vgmstream_from_STREAMFILE(sf);
         close_streamfile(sf);

fb2k/foo_streamfile.cpp

@@ -278,6 +278,9 @@ static STREAMFILE* open_foo_streamfile_buffer_by_file(service_ptr_t<file> m_file
     else
         this_sf->file_size = 0;
 
+    /* STDIO has an optimization to close unneeded FDs if file size is less than buffer,
+     * but seems foobar doesn't need this (reuses FDs?) */
+
     return &this_sf->vt;
 
 fail:

src/libvgmstream.vcxproj

@@ -116,6 +116,7 @@
     <ClInclude Include="meta\ea_eaac_opus_streamfile.h" />
     <ClInclude Include="meta\ea_schl_streamfile.h" />
     <ClInclude Include="meta\encrypted_bgm_streamfile.h" />
+    <ClInclude Include="meta\encrypted_mc161_streamfile.h" />
     <ClInclude Include="meta\fsb_encrypted_streamfile.h" />
     <ClInclude Include="meta\fsb_interleave_streamfile.h" />
     <ClInclude Include="meta\fsb5_streamfile.h" />

src/libvgmstream.vcxproj.filters

@@ -308,6 +308,9 @@
     <ClInclude Include="meta\encrypted_bgm_streamfile.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="meta\encrypted_mc161_streamfile.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="util\chunks.h">
       <Filter>Header Files</Filter>
     </ClInclude>

src/meta/encrypted.c

@@ -2,6 +2,7 @@
 #include "../coding/coding.h"
 #include "ogg_vorbis_streamfile.h"
 #include "encrypted_bgm_streamfile.h"
+#include "encrypted_mc161_streamfile.h"
 
 //todo fuse ogg encryptions and use generic names
 
@@ -158,7 +159,7 @@ static VGMSTREAM* init_vgmstream_encrypted_rpgmvo_riff(STREAMFILE* sf) {
 
     e.new_sf = setup_subfile_streamfile(e.temp_sf, 0x10, riff_size, "wav");
     if (!e.new_sf) goto fail;
-dump_streamfile(e.new_sf, 0);
+
     e.vgmstream = init_vgmstream_riff(e.new_sf);
     close_streamfile(e.temp_sf);
     close_streamfile(e.new_sf);
@@ -171,6 +172,26 @@ fail:
 }
 
 
+/* Minecraft (PC) before v1.6.1 (Java version) */
+static VGMSTREAM* init_vgmstream_encrypted_mc161(STREAMFILE* sf) {
+    encrypted_t e = {0};
+
+    if (!check_extensions(sf,"mus"))
+        goto fail;
+
+    /* all files use a different key so just fail on meta init */
+
+    e.temp_sf = setup_mc161_streamfile(sf);
+    if (!e.temp_sf) goto fail;
+
+    e.vgmstream = init_vgmstream_ogg_vorbis(e.temp_sf);
+    close_streamfile(e.temp_sf);
+    return e.vgmstream;
+fail:
+    return NULL;
+}
+
+
 /* parser for various encrypted games */
 VGMSTREAM* init_vgmstream_encrypted(STREAMFILE* sf) {
     VGMSTREAM* v = NULL;
@@ -187,5 +208,8 @@ VGMSTREAM* init_vgmstream_encrypted(STREAMFILE* sf) {
     v = init_vgmstream_encrypted_rpgmvo_riff(sf);
     if (v) return v;
 
+    v = init_vgmstream_encrypted_mc161(sf);
+    if (v) return v;
+
     return NULL;
 }

src/meta/encrypted_mc161_streamfile.h

@@ -0,0 +1,101 @@
+#ifndef _MC161_STREAMFILE_H_
+#define _MC161_STREAMFILE_H_
+#include "../streamfile.h"
+
+
+typedef struct {
+    int32_t base_key;
+    int32_t curr_key;
+    uint32_t curr_offset;
+} mc161_io_data;
+
+
+static void decrypt_chunk(uint8_t* buf, int buf_size, mc161_io_data* data) {
+    int i;
+    int32_t hash = data->curr_key;
+
+
+    for (i = 0; i < buf_size; i++) {
+    	buf[i] = (uint8_t)(buf[i] ^ ((hash >> 8) & 0xFF));
+	    hash = (int32_t)(hash * 498729871) + (85731 * (int8_t)buf[i]); /* signed */
+    }
+
+    data->curr_key = hash;
+    data->curr_offset += buf_size;
+}
+
+static void update_key(STREAMFILE* sf, off_t offset, mc161_io_data* data) {
+    uint8_t buf[0x800];
+    size_t bytes;
+    size_t to_skip;
+
+    if (offset < data->curr_offset || offset == 0x00) {
+        data->curr_key = data->base_key;
+        data->curr_offset = 0x00;
+        to_skip = offset;
+    }
+    else {
+        to_skip = offset - data->curr_offset;
+    }
+
+    /* update key by reading and decrypt all data between current offset + last known key to requested offset */
+    while (to_skip > 0) {
+        size_t read_size = sizeof(buf);
+        if (read_size > to_skip)
+            read_size = to_skip;
+
+        bytes = read_streamfile(buf, data->curr_offset, read_size, sf);
+        if (!bytes) /* ??? */
+            break;
+
+        decrypt_chunk(buf, bytes, data); /* updates curr_offset and key */
+        to_skip -= bytes;
+    }
+}
+
+/* XOR depends on decrypted data, meanings having to decrypt linearly to reach some offset. */
+static size_t mc161_io_read(STREAMFILE* sf, uint8_t* dest, off_t offset, size_t length, mc161_io_data* data) {
+    size_t bytes;
+
+    /* read and decrypt unneded data */
+    update_key(sf, offset, data);
+
+    /* read and decrypt current data */
+    bytes = read_streamfile(dest, offset, length, sf);
+    decrypt_chunk(dest, bytes, data);
+
+    return bytes;
+}
+
+/* String.hashCode() should be equivalent to this on Windows (though implementation-defined), note Java has no unsigned */
+static int32_t mc161_get_java_hashcode(STREAMFILE* sf) {
+    char filename[1024];
+    int i = 0;
+    int32_t hash = 0;
+
+    get_streamfile_filename(sf, filename, sizeof(filename));
+
+    while (filename[i] != '\0') {
+        hash = 31 * hash + (uint8_t)filename[i];
+        i++;
+    }
+
+    return hash;
+}
+
+/* decrypts Minecraft old streams (some info from: https://github.com/ata4/muscode) */
+static STREAMFILE* setup_mc161_streamfile(STREAMFILE* sf) {
+    STREAMFILE* new_sf = NULL;
+    mc161_io_data io_data = {0};
+
+    io_data.base_key = mc161_get_java_hashcode(sf);
+    io_data.curr_key = io_data.base_key;
+    io_data.curr_offset = 0x00;
+
+    new_sf = open_wrap_streamfile(sf);
+    new_sf = open_io_streamfile_f(new_sf, &io_data, sizeof(mc161_io_data), mc161_io_read, NULL);
+    new_sf = open_fakename_streamfile_f(new_sf, NULL, "ogg");
+    return new_sf;
+}
+
+#endif /* _MC161_STREAMFILE_H_ */

src/meta/riff.c

@@ -103,21 +103,21 @@ static int read_fmt(int big_endian, STREAMFILE* sf, off_t offset, riff_fmt_chunk
     fmt->size = read_u32(offset+0x04,sf);
 
     /* WAVEFORMAT */
-    fmt->codec          = read_u16(offset+0x08,sf);
+    fmt->codec          = read_u16(offset+0x08+0x00,sf);
-    fmt->channels       = read_u16(offset+0x0a,sf);
+    fmt->channels       = read_u16(offset+0x08+0x02,sf);
-    fmt->sample_rate    = read_u32(offset+0x0c,sf);
+    fmt->sample_rate    = read_u32(offset+0x08+0x04,sf);
-  //fmt->avg_bps        = read_u32(offset+0x10,sf);
+  //fmt->avg_bps        = read_u32(offset+0x08+0x08,sf);
-    fmt->block_size     = read_u16(offset+0x14,sf);
+    fmt->block_size     = read_u16(offset+0x08+0x0c,sf);
-    fmt->bps            = read_u16(offset+0x16,sf);
+    fmt->bps            = read_u16(offset+0x08+0x0e,sf);
     /* WAVEFORMATEX */
     if (fmt->size >= 0x10) {
-        fmt->extra_size = read_u16(offset+0x18,sf);
+        fmt->extra_size = read_u16(offset+0x08+0x10,sf);
         /* 0x1a+ depends on codec (ex. coef table for MSADPCM, samples_per_frame in MS-IMA, etc) */
     }
     /* WAVEFORMATEXTENSIBLE */
     if (fmt->codec == 0xFFFE && fmt->extra_size >= 0x16) {
-      //fmt->extra_samples  = read_u16(offset+0x1a,sf); /* valid_bits_per_sample or samples_per_block */
+      //fmt->extra_samples  = read_u16(offset+0x08+0x12,sf); /* valid_bits_per_sample or samples_per_block */
-        fmt->channel_layout = read_u32(offset+0x1c,sf);
+        fmt->channel_layout = read_u32(offset+0x08+0x14,sf);
         /* 0x10 guid at 0x20 */
 
         /* happens in various .at3/at9, may be a bug in their encoder b/c MS's defs set mono as FC */
@@ -135,7 +135,7 @@ static int read_fmt(int big_endian, STREAMFILE* sf, off_t offset, riff_fmt_chunk
         goto fail;
 
     switch (fmt->codec) {
-        case 0x00:  /* Yamaha AICA ADPCM [Headhunter (DC), Bomber hehhe (DC), Rayman 2 (DC)] (unofficial) */
+        case 0x0000:  /* Yamaha AICA ADPCM [Headhunter (DC), Bomber hehhe (DC), Rayman 2 (DC)] (unofficial) */
             if (fmt->bps != 4) goto fail;
             if (fmt->block_size != 0x02*fmt->channels &&
                 fmt->block_size != 0x01*fmt->channels) goto fail;
@@ -143,7 +143,7 @@ static int read_fmt(int big_endian, STREAMFILE* sf, off_t offset, riff_fmt_chunk
             fmt->interleave = 0x01;
             break;
 
-        case 0x01: /* PCM */
+        case 0x0001: /* PCM */
             switch (fmt->bps) {
                 case 24: /* Omori (PC) */
                     fmt->coding_type = coding_PCM24LE;
@@ -160,7 +160,7 @@ static int read_fmt(int big_endian, STREAMFILE* sf, off_t offset, riff_fmt_chunk
             fmt->interleave = fmt->block_size / fmt->channels;
             break;
 
-        case 0x02: /* MSADPCM */
+        case 0x0002: /* MSADPCM */
             if (fmt->bps == 4) {
                 fmt->coding_type = coding_MSADPCM;
                 if (!msadpcm_check_coefs(sf, fmt->offset + 0x08 + 0x14))
@@ -174,19 +174,29 @@ static int read_fmt(int big_endian, STREAMFILE* sf, off_t offset, riff_fmt_chunk
             }
             break;
 
-        case 0x11:  /* MS-IMA ADPCM [Layton Brothers: Mystery Room (iOS/Android)] */
+        case 0x0011:  /* MS-IMA ADPCM [Layton Brothers: Mystery Room (iOS/Android)] */
             if (fmt->bps != 4) goto fail;
             fmt->coding_type = coding_MS_IMA;
             break;
 
-        case 0x20:  /* Yamaha AICA ADPCM [Takuyo/Dynamix/etc DC games] */
+        case 0x0020:  /* Yamaha AICA ADPCM [Takuyo/Dynamix/etc DC games] (official-ish) */
             if (fmt->bps != 4) goto fail;
             fmt->coding_type = coding_AICA;
             /* official RIFF spec has 0x20 as 'Yamaha ADPCM', but data is probably not pure AICA
              * (maybe with headered frames and would need extra detection?) */
             break;
 
-        case 0x69:  /* XBOX IMA ADPCM [Dynasty Warriors 5 (Xbox)] */
+#ifdef VGM_USE_MPEG
+        case 0x0055: /* MP3 [Bear in the Big Blue House: Bear's Imagine That! (PC)] (official) */
+            fmt->coding_type = coding_MPEG_custom;
+            /* some oddities, unsure if part of standard: 
+             * - block size is 1 (in mono)
+             * - bps is 16
+             * - extra size 0x0c, has channels? and (possibly) approx frame size */
+            break;
+#endif
+
+        case 0x0069:  /* XBOX IMA ADPCM [Dynasty Warriors 5 (Xbox)] */
             if (fmt->bps != 4) goto fail;
             fmt->coding_type = coding_XBOX_IMA;
             break;
@@ -671,7 +681,11 @@ VGMSTREAM* init_vgmstream_riff(STREAMFILE* sf) {
             vgmstream->layout_type = layout_none;
             vgmstream->interleave_block_size = fmt.block_size;
             break;
-
+#ifdef VGM_USE_MPEG
+        case coding_MPEG_custom:
+            vgmstream->layout_type = layout_none;
+            break;
+#endif
         case coding_MSADPCM:
             vgmstream->layout_type = layout_none;
             vgmstream->frame_size = fmt.block_size;
@@ -814,6 +828,21 @@ VGMSTREAM* init_vgmstream_riff(STREAMFILE* sf) {
         }
 #endif
 
+#ifdef VGM_USE_MPEG
+        case coding_MPEG_custom: {
+            mpeg_custom_config cfg = {0};
+
+            vgmstream->codec_data = init_mpeg_custom(sf, start_offset, &vgmstream->coding_type, fmt.channels, MPEG_STANDARD, &cfg);
+            if (!vgmstream->codec_data) goto fail;
+
+            /* should provide "fact" but it's optional (some game files don't include it) */
+            if (!fact_sample_count)
+                fact_sample_count = mpeg_get_samples(sf, start_offset, data_size);
+            vgmstream->num_samples = fact_sample_count;
+        }
+        break;
+#endif
+
         default:
             goto fail;
     }

src/meta/rxws.c

@@ -2,54 +2,56 @@
 #include "../util.h"
 #include "../coding/coding.h"
 
-/* RXWS - from Sony SCEI PS2 games (Okage: Shadow King, Genji, Bokura no Kazoku) */
+/* RXWS - from Sony SCEI games [Okage: Shadow King (PS2), Genji (PS2), Bokura no Kazoku (PS2))] */
 VGMSTREAM* init_vgmstream_rxws(STREAMFILE* sf) {
     VGMSTREAM* vgmstream = NULL;
-    STREAMFILE* sh = NULL;
+    STREAMFILE* sf_head = NULL;
+    STREAMFILE* sf_body = NULL;
     off_t start_offset, chunk_offset, name_offset = 0;
     size_t stream_size, chunk_size;
-    int loop_flag = 0, channels, is_separate = 0, type, sample_rate;
+    int loop_flag = 0, channels, is_xwh = 0, type, sample_rate;
     int32_t num_samples, loop_start;
     int total_subsongs, target_subsong = sf->stream_index;
 
+
+    /* for plugins that start with .xwb */
+    if (check_extensions(sf,"xwb")) {
+        /* extra check to reject Microsoft's XWB faster */
+        if (is_id32be(0x00,sf,"WBND") || is_id32be(0x00,sf,"DNBW")) /* LE/BE */
+            goto fail;
+
+        sf_head = open_streamfile_by_ext(sf, "xwh");
+        if (!sf_head) goto fail;
+    }
+    else {
+        sf_head = sf;
+    }
+
+    if (!is_id32be(0x00,sf_head,"RXWS"))
+        goto fail;
+
     /* checks */
     /* .xws: header and data
      * .xwh+xwb: header + data (.bin+dat are also found in Wild Arms 4/5) */
     if (!check_extensions(sf,"xws,xwb"))
         goto fail;
-    is_separate = check_extensions(sf,"xwb");
-
-    /* xwh+xwb: use xwh as header; otherwise use the current file */
-    if (is_separate) {
-        /* extra check to reject Microsoft's XWB faster */
-        if (is_id32be(0x00,sf,"WBND") ||    /* (LE) */
-            is_id32be(0x00,sf,"DNBW"))      /* (BE) */
-            goto fail;
-
-        sh = open_streamfile_by_ext(sf, "xwh");
-        if (!sh) goto fail;
-    } else {
-        sh = sf;
-    }
-    if (!is_id32be(0x00,sh,"RXWS"))
-        goto fail;
 
     /* file size (just the .xwh/xws) */
-    if (read_u32le(0x04,sh) + 0x10 != get_streamfile_size(sh))
+    if (read_u32le(0x04,sf_head) + 0x10 != get_streamfile_size(sf_head))
         goto fail;
     /* 0x08: version (0x100/0x200)
      * 0x0C: null */
 
     /* typical chunks: FORM, FTXT, MARK, BODY (for .xws) */
-    if (!is_id32be(0x10,sh,"FORM")) /* main header (always first) */
+    if (!is_id32be(0x10,sf_head,"FORM")) /* main header (always first) */
         goto fail;
-    chunk_size = read_u32le(0x10+0x04,sh); /* size - 0x10 */
+    chunk_size = read_u32le(0x10+0x04,sf_head); /* size - 0x10 */
     /* 0x08 version (0x100), 0x0c: null */
     chunk_offset = 0x20;
 
 
     /* check multi-streams */
-    total_subsongs = read_s32le(chunk_offset+0x00,sh);
+    total_subsongs = read_s32le(chunk_offset+0x00,sf_head);
     if (target_subsong == 0) target_subsong = 1;
     if (target_subsong < 0 || target_subsong > total_subsongs || total_subsongs < 1) goto fail;
 
@@ -57,54 +59,66 @@ VGMSTREAM* init_vgmstream_rxws(STREAMFILE* sf) {
     /* read stream header */
     {
         off_t header_offset = chunk_offset + 0x4 + 0x1c * (target_subsong-1); /* position in FORM */
-        off_t stream_offset, next_stream_offset, data_offset = 0;
+        off_t stream_offset, next_stream_offset, body_offset;
 
-        type = read_u8(header_offset+0x00, sh);
+        type = read_u8(header_offset+0x00, sf_head);
         /* 0x01: unknown (always 0x1c) */
         /* 0x02: flags? (usually 8002/0002, & 0x01 if looped) */
         /* 0x04: vol/pan stuff? (0x00007F7F) */
         /* 0x08: null? */
-        channels =    read_u8(header_offset+0x09, sh);
+        channels      =    read_u8(header_offset+0x09, sf_head);
+        sample_rate   = read_u16le(header_offset+0x0a,sf_head);
         /* 0x0c: null? */
-        sample_rate   = read_u16le(header_offset+0x0a,sh);
+        stream_offset = read_u32le(header_offset+0x10,sf_head);
-        stream_offset = read_u32le(header_offset+0x10,sh);
+        num_samples   = read_s32le(header_offset+0x14,sf_head);
-        num_samples   = read_s32le(header_offset+0x14,sh);
+        loop_start    = read_s32le(header_offset+0x18,sf_head);
-        loop_start    = read_s32le(header_offset+0x18,sh);
         loop_flag = (loop_start >= 0);
 
-        /* find data start and size */
+        /* find body start and size (needed for stream_size) */
-        if (is_separate) {
+        {
-            data_offset = 0x00;
+            uint32_t current_chunk = 0x10;
-        }
+
-        else {
+            body_offset = 0x00;
-            off_t current_chunk = 0x10;
+            while (current_chunk < get_streamfile_size(sf_head)) {
-            /* note the extra 0x10 in chunk_size/offsets */
+                if (is_id32be(current_chunk,sf_head, "BODY")) {
-            while (current_chunk < get_streamfile_size(sf)) {
+                    body_offset = 0x10 + current_chunk;
-                if (is_id32be(current_chunk,sf, "BODY")) {
+                    is_xwh = 1;
-                    data_offset = 0x10 + current_chunk;
                     break;
                 }
-                current_chunk += 0x10 + read_u32le(current_chunk+4,sf);
+                /* note the extra 0x10 in chunk_size/offsets */
+                current_chunk += 0x10 + read_u32le(current_chunk + 0x04,sf_head);
+            }
+
+            /* .xwh and .xws are only different in that the latter has BODY chunk (no flags/sizes) */
+            is_xwh = !body_offset;
+
+            /* for plugins that start with .xwh (and don't check extensions) */
+            if (is_xwh && sf == sf_head) {
+                sf_body = open_streamfile_by_ext(sf, "xwb");
+                if (!sf_body) goto fail;
+            }
+            else {
+                sf_body = sf;
             }
-            if (!data_offset) goto fail;
         }
 
         if (target_subsong == total_subsongs) {
-            next_stream_offset = get_streamfile_size(is_separate ? sf : sh) - data_offset;
+            uint32_t max_size = get_streamfile_size(sf_body);
+            next_stream_offset = max_size - body_offset;
         } else {
             off_t next_header_offset = chunk_offset + 0x4 + 0x1c * (target_subsong);
-            next_stream_offset = read_u32le(next_header_offset+0x10,sh);
+            next_stream_offset = read_u32le(next_header_offset+0x10, sf_head);
         }
 
         stream_size = next_stream_offset - stream_offset;
-        start_offset = data_offset + stream_offset;
+        start_offset = body_offset + stream_offset;
     }
 
     /* get stream name (always follows FORM) */
-    if (is_id32be(0x10+0x10 + chunk_size,sh, "FTXT")) {
+    if (is_id32be(0x10+0x10 + chunk_size,sf_head, "FTXT")) {
         chunk_offset = 0x10+0x10 + chunk_size + 0x10;
-        if (read_s32le(chunk_offset+0x00,sh) == total_subsongs) {
+        if (read_s32le(chunk_offset+0x00,sf_head) == total_subsongs) {
-            name_offset = chunk_offset + read_u32le(chunk_offset+0x04 + (target_subsong-1)*0x04,sh);
+            name_offset = chunk_offset + read_u32le(chunk_offset+0x04 + (target_subsong-1)*0x04,sf_head);
         }
     }
 
@@ -118,7 +132,7 @@ VGMSTREAM* init_vgmstream_rxws(STREAMFILE* sf) {
     vgmstream->num_streams = total_subsongs;
     vgmstream->stream_size = stream_size;
     if (name_offset)
-        read_string(vgmstream->stream_name,STREAM_NAME_SIZE, name_offset,sh);
+        read_string(vgmstream->stream_name,STREAM_NAME_SIZE, name_offset, sf_head);
 
     switch (type) {
         case 0x00:      /* PS-ADPCM */
@@ -149,7 +163,7 @@ VGMSTREAM* init_vgmstream_rxws(STREAMFILE* sf) {
             encoder_delay = 1024 + 69*2; /* observed default */
             vgmstream->num_samples = num_samples - encoder_delay;
 
-            vgmstream->codec_data = init_ffmpeg_atrac3_raw(sf, start_offset,stream_size, vgmstream->num_samples,vgmstream->channels,vgmstream->sample_rate, block_align, encoder_delay);
+            vgmstream->codec_data = init_ffmpeg_atrac3_raw(sf_body, start_offset,stream_size, vgmstream->num_samples,vgmstream->channels,vgmstream->sample_rate, block_align, encoder_delay);
             if (!vgmstream->codec_data) goto fail;
             vgmstream->coding_type = coding_FFmpeg;
             vgmstream->layout_type = layout_none;
@@ -164,14 +178,16 @@ VGMSTREAM* init_vgmstream_rxws(STREAMFILE* sf) {
     }
 
     /* open the file for reading */
-    if (!vgmstream_open_stream(vgmstream, sf, start_offset))
+    if (!vgmstream_open_stream(vgmstream, sf_body, start_offset))
         goto fail;
 
-    if (is_separate && sh) close_streamfile(sh);
+    if (sf != sf_head) close_streamfile(sf_head);
+    if (sf != sf_body) close_streamfile(sf_body);
     return vgmstream;
 
 fail:
-    if (is_separate && sh) close_streamfile(sh);
+    if (sf != sf_head) close_streamfile(sf_head);
+    if (sf != sf_body) close_streamfile(sf_body);
     close_vgmstream(vgmstream);
     return NULL;
 }

src/meta/sgxd.c

@@ -6,93 +6,123 @@
 VGMSTREAM* init_vgmstream_sgxd(STREAMFILE* sf) {
     VGMSTREAM* vgmstream = NULL;
     STREAMFILE* sf_head = NULL;
+    STREAMFILE* sf_body = NULL;
     off_t start_offset, data_offset, chunk_offset, name_offset = 0;
     size_t stream_size;
+    uint32_t base1_offset, base2_offset, base3_offset;
 
-    int is_sgx, is_sgb = 0;
+    int is_sgx, is_sgd = 0;
-    int loop_flag, channels, codec;
+    int loop_flag, channels, codec, sample_rate;
-    int sample_rate, num_samples, loop_start_sample, loop_end_sample;
+    int32_t num_samples, loop_start_sample, loop_end_sample;
     int total_subsongs, target_subsong = sf->stream_index;
 
 
-    /* check extension, case insensitive */
+    /* for plugins that start with .sgb */
-    /* .sgx: header+data (Genji), .sgd: header+data, .sgh/sgd: header/data */
+    if (check_extensions(sf,"sgb")) {
-    if (!check_extensions(sf,"sgx,sgd,sgb"))
-        goto fail;
-    is_sgx = check_extensions(sf,"sgx");
-    is_sgb = check_extensions(sf,"sgb");
-
-    /* SGB+SGH: use SGH as header; otherwise use the current file as header */
-    if (is_sgb) {
         sf_head = open_streamfile_by_ext(sf, "sgh");
         if (!sf_head) goto fail;
-    } else {
+    }
+    else {
         sf_head = sf;
     }
 
-
+    if (!is_id32be(0x00,sf_head, "SGXD"))
-    /* SGXD base (size 0x10) */
-    if (read_32bitBE(0x00,sf_head) != 0x53475844) /* "SGXD" */
         goto fail;
-    /* 0x04  SGX: full header_size; SGD/SGH: unknown header_size (counting from 0x0/0x8/0x10, varies) */
+
-    /* 0x08  SGX: first chunk offset? (0x10); SGD/SGH: full header_size */
+    /* checks */
-    /* 0x0c  SGX/SGH: full data size with padding; SGD: full data size + 0x80000000 with padding */
+    /* .sgx: header+data (Genji)
-    if (is_sgb) {
+     * .sgd: header+data (common)
-        data_offset = 0x00;
+     * .sgh+sgd: header+data */
-    } else if ( is_sgx ) {
+    if (!check_extensions(sf,"sgx,sgd,sgb"))
-        data_offset = read_32bitLE(0x04,sf_head);
+        goto fail;
+
+    /* SGXD base (size 0x10), always LE even on PS3 */
+    /* 0x04: SGX = full header size
+             SGD/SGH = bank name offset (part of NAME table, usually same as filename) */
+    /* 0x08: SGX = first chunk offset? (0x10) 
+             SGD/SGH = full header size */
+    /* 0x0c: SGX/SGH = full data size with padding / 
+             SGD = full data size ^ (1<<31) with padding */
+    base1_offset = read_u32le(0x04, sf_head);
+    base2_offset = read_u32le(0x08, sf_head);
+    base3_offset = read_u32le(0x0c, sf_head);
+
+    is_sgx = base2_offset == 0x10; /* fixed size */
+    is_sgd = base3_offset & (1 << 31); /* flag */
+
+    /* Ogg SGXD don't have flag (probably due to codec hijack, or should be split), allow since it's not so obvious */
+    if (!(is_sgx || is_sgd) && get_streamfile_size(sf_head) != base2_offset) /* sgh but wrong header size must be sgd */
+        is_sgd = 1;
+
+    /* for plugins that start with .sgh (and don't check extensions) */
+    if (!(is_sgx || is_sgd) && sf == sf_head) {
+        sf_body = open_streamfile_by_ext(sf, "sgb");
+        if (!sf_body) goto fail;
+    }
+    else {
+        sf_body = sf;
+    }
+
+
+    if (is_sgx) {
+        data_offset = base1_offset;
+    } else if (is_sgd) {
+        data_offset = base2_offset;
     } else {
-        data_offset = read_32bitLE(0x08,sf_head);
+        data_offset = 0x00;
     }
 
 
     /* typical chunks: WAVE, RGND, NAME (strings for WAVE or RGND), SEQD (related to SFX), WSUR, WMKR, BUSS */
     /* WAVE chunk (size 0x10 + files * 0x38 + optional padding) */
     if (is_sgx) { /* position after chunk+size */
-        if (read_32bitBE(0x10,sf_head) != 0x57415645) goto fail;  /* "WAVE" */
+        if (!is_id32be(0x10,sf_head, "WAVE"))
+            goto fail;
         chunk_offset = 0x18;
     } else {
-        if (!find_chunk_le(sf_head, 0x57415645,0x10,0, &chunk_offset,NULL)) goto fail; /* "WAVE" */
+        if (!find_chunk_le(sf_head, get_id32be("WAVE"),0x10,0, &chunk_offset, NULL))
+            goto fail;
     }
     /* 0x04  SGX: unknown; SGD/SGH: chunk length,  0x08  null */
 
     /* check multi-streams (usually only SE containers; Puppeteer) */
-    total_subsongs = read_32bitLE(chunk_offset+0x04,sf_head);
+    total_subsongs = read_s32le(chunk_offset+0x04,sf_head);
     if (target_subsong == 0) target_subsong = 1;
     if (target_subsong < 0 || target_subsong > total_subsongs || total_subsongs < 1) goto fail;
 
     /* read stream header */
     {
-        off_t stream_offset;
+        uint32_t stream_offset;
         chunk_offset += 0x08 + 0x38 * (target_subsong-1); /* position in target header*/
 
         /* 0x00  ? (00/01/02) */
         if (!is_sgx) /* meaning unknown in .sgx; offset 0 = not a stream (a RGND sample) */
-            name_offset = read_32bitLE(chunk_offset+0x04,sf_head);
+            name_offset = read_u32le(chunk_offset+0x04,sf_head);
-        codec = read_8bit(chunk_offset+0x08,sf_head);
+        codec = read_u8(chunk_offset+0x08,sf_head);
-        channels = read_8bit(chunk_offset+0x09,sf_head);
+        channels = read_u8(chunk_offset+0x09,sf_head);
         /* 0x0a  null */
-        sample_rate = read_32bitLE(chunk_offset+0x0c,sf_head);
+        sample_rate = read_s32le(chunk_offset+0x0c,sf_head);
 
-        /* 0x10  info_type: meaning of the next value
+        /* 0x10: info_type, meaning of the next value
          *       (00=null, 30/40=data size without padding (ADPCM, ATRAC3plus), 80/A0=block size (AC3) */
-        /* 0x14  info_value (see above) */
+        /* 0x14: info_value (see above) */
-        /* 0x18  unknown (ex. 0x0008/0010/3307/CC02/etc)x2 */
+        /* 0x18: unknown (ex. 0x0008/0010/3307/CC02/etc)x2 */
-        /* 0x1c  null */
+        /* 0x1c: null */
 
-        num_samples = read_32bitLE(chunk_offset+0x20,sf_head);
+        num_samples = read_s32le(chunk_offset+0x20,sf_head);
-        loop_start_sample = read_32bitLE(chunk_offset+0x24,sf_head);
+        loop_start_sample = read_s32le(chunk_offset+0x24,sf_head);
-        loop_end_sample = read_32bitLE(chunk_offset+0x28,sf_head);
+        loop_end_sample = read_s32le(chunk_offset+0x28,sf_head);
-        stream_size = read_32bitLE(chunk_offset+0x2c,sf_head); /* stream size (without padding) / interleave (for type3) */
+        stream_size = read_u32le(chunk_offset+0x2c,sf_head); /* stream size (without padding) / interleave (for type3) */
 
         if (is_sgx) {
             stream_offset = 0x0;
         } else{
-            stream_offset = read_32bitLE(chunk_offset+0x30,sf_head);
+            stream_offset = read_u32le(chunk_offset+0x30,sf_head);
         }
-        /* 0x34 SGX: unknown; SGD/SGH: stream size (with padding) / interleave */
+        /* 0x34: SGX = unknown
+         *       SGD/SGH = stream size (with padding) / interleave */
 
-        loop_flag = loop_start_sample!=0xffffffff && loop_end_sample!=0xffffffff;
+        loop_flag = loop_start_sample != -1 && loop_end_sample != -1;
         start_offset = data_offset + stream_offset;
     }
 
@@ -121,7 +151,7 @@ VGMSTREAM* init_vgmstream_sgxd(STREAMFILE* sf) {
 
 #ifdef VGM_USE_VORBIS
         case 0x02:      /* Ogg Vorbis [Ni no Kuni: Wrath of the White Witch Remastered (PC)] (codec hijack?) */
-            vgmstream->codec_data = init_ogg_vorbis(sf, start_offset, stream_size, NULL);
+            vgmstream->codec_data = init_ogg_vorbis(sf_body, start_offset, stream_size, NULL);
             if (!vgmstream->codec_data) goto fail;
             vgmstream->coding_type = coding_OGG_VORBIS;
             vgmstream->layout_type = layout_none;
@@ -130,7 +160,7 @@ VGMSTREAM* init_vgmstream_sgxd(STREAMFILE* sf) {
         case 0x03:      /* PS-ADPCM [Genji (PS3), Ape Escape Move (PS3)]*/
             vgmstream->coding_type = coding_PSX;
             vgmstream->layout_type = layout_interleave;
-            if (is_sgx || is_sgb) {
+            if (!is_sgd) {
                 vgmstream->interleave_block_size = 0x10;
             } else { /* this only seems to happen with SFX */
                 vgmstream->interleave_block_size = stream_size;
@@ -143,7 +173,7 @@ VGMSTREAM* init_vgmstream_sgxd(STREAMFILE* sf) {
 
 #ifdef VGM_USE_FFMPEG
         case 0x04: {    /* ATRAC3plus [Kurohyo 1/2 (PSP), BraveStory (PSP)] */
-            vgmstream->codec_data = init_ffmpeg_atrac3_riff(sf, start_offset, NULL);
+            vgmstream->codec_data = init_ffmpeg_atrac3_riff(sf_body, start_offset, NULL);
             if (!vgmstream->codec_data) goto fail;
             vgmstream->coding_type = coding_FFmpeg;
             vgmstream->layout_type = layout_none;
@@ -163,7 +193,7 @@ VGMSTREAM* init_vgmstream_sgxd(STREAMFILE* sf) {
 
 #ifdef VGM_USE_FFMPEG
         case 0x06: {    /* AC3 [Tokyo Jungle (PS3), Afrika (PS3)] */
-            vgmstream->codec_data = init_ffmpeg_offset(sf, start_offset, stream_size);
+            vgmstream->codec_data = init_ffmpeg_offset(sf_body, start_offset, stream_size);
             if (!vgmstream->codec_data) goto fail;
             vgmstream->coding_type = coding_FFmpeg;
             vgmstream->layout_type = layout_none;
@@ -173,7 +203,6 @@ VGMSTREAM* init_vgmstream_sgxd(STREAMFILE* sf) {
             ffmpeg_set_skip_samples(vgmstream->codec_data, 256);
 
             /* SGXD loop/sample values are relative (without skip samples), no need to adjust */
-
             break;
         }
 #endif
@@ -183,14 +212,16 @@ VGMSTREAM* init_vgmstream_sgxd(STREAMFILE* sf) {
             goto fail;
     }
 
-    if (!vgmstream_open_stream(vgmstream, sf, start_offset))
+    if (!vgmstream_open_stream(vgmstream, sf_body, start_offset))
         goto fail;
 
-    if (is_sgb && sf_head) close_streamfile(sf_head);
+    if (sf != sf_head) close_streamfile(sf_head);
+    if (sf != sf_body) close_streamfile(sf_body);
     return vgmstream;
 
 fail:
-    if (is_sgb && sf_head) close_streamfile(sf_head);
+    if (sf != sf_head) close_streamfile(sf_head);
+    if (sf != sf_body) close_streamfile(sf_body);
     close_vgmstream(vgmstream);
     return NULL;
 }

src/streamfile.c

@@ -94,10 +94,10 @@ static STREAMFILE* open_stdio_streamfile_buffer_by_file(FILE *infile, const char
 static size_t stdio_read(STDIO_STREAMFILE* sf, uint8_t* dst, offv_t offset, size_t length) {
     size_t read_total = 0;
 
-    if (!sf->infile || !dst || length <= 0 || offset < 0)
+    if (/*!sf->infile ||*/ !dst || length <= 0 || offset < 0)
         return 0;
 
-    //;VGM_LOG("stdio: read %lx + %x (buf %lx + %x)\n", offset, length, sf->buf_offset, sf->valid_size);
+    //;VGM_LOG("stdio: read %lx + %x (buf %lx + %x)\n", offset, length, sf->buf_offset, sf->valid_size, sf->buf_size);
 
     /* is the part of the requested length in the buffer? */
     if (offset >= sf->buf_offset && offset < sf->buf_offset + sf->valid_size) {
@@ -125,6 +125,10 @@ static size_t stdio_read(STDIO_STREAMFILE* sf, uint8_t* dst, offv_t offset, size
     }
 #endif
 
+    /* possible if all data was copied to buf and FD closed */
+    if (!sf->infile)
+        return read_total;
+
     /* read the rest of the requested length */
     while (length > 0) {
         size_t length_to_read;
@@ -180,12 +184,15 @@ static size_t stdio_read(STDIO_STREAMFILE* sf, uint8_t* dst, offv_t offset, size
     sf->offset = offset; /* last fread offset */
     return read_total;
 }
+
 static size_t stdio_get_size(STDIO_STREAMFILE* sf) {
     return sf->file_size;
 }
+
 static offv_t stdio_get_offset(STDIO_STREAMFILE* sf) {
     return sf->offset;
 }
+
 static void stdio_get_name(STDIO_STREAMFILE* sf, char* name, size_t name_size) {
     int copy_size = sf->name_len + 1;
     if (copy_size > name_size)
@@ -222,7 +229,7 @@ static STREAMFILE* stdio_open(STDIO_STREAMFILE* sf, const char* const filename,
         /* on failure just close and try the default path (which will probably fail a second time) */
     }
 #endif
-    // a normal open, open a new file
+
     return open_stdio_streamfile_buffer(filename, buf_size);
 }
 
@@ -271,13 +278,29 @@ static STREAMFILE* open_stdio_streamfile_buffer_by_file(FILE* infile, const char
         this_sf->file_size = 0; /* allow virtual, non-existing files */
     }
 
-    /* Typically fseek(o)/ftell(o) may only handle up to ~2.14GB, signed 32b = 0x7FFFFFFF
+    /* Typically fseek(o)/ftell(o) may only handle up to ~2.14GB, signed 32b = 0x7FFFFFFF (rarely
-     * (happens in banks like FSB, though rarely). Should work if configured properly, log otherwise. */
+     * happens in giant banks like FSB/KTSR). Should work if configured properly using ftell_v, log otherwise. */
     if (this_sf->file_size == 0xFFFFFFFF) { /* -1 on error */
         vgm_logi("STREAMFILE: file size too big (report)\n");
         goto fail; /* can be ignored but may result in strange/unexpected behaviors */
     }
 
+    /* Rarely a TXTP needs to open *many* streamfiles = many file descriptors = reaches OS limit = error.
+     * Ideally should detect better and open/close as needed or reuse FDs for files that don't play at
+     * the same time, but it's complex since every SF is separate (would need some kind of FD manager).
+     * For the time being, if the file is smaller that buffer we can just read it fully and close the FD,
+     * that should help since big TXTP usually just need many small files.
+     * Doubles as an optimization as most files given will be read fully into buf on first read. */
+    if (this_sf->file_size && this_sf->file_size < this_sf->buf_size && this_sf->infile) {
+        //;VGM_LOG("stdio: fit filesize %x into buf %x\n", sf->file_size, sf->buf_size);
+
+        this_sf->buf_offset = 0;
+        this_sf->valid_size = fread(this_sf->buf, sizeof(uint8_t), this_sf->file_size, this_sf->infile);
+
+        fclose(this_sf->infile);
+        this_sf->infile = NULL;
+    }
+
     return &this_sf->vt;
 
 fail:

winamp/in_streamfile.c

@@ -19,6 +19,8 @@ static FILE* wa_fopen(const in_char* wpath) {
 #endif
 }
 
+/* in theory fdopen and _wfdopen are the same, except flags is a wchar */
+#if 0
 /* dupes a utf16 (unicode) file */
 static FILE* wa_fdopen(int fd) {
 #ifdef UNICODE_INPUT_PLUGIN
@@ -27,6 +29,7 @@ static FILE* wa_fdopen(int fd) {
     return fdopen(fd,"rb");
 #endif
 }
+#endif
 
 /* ************************************* */
 /* IN_STREAMFILE                         */
@@ -36,7 +39,6 @@ static FILE* wa_fdopen(int fd) {
 typedef struct {
     STREAMFILE vt;
     STREAMFILE* stdiosf;
-    FILE* infile_ref; /* pointer to the infile in stdiosf (partially handled by stdiosf) */
 } WINAMP_STREAMFILE;
 
 static STREAMFILE* open_winamp_streamfile_by_file(FILE* infile, const char* path);
@@ -64,32 +66,7 @@ static STREAMFILE* wasf_open(WINAMP_STREAMFILE* sf, const char* const filename,
     if (!filename)
         return NULL;
 
-#if !defined (__ANDROID__) && !defined (_MSC_VER)
+    /* no need to wfdopen here, may use standard IO */
-    /* When enabling this for MSVC it'll seemingly work, but there are issues possibly related to underlying
-     * IO buffers when using dup(), noticeable by re-opening the same streamfile with small buffer sizes
-     * (reads garbage). This reportedly causes issues in Android too */
-    {
-        char name[PATH_LIMIT];
-        sf->stdiosf->get_name(sf->stdiosf, name, PATH_LIMIT);
-        /* if same name, duplicate the file descriptor we already have open */ //unsure if all this is needed
-        if (sf->infile_ref && !strcmp(name,filename)) {
-            int new_fd;
-            FILE *new_file;
-
-            if (((new_fd = dup(fileno(sf->infile_ref))) >= 0) && (new_file = wa_fdopen(new_fd))) {
-                STREAMFILE* new_sf = open_winamp_streamfile_by_file(new_file, filename);
-                if (new_sf)
-                    return new_sf;
-                fclose(new_file);
-            }
-            if (new_fd >= 0 && !new_file)
-                close(new_fd); /* fdopen may fail when opening too many files */
-
-            /* on failure just close and try the default path (which will probably fail a second time) */
-        }
-    }
-#endif
-
     /* STREAMFILEs carry char/UTF8 names, convert to wchar for Winamp */
     wa_char_to_ichar(wpath, PATH_LIMIT, filename);
     return open_winamp_streamfile_by_ipath(wpath);
@@ -119,7 +96,6 @@ static STREAMFILE* open_winamp_streamfile_by_file(FILE* file, const char* path)
     this_sf->vt.close = (void*)wasf_close;
 
     this_sf->stdiosf = stdiosf;
-    this_sf->infile_ref = file;
 
     return &this_sf->vt;