From f6d27d66eb00d84b386cdd5245fbe4e5013f2996 Mon Sep 17 00:00:00 2001 From: bnnm Date: Sun, 26 Nov 2023 20:56:56 +0100 Subject: [PATCH 1/4] Fix some .awc [Red Read Redemption (PS4/SW)] --- src/layout/blocked_awc.c | 56 ++-- src/meta/awc.c | 571 +++++++++++++++++++++++++++------------ 2 files changed, 440 insertions(+), 187 deletions(-) diff --git a/src/layout/blocked_awc.c b/src/layout/blocked_awc.c index a2dda815..77f8c471 100644 --- a/src/layout/blocked_awc.c +++ b/src/layout/blocked_awc.c @@ -11,13 +11,19 @@ static size_t get_block_header_size(STREAMFILE* sf, off_t offset, size_t channel void block_update_awc(off_t block_offset, VGMSTREAM * vgmstream) { STREAMFILE* sf = vgmstream->ch[0].streamfile; int32_t (*read_32bit)(off_t,STREAMFILE*) = vgmstream->codec_endian ? read_32bitBE : read_32bitLE; - size_t header_size, entries, block_size, block_samples; - size_t channel_header_size; + size_t header_size, entries, block_size, block_samples, frame_size; + size_t channel_header_size; int i; - /* assumed only AWC_IMA enters here, MPEG/XMA2 need special parsing as blocked layout is too limited */ - entries = read_32bit(block_offset + 0x04, sf); /* se first channel, assume all are the same */ - //block_samples = entries * (0x800-4)*2; //todo use + /* assumes only AWC_IMA/DSP enters here, MPEG/XMA2 need special parsing as blocked layout is too limited. + * Block header (see awc.c for a complete description): + * - per channel: header table (size 0x18 or 0x10) + * - per channel: seek table (32b * entries = global samples per frame in each block) (not in DSP/Vorbis) + * - per channel: extra table (DSP only) + * - padding (not in ATRAC9/DSP) + */ + + entries = read_32bit(block_offset + 0x04, sf); /* se first channel, assume all are the same (not true in MPEG/XMA) */ block_samples = read_32bit(block_offset + 0x0c, sf); block_size = vgmstream->full_block_size; @@ -25,24 +31,32 @@ void block_update_awc(off_t block_offset, VGMSTREAM * vgmstream) { vgmstream->next_block_offset = block_offset + block_size; vgmstream->current_block_samples = block_samples; - /* starts with a header block */ - /* for each channel - * 0x00: start entry within channel (ie. entries * ch) but may be off by +1/+2 - * 0x04: entries - * 0x08: samples to discard in the beginning of this block (MPEG only?) - * 0x0c: samples in channel (for MPEG/XMA2 can vary between channels) - * (next fields don't exist in later versions for IMA) - * 0x10: (MPEG only, empty otherwise) close to number of frames but varies a bit? - * 0x14: (MPEG only, empty otherwise) channel usable data size (not counting padding) - * for each channel - * 32b * entries = global samples per frame in each block (for MPEG probably per full frame) - */ + switch(vgmstream->coding_type) { + case coding_NGC_DSP: + channel_header_size = 0x10; + frame_size = 0x08; + + /* coefs on every block but it's always the same */ + dsp_read_coefs_le(vgmstream, sf, block_offset + channel_header_size * vgmstream->channels + 0x10 + 0x1c + 0x00, 0x10 + 0x60); + dsp_read_hist_le (vgmstream, sf, block_offset + channel_header_size * vgmstream->channels + 0x10 + 0x1c + 0x20, 0x10 + 0x60); + + header_size = 0; + header_size += channel_header_size * vgmstream->channels; /* header table */ + /* no seek table */ + header_size += 0x70 * vgmstream->channels; /* extra table */ + /* no padding */ + + break; + + default: + channel_header_size = get_channel_header_size(sf, block_offset, vgmstream->codec_endian); + header_size = get_block_header_size(sf, block_offset, channel_header_size, vgmstream->channels, vgmstream->codec_endian); + frame_size = 0x800; + break; + } - channel_header_size = get_channel_header_size(sf, block_offset, vgmstream->codec_endian); - header_size = get_block_header_size(sf, block_offset, channel_header_size, vgmstream->channels, vgmstream->codec_endian); for (i = 0; i < vgmstream->channels; i++) { - vgmstream->ch[i].offset = block_offset + header_size + 0x800*entries*i; - VGM_ASSERT(entries != read_32bit(block_offset + channel_header_size*i + 0x04, sf), "AWC: variable number of entries found at %lx\n", block_offset); + vgmstream->ch[i].offset = block_offset + header_size + frame_size * entries * i; } } diff --git a/src/meta/awc.c b/src/meta/awc.c index be6ebf8f..65560183 100644 --- a/src/meta/awc.c +++ b/src/meta/awc.c @@ -3,10 +3,13 @@ #include "../layout/layout.h" #include "awc_xma_streamfile.h" + +#define AWC_MAX_MUSIC_CHANNELS 20 + typedef struct { int big_endian; int is_encrypted; - int is_music; + int is_streamed; /* implicit: streams=music, sfx=memory */ int total_subsongs; @@ -15,6 +18,7 @@ typedef struct { int codec; int num_samples; + int block_count; int block_chunk; off_t stream_offset; @@ -28,17 +32,17 @@ static int parse_awc_header(STREAMFILE* sf, awc_header* awc); static layered_layout_data* build_layered_awc(STREAMFILE* sf, awc_header* awc); -/* AWC - from RAGE (Rockstar Advanced Game Engine) audio [Red Dead Redemption, Max Payne 3, GTA5 (multi)] */ +/* AWC - Audio Wave Container from RAGE (Rockstar Advanced Game Engine) [Red Dead Redemption, Max Payne 3, GTA5 (multi)] */ VGMSTREAM* init_vgmstream_awc(STREAMFILE* sf) { VGMSTREAM* vgmstream = NULL; awc_header awc = {0}; /* checks */ - if (!check_extensions(sf,"awc")) - goto fail; if (!parse_awc_header(sf, &awc)) - goto fail; + return NULL; + if (!check_extensions(sf,"awc")) + return NULL; /* build the VGMSTREAM */ @@ -55,7 +59,7 @@ VGMSTREAM* init_vgmstream_awc(STREAMFILE* sf) { switch(awc.codec) { case 0x00: /* PCM (PC) sfx, very rare, lower sample rates? [Max Payne 3 (PC)] */ case 0x01: /* PCM (PC/PS3) sfx, rarely */ - if (awc.is_music) goto fail; /* blocked_awc needs to be prepared */ + if (awc.is_streamed) goto fail; /* blocked_awc needs to be prepared */ vgmstream->coding_type = awc.big_endian ? coding_PCM16BE : coding_PCM16LE; vgmstream->layout_type = layout_interleave; vgmstream->interleave_block_size = 0x02; @@ -63,7 +67,7 @@ VGMSTREAM* init_vgmstream_awc(STREAMFILE* sf) { case 0x04: /* IMA (PC) */ vgmstream->coding_type = coding_AWC_IMA; - vgmstream->layout_type = awc.is_music ? layout_blocked_awc : layout_none; + vgmstream->layout_type = awc.is_streamed ? layout_blocked_awc : layout_none; vgmstream->full_block_size = awc.block_chunk; vgmstream->codec_endian = awc.big_endian; break; @@ -72,7 +76,7 @@ VGMSTREAM* init_vgmstream_awc(STREAMFILE* sf) { case 0x05: { /* XMA2 (X360) */ uint32_t substream_size, substream_offset; - if (awc.is_music) { + if (awc.is_streamed) { /* 1ch XMAs in blocks, we'll use layered layout + custom IO to get multi-FFmpegs working */ int i; layered_layout_data * data = NULL; @@ -129,9 +133,8 @@ VGMSTREAM* init_vgmstream_awc(STREAMFILE* sf) { break; } - - #endif + #ifdef VGM_USE_MPEG case 0x07: { /* MPEG (PS3) */ mpeg_custom_config cfg = {0}; @@ -146,9 +149,10 @@ VGMSTREAM* init_vgmstream_awc(STREAMFILE* sf) { break; } #endif + #ifdef VGM_USE_VORBIS case 0x08: { /* Vorbis (PC) [Red Dead Redemption 2 (PC)] */ - if (awc.is_music) { + if (awc.is_streamed) { vgmstream->layout_data = build_layered_awc(sf, &awc); if (!vgmstream->layout_data) goto fail; vgmstream->layout_type = layout_layered; @@ -169,6 +173,62 @@ VGMSTREAM* init_vgmstream_awc(STREAMFILE* sf) { break; } #endif + +#ifdef VGM_USE_ATRAC9 + case 0x0F: { /* ATRAC9 (PC) [Red Dead Redemption (PS4)] */ + if (awc.is_streamed) { + vgmstream->layout_data = build_layered_awc(sf, &awc); + if (!vgmstream->layout_data) goto fail; + vgmstream->layout_type = layout_layered; + vgmstream->coding_type = coding_ATRAC9; + } + else { + VGMSTREAM* temp_vs = NULL; + STREAMFILE* temp_sf = NULL; + + temp_sf = setup_subfile_streamfile(sf, awc.stream_offset, awc.stream_size, "at9"); + if (!temp_sf) goto fail; + + temp_vs = init_vgmstream_riff(temp_sf); + close_streamfile(temp_sf); + if (!temp_vs) goto fail; + + temp_vs->num_streams = vgmstream->num_streams; + temp_vs->stream_size = vgmstream->stream_size; + temp_vs->meta_type = vgmstream->meta_type; + strcpy(temp_vs->stream_name, vgmstream->stream_name); + + close_vgmstream(vgmstream); + //vgmstream = temp_vs; + return temp_vs; + } + break; + } +#endif + + case 0x0C: /* DSP-sfx (Switch) */ + case 0x10: /* DSP-music (Switch) */ + vgmstream->coding_type = coding_NGC_DSP; + vgmstream->layout_type = awc.is_streamed ? layout_blocked_awc : layout_none; + vgmstream->full_block_size = awc.block_chunk; + + if (!awc.is_streamed) { + /* dsp header */ + dsp_read_coefs_le(vgmstream, sf, awc.stream_offset + 0x1c + 0x00, 0x00); + dsp_read_hist_le (vgmstream, sf, awc.stream_offset + 0x1c + 0x20, 0x00); + awc.stream_offset += 0x60; + + /* shouldn't be possible since it's only used for sfx anyway */ + if (awc.channels > 1) + goto fail; + } + break; + + case 0xFF: + vgmstream->coding_type = coding_SILENCE; + snprintf(vgmstream->stream_name, STREAM_NAME_SIZE, "[%s]", "midi"); + break; + default: VGM_LOG("AWC: unknown codec 0x%02x\n", awc.codec); goto fail; @@ -185,24 +245,42 @@ fail: } -/* Parse Rockstar's AWC header (much info from LibertyV: https://github.com/koolkdev/libertyv). - * Made of entries for N streams, each with a number of tags pointing to chunks (header, data, events, etc). */ +/* Parse Rockstar's AWC header (much info from LibertyV: https://github.com/koolkdev/libertyv). + * + * AWC defines logical streams/tracks, each with N tags (type+offset+size) that point to headers/tables with info. + * First stream may be a "music" type, then other streams are used as channels and not always define tags. + * When the "stream" flag is set data is divided into "blocks" (used for music), described later. + * Streams are ordered by hash/id and its tags go in order, but data may be unordered (1st stream audio + * or headers could go after others). Defined streams also may be unused/dummy. + * Hashes are actually reversable and more or less stream names (see other tools). + * + * Rough file format: + * - base header + * - stream tag starts [optional] + * - stream hash ids and tag counts (stream N has M tags) + * - tags per stream + * - data from tags (headers, tables, audio data, etc) + */ static int parse_awc_header(STREAMFILE* sf, awc_header* awc) { - uint64_t (*read_u64)(off_t,STREAMFILE*) = NULL; + uint64_t (*read_u64)(off_t,STREAMFILE*) = NULL; //TODO endian uint32_t (*read_u32)(off_t,STREAMFILE*) = NULL; uint16_t (*read_u16)(off_t,STREAMFILE*) = NULL; - int i, ch, entries; - uint32_t flags, info_header, tag_count = 0, tags_skip = 0; - off_t offset; + int entries; + uint32_t flags, tag_count = 0, tags_skip = 0; + uint32_t offset; int target_subsong = sf->stream_index; + /** base header **/ + if (is_id32be(0x00,sf,"ADAT")) { + awc->big_endian = false; + } + else if (is_id32be(0x00,sf,"TADA")) { + awc->big_endian = true; + } + else { + return false; + } - /* check header */ - if (read_u32be(0x00,sf) != 0x41444154 && /* "ADAT" (LE) */ - read_u32be(0x00,sf) != 0x54414441) /* "TADA" (BE) */ - goto fail; - - awc->big_endian = read_u32be(0x00,sf) == 0x54414441; if (awc->big_endian) { read_u64 = read_u64be; read_u32 = read_u32be; @@ -213,25 +291,32 @@ static int parse_awc_header(STREAMFILE* sf, awc_header* awc) { read_u16 = read_u16le; } - flags = read_u32(0x04,sf); entries = read_u32(0x08,sf); - //header_size = read_u32(0x0c,sf); /* after to stream id/tags, not including chunks */ + //header_size = read_u32(0x0c,sf); /* after stream id+tags */ offset = 0x10; + /* flags = 8b (always FF) + 8b (actual flags) + 16b (version, 00=rarely, 01=common) */ if ((flags & 0xFF00FFFF) != 0xFF000001 || (flags & 0x00F00000)) { VGM_LOG("AWC: unknown flags 0x%08x\n", flags); goto fail; } - if (flags & 0x00010000) /* some kind of mini offset table */ + /* stream tag starts (ex. stream#0 = 0, stream#1 = 4, stream#2 = 7: to read tags from stream#2 skip to 7th tag) */ + if (flags & 0x00010000) offset += 0x2 * entries; - //if (flags % 0x00020000) /* seems to indicate chunks are not ordered (ie. header may go after data) */ - // ... - //if (flags % 0x00040000) /* music/multichannel flag? (GTA5, not seen in RDR) */ - // awc->is_music = 1; - if (flags & 0x00080000) /* encrypted data chunk (most of GTA5 PC) */ + + /* seems to indicate chunks are not ordered (ie. header structures from tags may go after data), usually for non-streams */ + //if (flags % 0x00020000) + // awc->is_unordered = 1; + + /* stream/multichannel flag (GTA5 only) */ + //if (flags % 0x00040000) + // awc->is_multichannel = 1; + + /* encrypted data chunk (most of GTA5 PC for licensed audio) */ + if (flags & 0x00080000) awc->is_encrypted = 1; if (awc->is_encrypted) { @@ -239,48 +324,71 @@ static int parse_awc_header(STREAMFILE* sf, awc_header* awc) { goto fail; } - /* Music when the first id is 0 (base/fake entry with info for all channels), sfx pack otherwise. - * sfx = N single streams, music = N-1 interleaved mono channels (even for MP3/XMA). - * Music seems layered (N-1/2 stereo pairs), maybe set with events? */ - awc->is_music = (read_u32(offset + 0x00,sf) & 0x1FFFFFFF) == 0x00000000; - if (awc->is_music) { /* all streams except id 0 is a channel */ + + /* When first stream hash/id is 0 AWC it has fake entry with info for all channels = music, sfx pack otherwise. + * sfx = N single streams, music = N interleaved mono channels (even for MP3/XMA/Vorbis/etc). + * Channels set a stream hash/id that typically is one of the defined ones and its tags do apply to that + * channel, but rarely may not exist. Ex.: + * + * - bgm01.awc + * Stream ID 00000000 (implicit: music stream, all others aren't used) + * Tag: music header + * Channel 0: ID 9d66fe4c + * Channel 1: ID 7a3837ef + * Channel 2: ID 032c57e9 (not actually defined) + * Tag: data chunk + * #Tag: sfx header (only in buggy files) + * Stream ID 7a3837ef (no tags) + * Stream ID 9d66fe4c (notice this is channel 0 but streams are ordered by hash) + * Tag: Event config + * + * - sfx01.awc + * Stream ID 9d66fe4c + * Tag: sfx header + * Tag: data chunk + * Stream ID 7a3837ef + * Tag: sfx header + * Tag: data chunk + * + * Music 'stream' defines it's own (streamed/blocked) data chunk, so other stream's data or headers aren't used, + * but in rare cases they actually define a useless sfx header or even a separate cloned data chunk. That seems + * to be a bug and are ignored (ex. RDR's ftr_harmonica_01, or RDR SW's countdown_song_01). + */ + + awc->is_streamed = (read_u32(offset + 0x00,sf) & 0x1FFFFFFF) == 0x00000000; /* first stream's hash/id is 0 */ + if (awc->is_streamed) { /* music with N channels, other streams aren't used ignored */ awc->total_subsongs = 1; - target_subsong = 1; /* we only need id 0, though channels may have its own tags/chunks */ + target_subsong = 1; } - else { /* each stream is a single sound */ + else { /* sfx pack, each stream is a sound */ awc->total_subsongs = entries; if (target_subsong == 0) target_subsong = 1; if (target_subsong < 0 || target_subsong > awc->total_subsongs || awc->total_subsongs < 1) goto fail; } - /* get stream base info */ - for (i = 0; i < entries; i++) { - info_header = read_u32(offset + 0x04*i, sf); + /** stream ids and tag counts **/ + for (int i = 0; i < entries; i++) { + uint32_t info_header = read_u32(offset + 0x04*i, sf); tag_count = (info_header >> 29) & 0x7; /* 3b */ - //id = (info_header >> 0) & 0x1FFFFFFF; /* 29b */ - if (target_subsong-1 == i) + //hash_id = (info_header >> 0) & 0x1FFFFFFF; /* 29b */ + if (target_subsong - 1 == i) break; tags_skip += tag_count; /* tags to skip to reach target's tags, in the next header */ } - offset += 0x04*entries; - offset += 0x08*tags_skip; + offset += 0x04 * entries; + offset += 0x08 * tags_skip; - /* get stream tags */ - for (i = 0; i < tag_count; i++) { - uint64_t tag_header; - uint8_t tag_type; - size_t tag_size; - off_t tag_offset; - tag_header = read_u64(offset + 0x08*i,sf); - tag_type = (uint8_t)((tag_header >> 56) & 0xFF); /* 8b */ - tag_size = (size_t)((tag_header >> 28) & 0x0FFFFFFF); /* 28b */ - tag_offset = (off_t)((tag_header >> 0) & 0x0FFFFFFF); /* 28b */ - ;VGM_LOG("AWC: tag%i/%i at %lx: t=%x, o=%lx, s=%x\n", i, tag_count, offset + 0x08*i, tag_type, tag_offset, tag_size); + /** tags per stream **/ + for (int i = 0; i < tag_count; i++) { + uint64_t tag_header = read_u64(offset + 0x08*i,sf); + uint8_t tag_type = ((tag_header >> 56) & 0xFF); /* 8b */ + uint32_t tag_size = ((tag_header >> 28) & 0x0FFFFFFF); /* 28b */ + uint32_t tag_offset = ((tag_header >> 0) & 0x0FFFFFFF); /* 28b */ + //;VGM_LOG("AWC: tag %i/%i at %x: t=%x, o=%x, s=%x\n", i+1, tag_count, offset + 0x08*i, tag_type, tag_offset, tag_size); - /* Tags are apparently part of a hash derived from a word ("data", "format", etc). - * If music + 1ch, the header and data chunks can repeat for no reason (sometimes not even pointed). */ + /* types are apparently part of a hash derived from a word ("data", "format", etc). */ switch(tag_type) { case 0x55: /* data */ awc->stream_offset = tag_offset; @@ -288,24 +396,24 @@ static int parse_awc_header(STREAMFILE* sf, awc_header* awc) { break; case 0x48: /* music header */ - - if (!awc->is_music) { - VGM_LOG("AWC: music header found in sfx\n"); + if (!awc->is_streamed) { + VGM_LOG("AWC: music header found but not streamed\n"); goto fail; } - /* 0x00(32): unknown (some count?) */ + awc->block_count = read_u32(tag_offset + 0x00,sf); awc->block_chunk = read_u32(tag_offset + 0x04,sf); - awc->channels = read_u32(tag_offset + 0x08,sf); + awc->channels = read_u32(tag_offset + 0x08,sf); if (awc->channels != entries - 1) { /* not counting id-0 */ VGM_LOG("AWC: number of music channels doesn't match entries\n"); goto fail; } - for (ch = 0; ch < awc->channels; ch++) { + for (int ch = 0; ch < awc->channels; ch++) { int num_samples, sample_rate, codec; - /* 0x00): stream id (not always in the header entries order) */ + + /* 0x00: reference stream hash/id */ num_samples = read_u32(tag_offset + 0x0c + 0x10*ch + 0x04,sf); /* 0x08: headroom */ sample_rate = read_u16(tag_offset + 0x0c + 0x10*ch + 0x0a,sf); @@ -336,77 +444,86 @@ static int parse_awc_header(STREAMFILE* sf, awc_header* awc) { break; case 0xFA: /* sfx header */ - if (awc->is_music) { - VGM_LOG("AWC: sfx header found in music\n"); - goto fail; + if (awc->is_streamed) { + VGM_LOG("AWC: sfx header found but streamed\n"); + break; //goto fail; /* rare (RDR PC/Switch) */ } awc->num_samples = read_u32(tag_offset + 0x00,sf); /* 0x04: -1? */ awc->sample_rate = read_u16(tag_offset + 0x08,sf); - /* 0x0a: unknown x4 */ + /* 0x0a: headroom */ + /* 0x0c: unknown */ + /* 0x0e: unknown */ + /* 0x10: unknown */ /* 0x12: null? */ awc->codec = read_u8(tag_offset + 0x13, sf); + /* 0x14: ? (PS3 only, for any codec) */ + awc->channels = 1; break; case 0x76: /* sfx header for vorbis */ - if (awc->is_music) { - VGM_LOG("AWC: sfx header found in music\n"); + if (awc->is_streamed) { + VGM_LOG("AWC: sfx header found but streamed\n"); goto fail; } awc->num_samples = read_u32(tag_offset + 0x00,sf); /* 0x04: -1? */ awc->sample_rate = read_u16(tag_offset + 0x08,sf); - /* 0x0a: granule start? (negative) */ - /* 0x0c: granule max? */ + /* 0x0a: headroom */ + /* 0x0c: unknown */ + /* 0x0e: unknown */ /* 0x10: unknown */ awc->codec = read_u8(tag_offset + 0x1c, sf); /* 16b? */ - /* 0x1e: vorbis header size */ - awc->channels = 1; + /* 0x1e: vorbis setup size */ + awc->vorbis_offset[0] = tag_offset + 0x20; /* data up to vorbis setup size */ - awc->vorbis_offset[0] = tag_offset + 0x20; + awc->channels = 1; break; - case 0xA3: /* block-to-sample table (32b x number of blocks w/ num_samples at the start of each block) */ + case 0x68: /* midi data [Red Dead Redemption 2 (PC)] */ + /* set fake info so awc doesn't break */ + awc->stream_offset = tag_offset; + awc->stream_size = tag_size; + + awc->num_samples = 48000; + awc->sample_rate = 48000; + awc->codec = 0xFF; + awc->channels = 1; + break; + + case 0xA3: /* block-to-sample table (32b x number of blocks w/ num_samples at the start of each block) + * or frame-size table (16b x number of frames) in some cases (ex. sfx+mpeg but not sfx+vorbis) */ case 0xBD: /* events (32bx4): type_hash, params_hash, timestamp_ms, flags */ - case 0x5C: /* animation/RSC config? */ - default: /* 0x68=midi?, 0x36=hash thing?, 0x2B=sizes, 0x5A/0xD9=? */ + case 0x5C: /* animation/RSC info? */ + case 0x81: /* animation/CSR info? */ + case 0x36: /* list of hash-things? */ + case 0x2B: /* events/sizes? */ + case 0x7f: /* vorbis setup (for streams) */ + default: /* 0x68=midi?, 0x5A/0xD9=? */ //VGM_LOG("AWC: ignoring unknown tag 0x%02x\n", tag); break; } } + /* in music mode there tags for other streams we don't need, except for vorbis that have one setup packet */ + //TODO not correct (assumes 1 tag per stream and channel order doesn't match stream order) + // would need to read N tags and match channel id<>stream id, all vorbis setups are the same though) + if (awc->is_streamed && awc->codec == 0x08) { + offset += 0x08 * tag_count; + + for (int ch = 0; ch < awc->channels; ch++) { + awc->vorbis_offset[ch] = read_u16(offset + 0x08*ch + 0x00, sf); /* tag offset */ + } + } + if (!awc->stream_offset) { VGM_LOG("AWC: stream offset not found\n"); goto fail; } - /* vorbis offset table, somehow offsets are unordered and can go before tags */ - if (awc->is_music && awc->codec == 0x08) { - offset += 0x08 * tag_count; - - for (ch = 0; ch < awc->channels; ch++) { - awc->vorbis_offset[ch] = read_u16(offset + 0x08*ch + 0x00, sf); - /* 0x02: always 0xB000? */ - /* 0x04: always 0x00CD? */ - /* 0x06: always 0x7F00? */ - } - } - - - /* In music mode, data is divided into blocks of block_chunk size with padding. - * Each block has a header/seek table and interleaved data for all channels */ - { - int32_t seek_start = read_u32(awc->stream_offset, sf); /* -1 in later (RDR2) versions */ - if (awc->is_music && !(seek_start == 0 || seek_start == -1)) { - VGM_LOG("AWC: music found, but block doesn't start with seek table at %x\n", (uint32_t)awc->stream_offset); - goto fail; - } - } - - return 1; fail: return 0; @@ -414,111 +531,224 @@ fail: /* ************************************************************************* */ -//TODO: this method won't work properly, needs internal handling of blocks. -// -// This setups a decoder per block, but seems Vorbis' uses first frame as setup so it -// returns samples (576 vs 1024), making num_samples count in each block being off + causing -// gaps. So they must be using a single encoder + setting decode_to_discard per block -// to ge the thing working. -// -// However since blocks are probably also used for seeking, maybe they aren't resetting -// the decoder when seeking? or they force first frame to be 1024? -// -// In case of Vorvis, when setting skip samples seems repeated data from last block is -// exactly last 0x800 bytes of that channel. +typedef struct { + int start_entry; + int entries; + int32_t channel_skip; + int32_t channel_samples; -static VGMSTREAM* build_block_vgmstream(STREAMFILE* sf, awc_header* awc, int channel, int32_t num_samples, int32_t skip_samples, off_t block_start, size_t block_size) { - STREAMFILE* temp_sf = NULL; + uint32_t extradata; + + /* derived */ + uint32_t chunk_start; + uint32_t chunk_size; +} awc_block_t; + +typedef struct { + awc_block_t blk[AWC_MAX_MUSIC_CHANNELS]; +} awc_block_info_t; + +/* Block format: + * - block header for all channels (needed to find frame start) + * - frames from channel 1 + * - ... + * - frames from channel N + * - usually there is padding between channels or blocks (usually 0s but seen 0x97 in AT9) + * + * Header format: + * - per channel (frame start table) + * 0x00: start entry for that channel? (-1 in vorbis) + * may be off by +1/+2? + * ex. on block 0, ch0/1 have 0x007F frames, a start entry is: ch0=0x0000, ch1=0x007F (MP3) + * ex. on block 0, ch0/1 have 0x02A9 frames, a start entry is: ch0=0x0000, ch1=0x02AA (AT9) !! + * (sum of all values from all channels may go beyond all posible frames, no idea) + * 0x04: frames in this channel (may be different between channels) + * 'frames' here may be actual single decoder frames or a chunk of frames + * 0x08: samples to discard in the beginning of this block (MPEG/XMA2/Vorbis only?) + * 0x0c: samples in channel (for MPEG/XMA2 can vary between channels) + * full samples without removing samples to discard + * (next fields don't exist in later versions for IMA or AT9) + * 0x10: (MPEG only, empty otherwise) close to number of frames but varies a bit? + * 0x14: (MPEG only, empty otherwise) channel chunk size (not counting padding) + * - for each channel (seek table) + * 32b * entries = global samples per frame in each block (for MPEG probably per full frame) + * (AT9 doesn't have a seek table as it's CBR) + * - per channel (ATRAC9/DSP extra info): + * 0x00: "D11A" + * 0x04: frame size + * 0x06: frame samples + * 0x08: flags? (0x0103=AT9, 0x0104=DSP) + * 0x0a: sample rate + * 0x0c: ATRAC9 config (repeated but same for all blocks) or "D11E" (DSP) + * 0x10-0x70: padding with 0x77 (ATRAC3) or standard DSP header for original full file (DSP) + * - padding depending on codec (AT9/DSP: none, MPEG/XMA: closest 0x800) + */ +static bool read_awb_block(STREAMFILE* sf, awc_header* awc, awc_block_info_t* bi, uint32_t block_offset) { + uint32_t channel_entry_size, seek_entry_size, extra_entry_size, header_padding; + uint32_t offset = block_offset; + /* read stupid block crap + derived info at once so hopefully it's a bit easier to understand */ + + switch(awc->codec) { + case 0x08: /* Vorbis */ + channel_entry_size = 0x18; + seek_entry_size = 0x04; + extra_entry_size = 0x00; + header_padding = 0x800; + break; + case 0x0F: /* ATRAC9 */ + channel_entry_size = 0x10; + seek_entry_size = 0x00; + extra_entry_size = 0x70; + header_padding = 0x00; + break; + default: + goto fail; + } + + /* channel info table */ + for (int ch = 0; ch < awc->channels; ch++) { + bi->blk[ch].start_entry = read_u32le(offset + 0x00, sf); + bi->blk[ch].entries = read_u32le(offset + 0x04, sf); + bi->blk[ch].channel_skip = read_u32le(offset + 0x08, sf); + bi->blk[ch].channel_samples = read_u32le(offset + 0x0c, sf); + /* others: optional */ + + offset += channel_entry_size; + } + + /* seek table */ + for (int ch = 0; ch < awc->channels; ch++) { + offset += bi->blk[ch].entries * seek_entry_size; + } + + /* extra table and derived info */ + for (int ch = 0; ch < awc->channels; ch++) { + switch(awc->codec) { + case 0x08: /* Vorbis */ + /* each "frame" here is actually N vorbis frames then padding up to 0x800 (more or less like a big Ogg page) */ + bi->blk[ch].chunk_size = bi->blk[ch].entries * 0x800; + break; + + case 0x0F: { /* ATRAC9 */ + uint16_t frame_size = read_u16le(offset + 0x04, sf); + + bi->blk[ch].chunk_size = bi->blk[ch].entries * frame_size; + bi->blk[ch].extradata = read_u32be(offset + 0x0c, sf); + break; + } + default: + goto fail; + } + offset += extra_entry_size; + } + + /* header done, move into data start */ + if (header_padding) { + /* padding on the current size rather than file offset (block meant to be read into memory, probably) */ + uint32_t header_size = offset - block_offset; + offset = block_offset + align_size_to_block(header_size, header_padding); + } + + /* set frame starts per channel */ + for (int ch = 0; ch < awc->channels; ch++) { + bi->blk[ch].chunk_start = offset; + offset += bi->blk[ch].chunk_size; + } + + /* beyond this is padding until awc.block_chunk */ + + return true; +fail: + return false; +} + +/* ************************************************************************* */ + +static VGMSTREAM* build_block_vgmstream(STREAMFILE* sf, awc_header* awc, int channel, awc_block_info_t* bi) { VGMSTREAM* vgmstream = NULL; + awc_block_t* blk = &bi->blk[channel]; int block_channels = 1; + //;VGM_LOG("AWC: build ch%i at o=%x, s=%x\n", channel, blk->chunk_start, blk->chunk_size); /* build the VGMSTREAM */ vgmstream = allocate_vgmstream(block_channels, 0); if (!vgmstream) goto fail; vgmstream->sample_rate = awc->sample_rate; - vgmstream->num_samples = num_samples - skip_samples; - vgmstream->stream_size = block_size; + vgmstream->num_samples = blk->channel_samples - blk->channel_skip; + vgmstream->stream_size = blk->chunk_size; vgmstream->meta_type = meta_AWC; switch(awc->codec) { #ifdef VGM_USE_VORBIS - case 0x08: { /* Vorbis (PC) [Red Dead Redemption 2 (PC)] */ + case 0x08: { vorbis_custom_config cfg = {0}; cfg.channels = 1; cfg.sample_rate = awc->sample_rate; - cfg.header_offset = awc->vorbis_offset[channel]; + cfg.header_offset = awc->vorbis_offset[channel]; /* setup page goes first */ //cfg.skip_samples = skip_samples; //todo - - vgmstream->codec_data = init_vorbis_custom(sf, block_start, VORBIS_AWC, &cfg); + + vgmstream->codec_data = init_vorbis_custom(sf, blk->chunk_start, VORBIS_AWC, &cfg); if (!vgmstream->codec_data) goto fail; vgmstream->layout_type = layout_none; vgmstream->coding_type = coding_VORBIS_custom; + break; + } +#endif +#ifdef VGM_USE_ATRAC9 + case 0x0F: { + atrac9_config cfg = {0}; + + cfg.channels = block_channels; + cfg.encoder_delay = blk->channel_skip; + cfg.config_data = blk->extradata; + ;VGM_ASSERT(blk->channel_skip, "AWC discard found\n"); + + vgmstream->codec_data = init_atrac9(&cfg); + if (!vgmstream->codec_data) goto fail; + vgmstream->coding_type = coding_ATRAC9; + vgmstream->layout_type = layout_none; + + break; } - break; #endif default: goto fail; } - if (!vgmstream_open_stream(vgmstream, sf, block_start)) + if (!vgmstream_open_stream(vgmstream, sf, blk->chunk_start)) goto fail; - close_streamfile(temp_sf); return vgmstream; fail: - close_streamfile(temp_sf); + ;VGM_LOG("AWB: can't open decoder\n"); close_vgmstream(vgmstream); return NULL; } +/* per channel to possibly simplify block entry skips, though can't be handled right now */ static VGMSTREAM* build_blocks_vgmstream(STREAMFILE* sf, awc_header* awc, int channel) { VGMSTREAM* vgmstream = NULL; segmented_layout_data* data = NULL; - int i, ch; - int blocks = awc->stream_size / awc->block_chunk + (awc->stream_size % awc->block_chunk ? 1 : 0) ; + int blocks = awc->block_count; + awc_block_info_t bi = {0}; /* init layout */ data = init_layout_segmented(blocks); if (!data) goto fail; + /* one segment per block of this channel */ - for (i = 0; i < blocks; i++) { - off_t block_offset = awc->stream_offset + i * awc->block_chunk; - int32_t num_samples = 0, skip_samples = 0; - uint32_t header_skip = 0, block_skip = 0, block_start = 0, block_data = 0; + for (int i = 0; i < blocks; i++) { + uint32_t block_offset = awc->stream_offset + awc->block_chunk * i; - /* read stupid block crap to get proper offsets and whatnot, format: - * - per channel: number of channel entries + skip samples + num samples - * - per channel: seek table with N entries */ - for (ch = 0; ch < awc->channels; ch++) { - /* 0x00: -1 */ - int entries = read_u32le(block_offset + 0x18 * ch + 0x04, sf); - int32_t entry_skip = read_u32le(block_offset + 0x18 * ch + 0x08, sf); - int32_t entry_samples = read_u32le(block_offset + 0x18 * ch + 0x0c, sf); - - if (ch == channel) { - num_samples = entry_samples; - skip_samples = entry_skip; - - block_start = block_offset + block_skip; - block_data = entries * 0x800; - } - - header_skip += 0x18 + entries * 0x04; - block_skip += entries * 0x800; - } - - if (!block_start) + if (!read_awb_block(sf, awc, &bi, block_offset)) goto fail; - header_skip = align_size_to_block(header_skip, 0x800); - block_start += header_skip; - //;VGM_LOG("AWC: build ch%i, block=%i at %lx, o=%x, s=%x, ns=%i, ss=%i\n", channel, i, block_offset, block_start, block_data, num_samples, skip_samples); - - data->segments[i] = build_block_vgmstream(sf, awc, channel, num_samples, skip_samples, block_start, block_data); + //;VGM_LOG("AWC: block=%i at %x\n", i, block_offset); + data->segments[i] = build_block_vgmstream(sf, awc, channel, &bi); if (!data->segments[i]) goto fail; } @@ -540,14 +770,23 @@ fail: /* ************************************************************************* */ -/* Make layers per channel for AWC's abhorrent blocks. +/* Make layers per channel for AWC's abhorrent blocks (see read_awb_block). * - * File has N channels = N streams, that use their own mono decoder. - * Each block then has header + seek table for all channels. But in each block there is - * a "skip samples" value per channel, and blocks repeat some data from last block - * for this, so PCM must be discarded. Also, channels in a block don't need to have - * the same number of samples. + * A "music" .awc has N channels = N streams (each using their own mono decoder) chunked in "blocks". + * Each block then has header + seek table + etc for all channels. But when blocks change, each channel + * may have a "skip samples" value and blocks repeat some data from last block, so output PCM must be + * discarded to avoid channels desyncing. Channels in a block don't need to have the same number of samples. + * (mainly seen in MPEG). */ +//TODO: this method won't fully work, needs feed decoder + block handler that interacts with decoder(s?) +// (doesn't use multiple decoders since default encoder delay in Vorbis would discard too much per block) +// +// When blocks change presumably block handler needs to tell decoder to finish decoding all from prev block +// then skip samples from next decodes. Also since samples may vary per channel, each would handle blocks +// independently. +// +// This can be simulated by making one decoder per block (segmented, but opens too many SFs and can't skip +// samples correctly), or with a custom STREAMFILE that skips repeated block (works ok-ish but not all codecs). static layered_layout_data* build_layered_awc(STREAMFILE* sf, awc_header* awc) { int i; layered_layout_data* data = NULL; From fa89dcc8a056b2f504bcf1f14be51ec77c282585 Mon Sep 17 00:00:00 2001 From: bnnm Date: Sun, 26 Nov 2023 20:57:56 +0100 Subject: [PATCH 2/4] Fix some .str+wav [Taz: Wanted Beta (PC)] --- src/meta/str_wav.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/meta/str_wav.c b/src/meta/str_wav.c index 1e1399e6..c020b092 100644 --- a/src/meta/str_wav.c +++ b/src/meta/str_wav.c @@ -543,6 +543,32 @@ static int parse_header(STREAMFILE* sf_h, STREAMFILE* sf_b, strwav_header* strwa return 1; } + /* Taz Wanted (beta) (PC)[2002] */ + if ( read_u32be(0x04,sf_h) == 0x00000900 && + read_u32le(0x0c,sf_h) != header_size && + read_u32le(0x24,sf_h) != 0 && + read_u32le(0xd4,sf_h) != 0 && + read_u32le(0xdc,sf_h) == header_size + ) { + /* 0x08: null */ + /* 0x0c: hashname */ + strwav->num_samples = read_s32le(0x20,sf_h); + strwav->sample_rate = read_s32le(0x24,sf_h); + /* 0x28: 16 bps */ + strwav->flags = read_u32le(0x2c,sf_h); + strwav->loop_start = read_s32le(0x38,sf_h); + /* 0x54: number of chunks */ + strwav->tracks = read_s32le(0xd4,sf_h); + /* 0xdc: header size */ + + strwav->loop_end = strwav->num_samples; + + strwav->codec = IMA; + strwav->interleave = strwav->tracks > 1 ? 0x8000 : 0x10000; + ;VGM_LOG("STR+WAV: header TAZb (PC)\n"); + return 1; + } + /* Taz Wanted (PC)[2002] */ /* Zapper: One Wicked Cricket! Beta (Xbox)[2002] */ if ( read_u32be(0x04,sf_h) == 0x00000900 && From e8643c333cae13331d7f0fe230bd46b5f9ae241a Mon Sep 17 00:00:00 2001 From: bnnm Date: Sun, 26 Nov 2023 20:59:09 +0100 Subject: [PATCH 3/4] bitreaders: improve performance a bit for EALayer3 --- src/util/bitstream_lsb.h | 9 +++++---- src/util/bitstream_msb.h | 35 ++++++++++++++++++++++------------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/util/bitstream_lsb.h b/src/util/bitstream_lsb.h index 275cfba1..c6404a5f 100644 --- a/src/util/bitstream_lsb.h +++ b/src/util/bitstream_lsb.h @@ -1,10 +1,11 @@ #ifndef _BITSTREAM_LSB_H #define _BITSTREAM_LSB_H -#include "../streamtypes.h" +#include /* Simple bitreader for Vorbis' bit style, in 'least significant byte' (LSB) format. - * Example: 0x12345678 is read as 12,34,56,78 (continuous). + * Example: with 0x1234 = 00010010 00110100, reading 5b + 6b = 10010 100000 + * (first lower 5b, then next upper 3b and next lower 3b = 6b) * Kept in .h since it's slightly faster (compiler can optimize statics better using default compile flags). */ @@ -23,7 +24,7 @@ static inline void bl_setup(bitstream_t* b, uint8_t* buf, size_t bufsize) { /* same as (1 << bits) - 1, but that seems to trigger some nasty UB when bits = 32 * (though in theory (1 << 32) = 0, 0 - 1 = UINT_MAX, but gives 0 compiling in some cases, but not always) */ -static const uint32_t MASK_TABLE[33] = { +static const uint32_t MASK_TABLE_LSB[33] = { 0x00000000, 0x00000001, 0x00000003, 0x00000007, 0x0000000f, 0x0000001f, 0x0000003f, 0x0000007f, 0x000000ff, 0x000001ff, 0x000003ff, 0x000007ff, 0x00000fff, 0x00001fff, 0x00003fff, 0x00007fff, 0x0000ffff, 0x0001ffff, 0x0003ffff, 0x0007ffff, 0x000fffff, 0x001fffff, 0x003fffff, 0x007fffff, 0x00ffffff, 0x01ffffff, 0x03ffffff, @@ -40,7 +41,7 @@ static inline int bl_get(bitstream_t* ib, uint32_t bits, uint32_t* value) { pos = ib->b_off / 8; /* byte offset */ shift = ib->b_off % 8; /* bit sub-offset */ - mask = MASK_TABLE[bits]; /* to remove upper in highest byte */ + mask = MASK_TABLE_LSB[bits]; /* to remove upper in highest byte */ val = ib->buf[pos+0] >> shift; if (bits + shift > 8) { diff --git a/src/util/bitstream_msb.h b/src/util/bitstream_msb.h index 42bd9b29..e93166b9 100644 --- a/src/util/bitstream_msb.h +++ b/src/util/bitstream_msb.h @@ -1,10 +1,11 @@ #ifndef _BITSTREAM_MSB_H #define _BITSTREAM_MSB_H -#include "../streamtypes.h" +#include /* Simple bitreader for MPEG/standard bit style, in 'most significant byte' (MSB) format. - * Example: 0x12345678 is read as 78,56,34,12 then each byte's bits. + * Example: with 0x1234 = 00010010 00110100, reading 5b + 6b = 00010 010001 + * (first upper 5b, then next lower 3b and next upper 3b = 6b) * Kept in .h since it's slightly faster (compiler can optimize statics better using default compile flags). */ typedef struct { @@ -14,7 +15,7 @@ typedef struct { uint32_t b_off; /* current offset in bits inside buffer */ } bitstream_t; - +/* convenience util */ static inline void bm_setup(bitstream_t* bs, uint8_t* buf, size_t bufsize) { bs->buf = buf; bs->bufsize = bufsize; @@ -60,10 +61,20 @@ static inline int bm_pos(bitstream_t* bs) { return bs->b_off; } +/* same as (1 << bits) - 1, but that seems to trigger some nasty UB when bits = 32 + * (though in theory (1 << 32) = 0, 0 - 1 = UINT_MAX, but gives 0 compiling in some cases, but not always) */ +static const uint32_t MASK_TABLE_MSB[33] = { + 0x00000000, 0x00000001, 0x00000003, 0x00000007, 0x0000000f, 0x0000001f, 0x0000003f, 0x0000007f, 0x000000ff, + 0x000001ff, 0x000003ff, 0x000007ff, 0x00000fff, 0x00001fff, 0x00003fff, 0x00007fff, 0x0000ffff, 0x0001ffff, + 0x0003ffff, 0x0007ffff, 0x000fffff, 0x001fffff, 0x003fffff, 0x007fffff, 0x00ffffff, 0x01ffffff, 0x03ffffff, + 0x07ffffff, 0x0fffffff, 0x1fffffff, 0x3fffffff, 0x7fffffff, 0xffffffff +}; + /* Read bits (max 32) from buf and update the bit offset. Order is BE (MSB). */ static inline int bm_get(bitstream_t* ib, uint32_t bits, uint32_t* value) { - uint32_t shift, pos, val; - int i, bit_buf, bit_val; + uint32_t shift, pos, mask; + uint64_t val; //TODO: could use u32 with some shift fiddling + int i, bit_buf, bit_val, left; if (bits > 32 || ib->b_off + bits > ib->b_max) goto fail; @@ -71,7 +82,7 @@ static inline int bm_get(bitstream_t* ib, uint32_t bits, uint32_t* value) { pos = ib->b_off / 8; /* byte offset */ shift = ib->b_off % 8; /* bit sub-offset */ -#if 1 //naive approach +#if 0 //naive approach val = 0; for (i = 0; i < bits; i++) { bit_buf = (1U << (8-1-shift)) & 0xFF; /* bit check for buf */ @@ -86,12 +97,10 @@ static inline int bm_get(bitstream_t* ib, uint32_t bits, uint32_t* value) { pos++; } } -#else //has bugs - pos = ib->b_off / 8; /* byte offset */ - shift = ib->b_off % 8; /* bit sub-offset */ - uint32_t mask = MASK_TABLE[bits]; /* to remove upper in highest byte */ +#else + mask = MASK_TABLE_MSB[bits]; /* to remove upper in highest byte */ - int left = 0; + left = 0; if (bits == 0) val = 0; else @@ -102,12 +111,12 @@ static inline int bm_get(bitstream_t* ib, uint32_t bits, uint32_t* value) { left = 16 - (bits + shift); if (bits + shift > 16) { val = (val << 8u) | ib->buf[pos+2]; - left = 32 - (bits + shift); + left = 24 - (bits + shift); if (bits + shift > 24) { val = (val << 8u) | ib->buf[pos+3]; left = 32 - (bits + shift); if (bits + shift > 32) { - val = (val << 8u) | ib->buf[pos+4]; /* upper bits are lost (shifting over 32) */ TO-DO + val = (val << 8u) | ib->buf[pos+4]; left = 40 - (bits + shift); } } From 7c3b95e8b573a86954a7af69cd271d0ca7b02bee Mon Sep 17 00:00:00 2001 From: bnnm Date: Sun, 26 Nov 2023 20:59:22 +0100 Subject: [PATCH 4/4] vrts: fix performance testing --- cli/tools/vrts.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cli/tools/vrts.py b/cli/tools/vrts.py index b280eebc..adcc4d91 100644 --- a/cli/tools/vrts.py +++ b/cli/tools/vrts.py @@ -409,7 +409,7 @@ class VrtsFiles: # same file N times if self._args.performance and self._args.performance_repeat: - for i in range(self._args.performance_repeat): + for _ in range(self._args.performance_repeat): self.filenames.append(file) @@ -541,7 +541,7 @@ class VrtsApp: flag_looping = '-i' # pases all files at once, as it's faster than 1 by 1 (that has to init program every time) - if self._performance_new: + if self._args.performance_new: self._p.info("testing new performance") ts_st = time.time() @@ -551,7 +551,7 @@ class VrtsApp: ts_ed = time.time() self._p.info("done: elapsed %ss" % (ts_ed - ts_st)) - if self._performance_old: + if self._args.performance_old: self._p.info("testing old performance") ts_st = time.time() @@ -559,10 +559,10 @@ class VrtsApp: res = self._prc.call(args) ts_ed = time.time() - self._p.info("done: elapsed %ss (%s)" % (ts_ed - ts_st)) + self._p.info("done: elapsed %ss" % (ts_ed - ts_st)) #if self._performance_both: - # ... + # handled above # returns max fuzzy count, except for non-fuzzable files (that use int math)