Merge pull request #864 from bnnm/isb-etc

isb etc

README.md

@@ -171,7 +171,7 @@ those files automatically into the playlist. For others without support, you can
 multiple .txtp (explained below) to select one of the subsongs (like `bgm.sxd#10.txtp`).
 
 You can use this python script to autogenerate one `.txtp` per subsong:
-https://github.com/vgmstream/vgmstream/tree/master/cli/txtp_maker.py
+https://github.com/vgmstream/vgmstream/tree/master/cli/tools/txtp_maker.py
 Put in the same dir as test.exe/vgmstream_cli, then to drag-and-drop files with
 subsongs to `txtp_maker.py` (it has CLI options to control output too).
 
@@ -620,6 +620,11 @@ Support for this feature is limited by player itself, as foobar and Winamp allow
 non-existant files referenced in a `.m3u`, while other players may filter them
 first.
 
+You can use this python script to autogenerate one `.txtp` per virtual-txtp:
+https://github.com/vgmstream/vgmstream/tree/master/cli/tools/txtp_dumper.py
+Drag and drop the `.m3u`, or any text file with .txtp  (it has CLI options
+to control output too).
+
 
 ## Supported codec types
 Quick list of codecs vgmstream supports, including many obscure ones that

cli/tools/txtm_maker.py

@@ -15,6 +15,13 @@ class Cli(object):
         )
         epilog = (
             "examples:\n"
+            "  %(prog)s *.fsb \n"
+            "  - make .txtp for subsongs in current dir\n\n"
+            "  %(prog)s bgm/*.fsb \n"
+            "  - make .txtp for subsongs in bgm subdir\n\n"
+            "  %(prog)s * -r -fss 1\n"
+            "  - make .txtp for all files in any subdirs with at least 1 subsong\n"
+            "    (ignores formats without subsongs)\n\n"
             "  %(prog)s bgm.fsb -in -fcm 2 -fms 5.0\n"
             "  - make .txtp for subsongs with at least 2 channels and 5 seconds\n\n"
             "  %(prog)s *.scd -r -fd -l 2\n"
@@ -23,9 +30,6 @@ class Cli(object):
             "  - make .txtp for all .sm1 excluding subsongs ending with 'STREAM.SS0..9'\n\n"
             "  %(prog)s samples.bnk -fni ^bgm.?\n"
             "  - make .txtp for in .bnk including only subsong names that start with 'bgm'\n\n"
-            "  %(prog)s * -r -fss 1\n"
-            "  - make .txtp for all files in subdirs with at least 1 subsong\n"
-            "    (ignores formats without subsongs)\n\n"
             "  %(prog)s *.fsb -n \"{fn}<__{ss}>< [{in}]>\" -z 4 -o\n"
             "  - make .txtp for all fsb, adding subsongs and stream name if they exist\n\n"
         )
@@ -45,8 +49,9 @@ class Cli(object):
         p.add_argument('-ie', dest='no_internal_ext', help="Remove internal name's extension if any", action='store_true')
         p.add_argument('-m',  dest='mini_txtp', help="Create mini-txtp", action='store_true')
         p.add_argument('-o',  dest='overwrite', help="Overwrite existing .txtp\n(beware when using with internal names alone)", action='store_true')
-        p.add_argument('-O',  dest='overwrite_rename', help="Rename rather than overwriting", action='store_true')
+        p.add_argument('-oi', dest='overwrite_ignore', help="Ignore repeated rather than overwritting.txtp\n", action='store_true')
-        p.add_argument('-Os', dest='overwrite_suffix', help="Rename with a suffix")
+        p.add_argument('-or', dest='overwrite_rename', help="Rename rather than overwriting", action='store_true')
+        p.add_argument('-os', dest='overwrite_suffix', help="Rename with a suffix")
         p.add_argument('-l',  dest='layers', help="Create .txtp per subsong layers, every N channels", type=int)
         p.add_argument('-fd', dest='test_dupes', help="Skip .txtp that point to duplicate streams (slower)", action='store_true')
         p.add_argument('-fcm', dest='min_channels', help="Filter by min channels", type=int)
@@ -150,8 +155,9 @@ class TxtpMaker(object):
         self.stream_name = self._get_text("stream name: ")
         self.encoding = self._get_text("encoding: ")
 
+        # in case vgmstream returns error, but output code wasn't EXIT_FAILURE
         if self.channels <= 0 or self.sample_rate <= 0:
-            raise ValueError('Incorrect command result')
+            raise ValueError('Incorrect vgmstream command')
 
         self.stream_seconds = self.num_samples / self.sample_rate
         self.ignorable = self._is_ignorable(cfg)
@@ -255,7 +261,8 @@ class TxtpMaker(object):
         outname += '.txtp'
 
         cfg = self.cfg
-        if (cfg.overwrite_rename or cfg.overwrite_suffix) and os.path.exists(outname):
+        exists = os.path.exists(outname)
+        if exists and (cfg.overwrite_rename or cfg.overwrite_suffix):
             must_rename = True
             if cfg.overwrite_suffix:
                 outname = outname.replace(".txtp", "%s.txtp" % (cfg.overwrite_suffix))
@@ -269,8 +276,14 @@ class TxtpMaker(object):
                 self.rename_map[outname] = rename_count + 1
                 outname = outname.replace(".txtp", "_%08i.txtp" % (rename_count))
 
-        if not cfg.overwrite and os.path.exists(outname):
+        # decide action after (possible) renames above
-            raise ValueError('TXTP exists in path: ' + outname)
+        if os.path.exists(outname):
+            if cfg.overwrite_ignore:
+                log.debug("ignored: " + outname)
+                return
+
+            if not cfg.overwrite:
+                raise ValueError('TXTP exists in path: ' + outname)
 
         ftxtp = open(outname,"w+")
         if line:
@@ -436,7 +449,14 @@ class App(object):
         if os.path.isdir(pattern):
             dir = pattern
             pattern = None
-    
+
+        for dirsep in '/\\':
+            if dirsep in pattern:
+                index = pattern.rfind(dirsep)
+                if index >= 0:
+                    dir = pattern[0:index]
+                    pattern = pattern[index+1:]
+
         files = []
         for root, dirnames, filenames in os.walk(dir):
             for filename in fnmatch.filter(filenames, pattern):
@@ -456,7 +476,6 @@ class App(object):
         for filename in self.cfg.files:
             filenames_in += self._find_files('.', filename)
 
-
         rename_map = {}
 
         total_created = 0
@@ -480,19 +499,22 @@ class App(object):
             target_subsong = 1
             while True:
                 try:
+                    # main call to vgmstream
                     cmd = self._make_cmd(filename_in, filename_out, target_subsong)
                     log.debug("calling: %s", cmd)
                     output_b = subprocess.check_output(cmd, shell=False) #stderr=subprocess.STDOUT
-                except subprocess.CalledProcessError as e:
+
-                    log.debug("ignoring CLI error in %s #%s: %s", filename_in, target_subsong, str(e.output))
+                    # basic parse of vgmstream info
+                    maker = TxtpMaker(self.cfg, output_b, rename_map)
+
+                except (subprocess.CalledProcessError, ValueError) as e:
+                    log.debug("ignoring CLI error in %s #%s: %s", filename_in, target_subsong, str(e))
                     errors += 1
                     break
 
                 if target_subsong == 1:
                     log.debug("processing %s...", filename_in_clean)
 
-                maker = TxtpMaker(self.cfg, output_b, rename_map)
-
                 if not maker.is_ignorable():
                     self.crc32.update(filename_out)
 

cli/tools/txtp_dumper.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+import os, glob, argparse
+
+#******************************************************************************
+# TXTP DUMPER
+#
+# Creates .txtp from a text list, mainly .m3u with virtual txtp to actual .txtp
+# example:
+# 
+# # %TITLE Stage 1
+# bgm.awb#1 .txtp
+# >> creates bgm.awb#1 .txtp
+#
+#******************************************************************************
+
+class Cli(object):
+    def _parse(self):
+        description = (
+            "Makes TXTP from list"
+        )
+        epilog = (
+            "examples:\n"
+            "  %(prog)s !tags.m3u\n"
+            "  - make .txtp per line in !tags.m3u\n\n"
+            "  %(prog)s !tags.m3u -m \n"
+            "  - make full txtp rather than mini-txtp\n    (may overwrite files when using subsongs)\n\n"
+            "  %(prog)s lines.txt -s sound \n"
+            "  - make .txtp per line, setting a subdir\n\n"
+        )
+
+        p = argparse.ArgumentParser(description=description, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter)
+        p.add_argument('files', help="Files to get (wildcards work)", nargs='+')
+        p.add_argument('-s',  dest='subdir', help="Create .txtp pointing to subdir")
+        p.add_argument('-m',  dest='maxitxtp', help="Create regular txtp rather than mini-txtp", action='store_true')
+        p.add_argument('-f',  dest='force', help="Make .txtp even if line isn't a .txtp", action='store_true')
+        p.add_argument('-o',  dest='output', help="Output dir (default: current)")
+        #p.add_argument('-n',  dest='normalize', help="Normalize .txtp formatting")
+        return p.parse_args()
+
+    def start(self):
+        args = self._parse()
+        if not args.files:
+            return
+        App(args).start()
+
+#******************************************************************************
+
+class App(object):
+    def __init__(self, args):
+        self.args = args
+
+    def start(self):
+        filenames = []
+        for filename in self.args.files:
+            filenames += glob.glob(filename)
+
+        for filename in filenames:
+            path = self.args.output or '.'
+            if self.args.output:
+                try:
+                    os.makedirs(self.args.output)
+                except OSError:
+                    pass
+            
+            with open(filename) as fi:
+                for line in fi:
+                    line = line.strip()
+                    line = line.rstrip()
+                    if line.startswith('#'):
+                        continue
+
+                    if not line.endswith('.txtp'):
+                        if self.args.force:
+                            line += '.txtp'
+                        else:
+                            continue
+
+                    line.replace('\\', '/')
+
+                    subdir = self.args.subdir
+                    if self.args.maxitxtp or subdir:
+                        index = line.find('.') #first extension
+
+                        if line[index:].startswith('.txtp'): #???
+                            continue
+
+                        name = line[0:index] + '.txtp'
+
+                        text = line.replace('.txtp', '').strip()
+                        if subdir:
+                            subdir.replace('\\', '/')
+                            if not subdir.endswith('/'):
+                                subdir = subdir + '/'
+                            text = subdir + text
+                    else:
+                        name = line
+                        text = ''
+
+                    outpath = os.path.join(path, name)
+
+                    with open(outpath, 'w') as fo:
+                        if text:
+                            fo.write(text)
+                        pass
+
+if __name__ == "__main__":
+    Cli().start()

cli/vgmstream_cli.c

@@ -557,7 +557,7 @@ static int convert_file(cli_config* cfg);
 
 int main(int argc, char** argv) {
     cli_config cfg = {0};
-    int i, res;
+    int i, res, ok;
 
 
     /* read args */
@@ -574,15 +574,20 @@ int main(int argc, char** argv) {
     res = validate_config(&cfg);
     if (!res) goto fail;
 
+    ok = 0;
     for (i = 0; i < cfg.infilenames_count; i++) {
         /* current name, to avoid passing params all the time */
         cfg.infilename = cfg.infilenames[i];
-        cfg.outfilename = NULL;
+        if (cfg.outfilename_config)
+            cfg.outfilename = NULL;
 
-        convert_file(&cfg);
+        res = convert_file(&cfg);
         //if (!res) goto fail; /* keep on truckin' */
+        if (res) ok = 1; /* return ok if at least one succeeds, for programs that check result code */
     }
 
+    if (!ok)
+        goto fail;
 
     return EXIT_SUCCESS;
 fail:
@@ -713,7 +718,7 @@ static int convert_file(cli_config* cfg) {
                 fclose(outfile);
         }
         close_vgmstream(vgmstream);
-        return EXIT_SUCCESS;
+        return 1;
     }
 
     if (cfg->seek_samples1 < -1) /* ex value for loop testing */

src/meta/isb.c

@@ -7,7 +7,7 @@ VGMSTREAM * init_vgmstream_isb(STREAMFILE *sf) {
     VGMSTREAM * vgmstream = NULL;
     off_t start_offset = 0, name_offset = 0;
     size_t stream_size = 0, name_size = 0;
-    int loop_flag = 0, channel_count = 0, sample_rate = 0, codec = 0, pcm_bytes = 0, bps = 0;
+    int loop_flag = 0, channels = 0, sample_rate = 0, codec = 0, pcm_bytes = 0, bps = 0;
     int total_subsongs, target_subsong = sf->stream_index;
     uint32_t (*read_u32me)(off_t,STREAMFILE*);
     uint32_t (*read_u32ce)(off_t,STREAMFILE*);
@@ -18,15 +18,15 @@ VGMSTREAM * init_vgmstream_isb(STREAMFILE *sf) {
     if (!check_extensions(sf, "isb"))
         goto fail;
 
-    big_endian = read_u32be(0x00,sf) == 0x46464952; /* "FFIR" */
+    big_endian = read_u32be(0x00,sf) == get_id32be("FFIR"); /* PS3, most X360 */
     read_u32me = big_endian ? read_u32be : read_u32le; /* machine endianness... */
     read_u32ce = big_endian ? read_u32le : read_u32be; /* chunks change with endianness but this just reads as BE */
 
-    if (read_u32ce(0x00,sf) != 0x52494646) /* "RIFF" */
+    if (read_u32ce(0x00,sf) != get_id32be("RIFF"))
         goto fail;
     if (read_u32me(0x04,sf) + 0x08 != get_streamfile_size(sf))
         goto fail;
-    if (read_u32ce(0x08,sf) != 0x69736266) /* "isbf" */
+    if (read_u32ce(0x08,sf) != get_id32be("isbf"))
         goto fail;
 
     /* some files have a companion .icb, seems to be a cue file pointing here */
@@ -36,7 +36,7 @@ VGMSTREAM * init_vgmstream_isb(STREAMFILE *sf) {
      * seems to use the format as a simple audio bank. Psychonauts Xbox/PS2 doesn't use ISACT. */
 
     {
-        off_t offset, max_offset, header_offset = 0;
+        off_t offset, max_offset, header_offset = 0, suboffset, submax_offset;
         size_t header_size = 0;
 
         total_subsongs = 0; /* not specified */
@@ -50,16 +50,51 @@ VGMSTREAM * init_vgmstream_isb(STREAMFILE *sf) {
             uint32_t chunk_size = read_u32me(offset + 0x04,sf);
             offset += 0x08;
 
+            //VGM_LOG("offset=%lx, sub=%c%c%c%c\n", offset,(chunk_type>>24 & 0xFF), (chunk_type>>16 & 0xFF), (chunk_type>>8 & 0xFF), (chunk_type & 0xFF));
+
+
             switch(chunk_type) {
                 case 0x4C495354: /* "LIST" */
-                    if (read_u32ce(offset, sf) != 0x73616D70) /* "samp" */
+                    if (read_u32ce(offset, sf) == get_id32be("samp")) {
-                        break; /* there are "bfob" LIST without data */
+                        /* sample header */
-
+                        total_subsongs++;
-                    total_subsongs++;
+                        if (target_subsong == total_subsongs && header_offset == 0) {
-                    if (target_subsong == total_subsongs && header_offset == 0) {
+                            header_offset = offset;
-                        header_offset = offset;
+                            header_size = chunk_size;
-                        header_size = chunk_size;
+                        }
                     }
+                    else if (read_u32ce(offset, sf) == get_id32be("fldr")) {
+                        /* subfolder with another LIST inside, for example "stingers" > N smpl (seen in some music_bank) */
+
+                        suboffset = offset + 0x04;
+                        submax_offset = offset + chunk_size;
+                        while (suboffset < submax_offset) {
+                            uint32_t subchunk_type = read_u32ce(suboffset + 0x00,sf);
+                            uint32_t subchunk_size = read_u32me(suboffset + 0x04,sf);
+                            suboffset += 0x08;
+
+                            if (subchunk_type == get_id32be("LIST")) {
+                                uint32_t subsubchunk_type = read_u32ce(suboffset, sf);
+
+                                if (subsubchunk_type == get_id32be("samp")) {
+                                    total_subsongs++;
+                                    if (target_subsong == total_subsongs && header_offset == 0) {
+                                        header_offset = suboffset;
+                                        header_size = chunk_size;
+                                    }
+                                }
+                                else if (subsubchunk_type == get_id32be("fldr")) {
+                                    VGM_LOG("ISB: subfolder with subfolder at %lx\n", suboffset);
+                                    goto fail;
+                                }
+
+                                //break; /* there can be N subLIST+samps */
+                            }
+
+                            suboffset += subchunk_size;
+                        }
+                    }
+
                     break;
 
                 default: /* most are common chunks at the start that seem to contain defaults */
@@ -89,7 +124,7 @@ VGMSTREAM * init_vgmstream_isb(STREAMFILE *sf) {
                     break;
 
                 case 0x63686E6B: /* "chnk" */
-                    channel_count = read_u32me(offset + 0x00, sf);
+                    channels = read_u32me(offset + 0x00, sf);
                     break;
 
                 case 0x73696E66: /* "sinf" */
@@ -132,12 +167,12 @@ VGMSTREAM * init_vgmstream_isb(STREAMFILE *sf) {
     }
 
 
-    /* some files are marked */
+    /* some files are marked with "loop" but have value 0? */
-    loop_flag  = 0;
+    loop_flag = 0;
 
 
     /* build the VGMSTREAM */
-    vgmstream = allocate_vgmstream(channel_count, loop_flag);
+    vgmstream = allocate_vgmstream(channels, loop_flag);
     if (!vgmstream) goto fail;
 
     vgmstream->meta_type = meta_ISB;
@@ -163,13 +198,13 @@ VGMSTREAM * init_vgmstream_isb(STREAMFILE *sf) {
                 vgmstream->layout_type = layout_interleave;
                 vgmstream->interleave_block_size = 0x02;
             }
-            vgmstream->num_samples = pcm_bytes_to_samples(stream_size, channel_count, bps); /* unsure about pcm_bytes */
+            vgmstream->num_samples = pcm_bytes_to_samples(stream_size, channels, bps); /* unsure about pcm_bytes */
             break;
 
         case 0x01:
             vgmstream->coding_type = coding_XBOX_IMA;
             vgmstream->layout_type = layout_none;
-            vgmstream->num_samples = xbox_ima_bytes_to_samples(stream_size, channel_count); /* pcm_bytes has excess data */
+            vgmstream->num_samples = xbox_ima_bytes_to_samples(stream_size, channels); /* pcm_bytes has excess data */
             break;
 
 #ifdef VGM_USE_VORBIS
@@ -178,7 +213,7 @@ VGMSTREAM * init_vgmstream_isb(STREAMFILE *sf) {
             if (!vgmstream->codec_data) goto fail;
             vgmstream->coding_type = coding_OGG_VORBIS;
             vgmstream->layout_type = layout_none;
-            vgmstream->num_samples = pcm_bytes / channel_count / (bps/8);
+            vgmstream->num_samples = pcm_bytes / channels / (bps/8);
             break;
 #endif
 
@@ -199,7 +234,7 @@ VGMSTREAM * init_vgmstream_isb(STREAMFILE *sf) {
             vgmstream->coding_type = coding_FFmpeg;
             vgmstream->layout_type = layout_none;
 
-            vgmstream->num_samples = pcm_bytes / channel_count / (bps/8);
+            vgmstream->num_samples = pcm_bytes / channels / (bps/8);
             xma_fix_raw_samples(vgmstream, sf, start_offset, stream_size, fmt_offset, 1,1);
             break;
         }

src/meta/ogg_vorbis.c

@@ -438,6 +438,7 @@ VGMSTREAM* init_vgmstream_ogg_vorbis_callbacks(STREAMFILE* sf, ov_callbacks* cal
     int32_t loop_length = ovmi->loop_length;
     int loop_end_found = ovmi->loop_end_found;
     int32_t loop_end = ovmi->loop_end;
+
     size_t stream_size = ovmi->stream_size ?
             ovmi->stream_size :
             get_streamfile_size(sf) - start;
@@ -463,14 +464,14 @@ VGMSTREAM* init_vgmstream_ogg_vorbis_callbacks(STREAMFILE* sf, ov_callbacks* cal
 
         while (ogg_vorbis_get_comment(data, &comment)) {
 
-            if (strstr(comment,"loop_start=") == comment || /* PSO4 */
+            if (strstr(comment,"loop_start=") == comment || /* Phantasy Star Online: Blue Burst (PC) (no loop_end pair) */
-                strstr(comment,"LOOP_START=") == comment || /* PSO4 */
+                strstr(comment,"LOOP_START=") == comment || /* Phantasy Star Online: Blue Burst (PC), common */
                 strstr(comment,"LOOPPOINT=") == comment || /* Sonic Robo Blast 2 */
                 strstr(comment,"COMMENT=LOOPPOINT=") == comment ||
                 strstr(comment,"LOOPSTART=") == comment ||
                 strstr(comment,"um3.stream.looppoint.start=") == comment ||
                 strstr(comment,"LOOP_BEGIN=") == comment || /* Hatsune Miku: Project Diva F (PS3) */
-                strstr(comment,"LoopStart=") == comment ||  /* Devil May Cry 4 (PC) */
+                strstr(comment,"LoopStart=") == comment ||  /* Capcom games [Devil May Cry 4 (PC)] */
                 strstr(comment,"LOOP=") == comment || /* Duke Nukem 3D: 20th Anniversary World Tour */
                 strstr(comment,"XIPH_CUE_LOOPSTART=") == comment) {  /* DeNa games [Super Mario Run (Android), FF Record Keeper (Android)] */
                 loop_start = atol(strrchr(comment,'=')+1);
@@ -486,25 +487,21 @@ VGMSTREAM* init_vgmstream_ogg_vorbis_callbacks(STREAMFILE* sf, ov_callbacks* cal
             }
             else if (strstr(comment,"album=-lpe") == comment) { /* (title=-lps pair) */
                 loop_end = atol(comment+10);
-                loop_flag = 1;
                 loop_end_found = 1;
+                loop_flag = 1;
             }
             else if (strstr(comment,"LoopEnd=") == comment) { /* (LoopStart pair) */
-                if(loop_flag) {
+                loop_end = atol(strrchr(comment,'=')+1);
-                    loop_length = atol(strrchr(comment,'=')+1)-loop_start;
+                loop_end_found = 1;
-                    loop_length_found = 1;
-                }
             }
-            else if (strstr(comment,"LOOP_END=") == comment) { /* (LOOP_BEGIN pair) */
+            else if (strstr(comment,"LOOP_END=") == comment) { /* (LOOP_START/LOOP_BEGIN pair) */
-                if(loop_flag) {
+                loop_end = atol(strrchr(comment,'=')+1);
-                    loop_length = atol(strrchr(comment,'=')+1)-loop_start;
+                loop_end_found = 1;
-                    loop_length_found = 1;
-                }
             }
             else if (strstr(comment,"lp=") == comment) {
                 sscanf(strrchr(comment,'=')+1,"%d,%d", &loop_start,&loop_end);
-                loop_flag = 1;
                 loop_end_found = 1;
+                loop_flag = 1;
             }
             else if (strstr(comment,"LOOPDEFS=") == comment) { /* Fairy Fencer F: Advent Dark Force */
                 sscanf(strrchr(comment,'=')+1,"%d,%d", &loop_start,&loop_end);
@@ -517,10 +514,8 @@ VGMSTREAM* init_vgmstream_ogg_vorbis_callbacks(STREAMFILE* sf, ov_callbacks* cal
                 loop_end_found = 1;
             }
             else if (strstr(comment, "XIPH_CUE_LOOPEND=") == comment) { /* (XIPH_CUE_LOOPSTART pair) */
-                if (loop_flag) {
+                loop_end = atol(strrchr(comment, '=') + 1);
-                    loop_length = atol(strrchr(comment, '=') + 1) - loop_start;
+                loop_end_found = 1;
-                    loop_length_found = 1;
-                }
             }
             else if (strstr(comment, "omment=") == comment) { /* Air (Android) */
                 sscanf(strstr(comment, "=LOOPSTART=") + 11, "%d,LOOPEND=%d", &loop_start, &loop_end);