Merge pull request #864 from bnnm/isb-etc

isb etc
This commit is contained in:
bnnm 2021-05-17 01:09:41 +02:00 committed by GitHub
commit 286d5c6953
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 227 additions and 58 deletions

View File

@ -171,7 +171,7 @@ those files automatically into the playlist. For others without support, you can
multiple .txtp (explained below) to select one of the subsongs (like `bgm.sxd#10.txtp`).
You can use this python script to autogenerate one `.txtp` per subsong:
https://github.com/vgmstream/vgmstream/tree/master/cli/txtp_maker.py
https://github.com/vgmstream/vgmstream/tree/master/cli/tools/txtp_maker.py
Put in the same dir as test.exe/vgmstream_cli, then to drag-and-drop files with
subsongs to `txtp_maker.py` (it has CLI options to control output too).
@ -620,6 +620,11 @@ Support for this feature is limited by player itself, as foobar and Winamp allow
non-existant files referenced in a `.m3u`, while other players may filter them
first.
You can use this python script to autogenerate one `.txtp` per virtual-txtp:
https://github.com/vgmstream/vgmstream/tree/master/cli/tools/txtp_dumper.py
Drag and drop the `.m3u`, or any text file with .txtp (it has CLI options
to control output too).
## Supported codec types
Quick list of codecs vgmstream supports, including many obscure ones that

View File

@ -15,6 +15,13 @@ class Cli(object):
)
epilog = (
"examples:\n"
" %(prog)s *.fsb \n"
" - make .txtp for subsongs in current dir\n\n"
" %(prog)s bgm/*.fsb \n"
" - make .txtp for subsongs in bgm subdir\n\n"
" %(prog)s * -r -fss 1\n"
" - make .txtp for all files in any subdirs with at least 1 subsong\n"
" (ignores formats without subsongs)\n\n"
" %(prog)s bgm.fsb -in -fcm 2 -fms 5.0\n"
" - make .txtp for subsongs with at least 2 channels and 5 seconds\n\n"
" %(prog)s *.scd -r -fd -l 2\n"
@ -23,9 +30,6 @@ class Cli(object):
" - make .txtp for all .sm1 excluding subsongs ending with 'STREAM.SS0..9'\n\n"
" %(prog)s samples.bnk -fni ^bgm.?\n"
" - make .txtp for in .bnk including only subsong names that start with 'bgm'\n\n"
" %(prog)s * -r -fss 1\n"
" - make .txtp for all files in subdirs with at least 1 subsong\n"
" (ignores formats without subsongs)\n\n"
" %(prog)s *.fsb -n \"{fn}<__{ss}>< [{in}]>\" -z 4 -o\n"
" - make .txtp for all fsb, adding subsongs and stream name if they exist\n\n"
)
@ -45,8 +49,9 @@ class Cli(object):
p.add_argument('-ie', dest='no_internal_ext', help="Remove internal name's extension if any", action='store_true')
p.add_argument('-m', dest='mini_txtp', help="Create mini-txtp", action='store_true')
p.add_argument('-o', dest='overwrite', help="Overwrite existing .txtp\n(beware when using with internal names alone)", action='store_true')
p.add_argument('-O', dest='overwrite_rename', help="Rename rather than overwriting", action='store_true')
p.add_argument('-Os', dest='overwrite_suffix', help="Rename with a suffix")
p.add_argument('-oi', dest='overwrite_ignore', help="Ignore repeated rather than overwritting.txtp\n", action='store_true')
p.add_argument('-or', dest='overwrite_rename', help="Rename rather than overwriting", action='store_true')
p.add_argument('-os', dest='overwrite_suffix', help="Rename with a suffix")
p.add_argument('-l', dest='layers', help="Create .txtp per subsong layers, every N channels", type=int)
p.add_argument('-fd', dest='test_dupes', help="Skip .txtp that point to duplicate streams (slower)", action='store_true')
p.add_argument('-fcm', dest='min_channels', help="Filter by min channels", type=int)
@ -150,8 +155,9 @@ class TxtpMaker(object):
self.stream_name = self._get_text("stream name: ")
self.encoding = self._get_text("encoding: ")
# in case vgmstream returns error, but output code wasn't EXIT_FAILURE
if self.channels <= 0 or self.sample_rate <= 0:
raise ValueError('Incorrect command result')
raise ValueError('Incorrect vgmstream command')
self.stream_seconds = self.num_samples / self.sample_rate
self.ignorable = self._is_ignorable(cfg)
@ -255,7 +261,8 @@ class TxtpMaker(object):
outname += '.txtp'
cfg = self.cfg
if (cfg.overwrite_rename or cfg.overwrite_suffix) and os.path.exists(outname):
exists = os.path.exists(outname)
if exists and (cfg.overwrite_rename or cfg.overwrite_suffix):
must_rename = True
if cfg.overwrite_suffix:
outname = outname.replace(".txtp", "%s.txtp" % (cfg.overwrite_suffix))
@ -269,8 +276,14 @@ class TxtpMaker(object):
self.rename_map[outname] = rename_count + 1
outname = outname.replace(".txtp", "_%08i.txtp" % (rename_count))
if not cfg.overwrite and os.path.exists(outname):
raise ValueError('TXTP exists in path: ' + outname)
# decide action after (possible) renames above
if os.path.exists(outname):
if cfg.overwrite_ignore:
log.debug("ignored: " + outname)
return
if not cfg.overwrite:
raise ValueError('TXTP exists in path: ' + outname)
ftxtp = open(outname,"w+")
if line:
@ -436,7 +449,14 @@ class App(object):
if os.path.isdir(pattern):
dir = pattern
pattern = None
for dirsep in '/\\':
if dirsep in pattern:
index = pattern.rfind(dirsep)
if index >= 0:
dir = pattern[0:index]
pattern = pattern[index+1:]
files = []
for root, dirnames, filenames in os.walk(dir):
for filename in fnmatch.filter(filenames, pattern):
@ -456,7 +476,6 @@ class App(object):
for filename in self.cfg.files:
filenames_in += self._find_files('.', filename)
rename_map = {}
total_created = 0
@ -480,19 +499,22 @@ class App(object):
target_subsong = 1
while True:
try:
# main call to vgmstream
cmd = self._make_cmd(filename_in, filename_out, target_subsong)
log.debug("calling: %s", cmd)
output_b = subprocess.check_output(cmd, shell=False) #stderr=subprocess.STDOUT
except subprocess.CalledProcessError as e:
log.debug("ignoring CLI error in %s #%s: %s", filename_in, target_subsong, str(e.output))
# basic parse of vgmstream info
maker = TxtpMaker(self.cfg, output_b, rename_map)
except (subprocess.CalledProcessError, ValueError) as e:
log.debug("ignoring CLI error in %s #%s: %s", filename_in, target_subsong, str(e))
errors += 1
break
if target_subsong == 1:
log.debug("processing %s...", filename_in_clean)
maker = TxtpMaker(self.cfg, output_b, rename_map)
if not maker.is_ignorable():
self.crc32.update(filename_out)

107
cli/tools/txtp_dumper.py Normal file
View File

@ -0,0 +1,107 @@
#!/usr/bin/env python3
import os, glob, argparse
#******************************************************************************
# TXTP DUMPER
#
# Creates .txtp from a text list, mainly .m3u with virtual txtp to actual .txtp
# example:
#
# # %TITLE Stage 1
# bgm.awb#1 .txtp
# >> creates bgm.awb#1 .txtp
#
#******************************************************************************
class Cli(object):
def _parse(self):
description = (
"Makes TXTP from list"
)
epilog = (
"examples:\n"
" %(prog)s !tags.m3u\n"
" - make .txtp per line in !tags.m3u\n\n"
" %(prog)s !tags.m3u -m \n"
" - make full txtp rather than mini-txtp\n (may overwrite files when using subsongs)\n\n"
" %(prog)s lines.txt -s sound \n"
" - make .txtp per line, setting a subdir\n\n"
)
p = argparse.ArgumentParser(description=description, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter)
p.add_argument('files', help="Files to get (wildcards work)", nargs='+')
p.add_argument('-s', dest='subdir', help="Create .txtp pointing to subdir")
p.add_argument('-m', dest='maxitxtp', help="Create regular txtp rather than mini-txtp", action='store_true')
p.add_argument('-f', dest='force', help="Make .txtp even if line isn't a .txtp", action='store_true')
p.add_argument('-o', dest='output', help="Output dir (default: current)")
#p.add_argument('-n', dest='normalize', help="Normalize .txtp formatting")
return p.parse_args()
def start(self):
args = self._parse()
if not args.files:
return
App(args).start()
#******************************************************************************
class App(object):
def __init__(self, args):
self.args = args
def start(self):
filenames = []
for filename in self.args.files:
filenames += glob.glob(filename)
for filename in filenames:
path = self.args.output or '.'
if self.args.output:
try:
os.makedirs(self.args.output)
except OSError:
pass
with open(filename) as fi:
for line in fi:
line = line.strip()
line = line.rstrip()
if line.startswith('#'):
continue
if not line.endswith('.txtp'):
if self.args.force:
line += '.txtp'
else:
continue
line.replace('\\', '/')
subdir = self.args.subdir
if self.args.maxitxtp or subdir:
index = line.find('.') #first extension
if line[index:].startswith('.txtp'): #???
continue
name = line[0:index] + '.txtp'
text = line.replace('.txtp', '').strip()
if subdir:
subdir.replace('\\', '/')
if not subdir.endswith('/'):
subdir = subdir + '/'
text = subdir + text
else:
name = line
text = ''
outpath = os.path.join(path, name)
with open(outpath, 'w') as fo:
if text:
fo.write(text)
pass
if __name__ == "__main__":
Cli().start()

View File

@ -557,7 +557,7 @@ static int convert_file(cli_config* cfg);
int main(int argc, char** argv) {
cli_config cfg = {0};
int i, res;
int i, res, ok;
/* read args */
@ -574,15 +574,20 @@ int main(int argc, char** argv) {
res = validate_config(&cfg);
if (!res) goto fail;
ok = 0;
for (i = 0; i < cfg.infilenames_count; i++) {
/* current name, to avoid passing params all the time */
cfg.infilename = cfg.infilenames[i];
cfg.outfilename = NULL;
if (cfg.outfilename_config)
cfg.outfilename = NULL;
convert_file(&cfg);
res = convert_file(&cfg);
//if (!res) goto fail; /* keep on truckin' */
if (res) ok = 1; /* return ok if at least one succeeds, for programs that check result code */
}
if (!ok)
goto fail;
return EXIT_SUCCESS;
fail:
@ -713,7 +718,7 @@ static int convert_file(cli_config* cfg) {
fclose(outfile);
}
close_vgmstream(vgmstream);
return EXIT_SUCCESS;
return 1;
}
if (cfg->seek_samples1 < -1) /* ex value for loop testing */

View File

@ -7,7 +7,7 @@ VGMSTREAM * init_vgmstream_isb(STREAMFILE *sf) {
VGMSTREAM * vgmstream = NULL;
off_t start_offset = 0, name_offset = 0;
size_t stream_size = 0, name_size = 0;
int loop_flag = 0, channel_count = 0, sample_rate = 0, codec = 0, pcm_bytes = 0, bps = 0;
int loop_flag = 0, channels = 0, sample_rate = 0, codec = 0, pcm_bytes = 0, bps = 0;
int total_subsongs, target_subsong = sf->stream_index;
uint32_t (*read_u32me)(off_t,STREAMFILE*);
uint32_t (*read_u32ce)(off_t,STREAMFILE*);
@ -18,15 +18,15 @@ VGMSTREAM * init_vgmstream_isb(STREAMFILE *sf) {
if (!check_extensions(sf, "isb"))
goto fail;
big_endian = read_u32be(0x00,sf) == 0x46464952; /* "FFIR" */
big_endian = read_u32be(0x00,sf) == get_id32be("FFIR"); /* PS3, most X360 */
read_u32me = big_endian ? read_u32be : read_u32le; /* machine endianness... */
read_u32ce = big_endian ? read_u32le : read_u32be; /* chunks change with endianness but this just reads as BE */
if (read_u32ce(0x00,sf) != 0x52494646) /* "RIFF" */
if (read_u32ce(0x00,sf) != get_id32be("RIFF"))
goto fail;
if (read_u32me(0x04,sf) + 0x08 != get_streamfile_size(sf))
goto fail;
if (read_u32ce(0x08,sf) != 0x69736266) /* "isbf" */
if (read_u32ce(0x08,sf) != get_id32be("isbf"))
goto fail;
/* some files have a companion .icb, seems to be a cue file pointing here */
@ -36,7 +36,7 @@ VGMSTREAM * init_vgmstream_isb(STREAMFILE *sf) {
* seems to use the format as a simple audio bank. Psychonauts Xbox/PS2 doesn't use ISACT. */
{
off_t offset, max_offset, header_offset = 0;
off_t offset, max_offset, header_offset = 0, suboffset, submax_offset;
size_t header_size = 0;
total_subsongs = 0; /* not specified */
@ -50,16 +50,51 @@ VGMSTREAM * init_vgmstream_isb(STREAMFILE *sf) {
uint32_t chunk_size = read_u32me(offset + 0x04,sf);
offset += 0x08;
//VGM_LOG("offset=%lx, sub=%c%c%c%c\n", offset,(chunk_type>>24 & 0xFF), (chunk_type>>16 & 0xFF), (chunk_type>>8 & 0xFF), (chunk_type & 0xFF));
switch(chunk_type) {
case 0x4C495354: /* "LIST" */
if (read_u32ce(offset, sf) != 0x73616D70) /* "samp" */
break; /* there are "bfob" LIST without data */
total_subsongs++;
if (target_subsong == total_subsongs && header_offset == 0) {
header_offset = offset;
header_size = chunk_size;
if (read_u32ce(offset, sf) == get_id32be("samp")) {
/* sample header */
total_subsongs++;
if (target_subsong == total_subsongs && header_offset == 0) {
header_offset = offset;
header_size = chunk_size;
}
}
else if (read_u32ce(offset, sf) == get_id32be("fldr")) {
/* subfolder with another LIST inside, for example "stingers" > N smpl (seen in some music_bank) */
suboffset = offset + 0x04;
submax_offset = offset + chunk_size;
while (suboffset < submax_offset) {
uint32_t subchunk_type = read_u32ce(suboffset + 0x00,sf);
uint32_t subchunk_size = read_u32me(suboffset + 0x04,sf);
suboffset += 0x08;
if (subchunk_type == get_id32be("LIST")) {
uint32_t subsubchunk_type = read_u32ce(suboffset, sf);
if (subsubchunk_type == get_id32be("samp")) {
total_subsongs++;
if (target_subsong == total_subsongs && header_offset == 0) {
header_offset = suboffset;
header_size = chunk_size;
}
}
else if (subsubchunk_type == get_id32be("fldr")) {
VGM_LOG("ISB: subfolder with subfolder at %lx\n", suboffset);
goto fail;
}
//break; /* there can be N subLIST+samps */
}
suboffset += subchunk_size;
}
}
break;
default: /* most are common chunks at the start that seem to contain defaults */
@ -89,7 +124,7 @@ VGMSTREAM * init_vgmstream_isb(STREAMFILE *sf) {
break;
case 0x63686E6B: /* "chnk" */
channel_count = read_u32me(offset + 0x00, sf);
channels = read_u32me(offset + 0x00, sf);
break;
case 0x73696E66: /* "sinf" */
@ -132,12 +167,12 @@ VGMSTREAM * init_vgmstream_isb(STREAMFILE *sf) {
}
/* some files are marked */
loop_flag = 0;
/* some files are marked with "loop" but have value 0? */
loop_flag = 0;
/* build the VGMSTREAM */
vgmstream = allocate_vgmstream(channel_count, loop_flag);
vgmstream = allocate_vgmstream(channels, loop_flag);
if (!vgmstream) goto fail;
vgmstream->meta_type = meta_ISB;
@ -163,13 +198,13 @@ VGMSTREAM * init_vgmstream_isb(STREAMFILE *sf) {
vgmstream->layout_type = layout_interleave;
vgmstream->interleave_block_size = 0x02;
}
vgmstream->num_samples = pcm_bytes_to_samples(stream_size, channel_count, bps); /* unsure about pcm_bytes */
vgmstream->num_samples = pcm_bytes_to_samples(stream_size, channels, bps); /* unsure about pcm_bytes */
break;
case 0x01:
vgmstream->coding_type = coding_XBOX_IMA;
vgmstream->layout_type = layout_none;
vgmstream->num_samples = xbox_ima_bytes_to_samples(stream_size, channel_count); /* pcm_bytes has excess data */
vgmstream->num_samples = xbox_ima_bytes_to_samples(stream_size, channels); /* pcm_bytes has excess data */
break;
#ifdef VGM_USE_VORBIS
@ -178,7 +213,7 @@ VGMSTREAM * init_vgmstream_isb(STREAMFILE *sf) {
if (!vgmstream->codec_data) goto fail;
vgmstream->coding_type = coding_OGG_VORBIS;
vgmstream->layout_type = layout_none;
vgmstream->num_samples = pcm_bytes / channel_count / (bps/8);
vgmstream->num_samples = pcm_bytes / channels / (bps/8);
break;
#endif
@ -199,7 +234,7 @@ VGMSTREAM * init_vgmstream_isb(STREAMFILE *sf) {
vgmstream->coding_type = coding_FFmpeg;
vgmstream->layout_type = layout_none;
vgmstream->num_samples = pcm_bytes / channel_count / (bps/8);
vgmstream->num_samples = pcm_bytes / channels / (bps/8);
xma_fix_raw_samples(vgmstream, sf, start_offset, stream_size, fmt_offset, 1,1);
break;
}

View File

@ -438,6 +438,7 @@ VGMSTREAM* init_vgmstream_ogg_vorbis_callbacks(STREAMFILE* sf, ov_callbacks* cal
int32_t loop_length = ovmi->loop_length;
int loop_end_found = ovmi->loop_end_found;
int32_t loop_end = ovmi->loop_end;
size_t stream_size = ovmi->stream_size ?
ovmi->stream_size :
get_streamfile_size(sf) - start;
@ -463,14 +464,14 @@ VGMSTREAM* init_vgmstream_ogg_vorbis_callbacks(STREAMFILE* sf, ov_callbacks* cal
while (ogg_vorbis_get_comment(data, &comment)) {
if (strstr(comment,"loop_start=") == comment || /* PSO4 */
strstr(comment,"LOOP_START=") == comment || /* PSO4 */
if (strstr(comment,"loop_start=") == comment || /* Phantasy Star Online: Blue Burst (PC) (no loop_end pair) */
strstr(comment,"LOOP_START=") == comment || /* Phantasy Star Online: Blue Burst (PC), common */
strstr(comment,"LOOPPOINT=") == comment || /* Sonic Robo Blast 2 */
strstr(comment,"COMMENT=LOOPPOINT=") == comment ||
strstr(comment,"LOOPSTART=") == comment ||
strstr(comment,"um3.stream.looppoint.start=") == comment ||
strstr(comment,"LOOP_BEGIN=") == comment || /* Hatsune Miku: Project Diva F (PS3) */
strstr(comment,"LoopStart=") == comment || /* Devil May Cry 4 (PC) */
strstr(comment,"LoopStart=") == comment || /* Capcom games [Devil May Cry 4 (PC)] */
strstr(comment,"LOOP=") == comment || /* Duke Nukem 3D: 20th Anniversary World Tour */
strstr(comment,"XIPH_CUE_LOOPSTART=") == comment) { /* DeNa games [Super Mario Run (Android), FF Record Keeper (Android)] */
loop_start = atol(strrchr(comment,'=')+1);
@ -486,25 +487,21 @@ VGMSTREAM* init_vgmstream_ogg_vorbis_callbacks(STREAMFILE* sf, ov_callbacks* cal
}
else if (strstr(comment,"album=-lpe") == comment) { /* (title=-lps pair) */
loop_end = atol(comment+10);
loop_flag = 1;
loop_end_found = 1;
loop_flag = 1;
}
else if (strstr(comment,"LoopEnd=") == comment) { /* (LoopStart pair) */
if(loop_flag) {
loop_length = atol(strrchr(comment,'=')+1)-loop_start;
loop_length_found = 1;
}
loop_end = atol(strrchr(comment,'=')+1);
loop_end_found = 1;
}
else if (strstr(comment,"LOOP_END=") == comment) { /* (LOOP_BEGIN pair) */
if(loop_flag) {
loop_length = atol(strrchr(comment,'=')+1)-loop_start;
loop_length_found = 1;
}
else if (strstr(comment,"LOOP_END=") == comment) { /* (LOOP_START/LOOP_BEGIN pair) */
loop_end = atol(strrchr(comment,'=')+1);
loop_end_found = 1;
}
else if (strstr(comment,"lp=") == comment) {
sscanf(strrchr(comment,'=')+1,"%d,%d", &loop_start,&loop_end);
loop_flag = 1;
loop_end_found = 1;
loop_flag = 1;
}
else if (strstr(comment,"LOOPDEFS=") == comment) { /* Fairy Fencer F: Advent Dark Force */
sscanf(strrchr(comment,'=')+1,"%d,%d", &loop_start,&loop_end);
@ -517,10 +514,8 @@ VGMSTREAM* init_vgmstream_ogg_vorbis_callbacks(STREAMFILE* sf, ov_callbacks* cal
loop_end_found = 1;
}
else if (strstr(comment, "XIPH_CUE_LOOPEND=") == comment) { /* (XIPH_CUE_LOOPSTART pair) */
if (loop_flag) {
loop_length = atol(strrchr(comment, '=') + 1) - loop_start;
loop_length_found = 1;
}
loop_end = atol(strrchr(comment, '=') + 1);
loop_end_found = 1;
}
else if (strstr(comment, "omment=") == comment) { /* Air (Android) */
sscanf(strstr(comment, "=LOOPSTART=") + 11, "%d,LOOPEND=%d", &loop_start, &loop_end);