Merge pull request #53 from bnnm/misc-changes

Misc changes

!!!files_to_change_for_metas.txt

@@ -1 +0,0 @@
-vgmstream.c, in_vgmstream.c, vgmstream.h, meta.h, data.c, vgmstream\src\Makefile, vgmstream\src\meta\Makefile.unix.am, fb2k/in_vgmstream.cpp, unix/data.c

BUILD.md

@@ -0,0 +1,101 @@
+# vgmstream
+
+## Compilation requirements
+
+**GCC**: you need GCC and MAKE somewhere in path. In Windows this means one of these:
+- MinGW-w64 (32bit version): https://sourceforge.net/projects/mingw-w64/
+- MSYS2 with the MinGW-w64_shell (32bit) package: https://msys2.github.io/
+
+**MSVC / Visual Studio**: Visual Studio Community 2015 (free) should work:
+- Visual Studio: https://www.visualstudio.com/downloads/
+
+**Git**: optional, to generate version numbers:
+- Git for Windows: https://git-scm.com/download/win
+
+## Compiling modules
+
+### test.exe / in_vgmstream (Winamp)
+
+**With GCC**: use the *./Makefile* in the root folder, see inside for options. For compilation flags check *test/Makefile.mingw* or *winamp/Makefile.mingw*.
+You need to manually rebuild if you change a *.h* file (use *make clean*).
+
+In Linux you may need to use *Makefile.unix.am* instead, and note that some Linux makefiles aren't up to date.
+
+Windows CMD example for test.exe:
+```
+set PATH=%PATH%;C:\Git\usr\bin
+set PATH=%PATH%;C:\mingw-w64\i686-5.4.0-win32-sjlj-rt_v5-rev0\mingw32\bin
+
+cd vgmstream
+
+mingw32-make.exe mingw_test -f Makefile ^
+ VGM_ENABLE_FFMPEG=1 VGM_ENABLE_MAIATRAC3PLUS=0 ^
+ SHELL=sh.exe CC=gcc.exe AR=ar.exe STRIP=strip.exe DLLTOOL=dlltool.exe WINDRES=windres.exe
+```
+
+**With MSVC**: open *./vgmstream.sln* and compile in Visual Studio.
+
+### foo_input_vgmstream (foobar2000)
+Requires MSVC (foobar/SDK only links to MSVC C++ DLLs) and these dependencies:
+- foobar2000 SDK, in ../foobar/: http://www.foobar2000.org/SDK
+- WTL includes (if needed): http://wtl.sourceforge.net/
+- FDK-AAC, in ../fdk-aac/: https://github.com/kode54/fdk-aac
+- QAAC, in ../qaac/: https://github.com/kode54/qaac
+FDK-AAC/QAAC can be disabled by removing VGM_USE_MP4V2 and VGM_USE_FDKAAC.
+
+Open *./vgmstream.sln* as a base and add *fb2k/foo_input_vgmstream.vcxproj*, which expects the above.
+You may need to manually add includes and libs from WTL and ../foobar/foobar2000/shared/ in the compiler/linker options.
+VS2013 may not be compatible with the SDK.
+
+### xmp-vgmstream (XMPlay)
+Currently only MSVC is supported, though it should be compilable with GCC.
+
+Use *xmp-vgmstream/xmp-vgmstream.sln*; FDK-AAC/QAAC may be needed (see above).
+
+
+## Development
+
+### Structure
+vgmstream uses C (C89 when possible), except the foobar2000 plugin (C++).
+
+```
+./                   docs, scripts
+./ext_includes/      external includes for compiling
+./ext_libs/          external libs/DLLs for linking
+./fb2k/              foobar2000 plugin
+./src/               main vgmstream code and helpers
+./src/coding/        format sample decoders
+./src/layout/        format data demuxers
+./src/meta/          format header parsers
+./test/              test.exe CLI
+./unix/              Audacious plugin
+./winamp/            Winamp plugin
+./xmp-vgmstream/     XMPlay plugin
+```
+
+### Overview
+vgmstream works by parsing a music stream header (*meta/*), reading/demuxing data or looping (*layout/*) and decoding the compressed data into listenable PCM samples (*coding/*).
+
+Very simplified it goes like this:
+- player (test.exe, winamp plugin, etc) inits the stream *[main]*
+- init tries all parsers (metas) until one works *[init_vgmstream]*
+- parser reads header (channels, sample rate, loop points) and set ups a VGMSTREAM struct + layout/coding, if the format is correct *[init_vgmstream_(format-name)]*
+- player gets total_samples to play, based on the number of loops and other settings *[get_vgmstream_play_samples]*
+- player asks to fill a small sample buffer *[render_vgmstream]*
+- layout prepares bytes to read from the stream *[render_vgmstream_(layout)]*
+- decoder decodes bytes into PCM samples *[decode_vgmstream_(coding)]*
+- player plays those samples, asks to fill sample buffer, repeats until total_samples
+- layout moves back to loop_start when loop_end is reached *[vgmstream_do_loop]*
+
+### Adding new formats
+For new simple formats, assuming existing layout/coding:
+- *src/meta/(format-name).c*: create new format parser that reads all needed info from the stream header and inits VGMSTREAM
+- *src/meta/meta.h*: register parser's init
+- *src/vgmstream.h*: register new meta
+- *src/vgmstream.c*: add parser init to search list, add meta description
+- *winamp/in_vgmstream.c*
+  *fb2k/in_vgmstream.cpp*
+  *xml-vgmstream/DllMain.c*: add new extension to the format list
+- *src/Makefile*
+  *src/meta/Makefile.unix.am*
+  *src/libvgmstream.vcproj/vcxproj*: to compile new (format-name).c parser

readme.txt

@@ -8,7 +8,7 @@ called "xmp-vgmstream".
 
 *********** IMPORTANT!! ***********
 --- needed files (for Windows)  ---
-Since Ogg Vorbis, MPEG audio,and other formats are now supported, you will
+Since Ogg Vorbis, MPEG audio, and other formats are now supported, you will
 need to have certain DLL files.
 You can get these from https://f.losno.co/vgmstream-win32-deps.zip, or in
 the case of the foobar2000 component, they are all bundled for convenience.
@@ -17,8 +17,8 @@ Put libvorbis.dll, libmpg123-0.dll, libg7221_decode.dll, libg719_decode.dll,
 at3plusdecoder.dll, avcodec-vgmstream-57.dll, avformat-vgmstream-57.dll, and
 avutil-vgmstream-55.dll somewhere Windows can find them.
 For in_vgmstream this means in the directory with winamp.exe, or in a
-system directory. For test.exe this means in the same directory as test.exe,
+system directory or other directory in the PATH variable. For test.exe this
-or in a system directory.
+means in the same directory as test.exe, or in a system directory/PATH.
 
 --- test.exe ---
 Usage: ./test [-o outfile.wav] [-l loop count]
@@ -55,6 +55,8 @@ the above instructions for installing the other files needed.
 As manakoAT likes to say, the extension doesn't really mean anything, but it's
 the most obvious way to identify files.
 
+This list is not complete and many other files are supported.
+
 PS2/PSX ADPCM:
 - .ads/.ss2
 - .ass
@@ -221,6 +223,7 @@ multi:
 - .emff (PSX APDCM, GC DSP ADPCM)
 - .fsb, .wii (PSX ADPCM, GC DSP ADPCM, Xbox IMA ADPCM)
 - .genh (lots)
+- .msf (PCM, PSX ADPCM, ATRAC3, MP3)
 - .musx (PSX ADPCM, Xbox IMA ADPCM, DAT4 IMA ADPCM)
 - .nwa (16 bit PCM, NWA DPCM)
 - .psw (PSX ADPCM, GC DSP ADPCM)
@@ -229,6 +232,7 @@ multi:
 - .rsd (PSX ADPCM, 16 bit PCM, GC DSP ADPCM, Xbox IMA ADPCM, Radical ADPCM)
 - .rrds (NDS IMA ADPCM)
 - .sad (GC DSP ADPCM, NDS IMA ADPCM, Procyon Studios NDS ADPCM)
+- .sgd/sgb/sgx (PSX ADPCM, ATRAC3plus, AC3)
 - .seg (Xbox IMA ADPCM, PS2 ADPCM)
 - .sng, .asf, .str, .eam (EA/XA ADPCM or PSX ADPCM)
 - .strm (NDS IMA ADPCM, 8/16 bit PCM)
@@ -246,11 +250,13 @@ etc:
 - .afc (GC AFC ADPCM)
 - .ahx (MPEG-2 Layer II)
 - .aix (CRI ADX ADPCM)
+- .at3 (Sony ATRAC3 / ATRAC3plus)
 - .baf (Blur ADPCM)
 - .bgw (FFXI PS-like ADPCM)
 - .bnsf (G.722.1)
 - .caf (Apple IMA4 ADPCM)
 - .de2 (MS ADPCM)
+- .hca (CRI)
 - .kcey (EACS IMA ADPCM)
 - .lsf (LSF ADPCM)
 - .mwv (Level-5 0x555 ADPCM)
@@ -276,5 +282,10 @@ loop assists:
 - .sli (loop info for .ogg)
 - .sfl (loop info for .ogg)
 
+other:
+- .adxkey (decryption key for .adx, in start/mult/add format)
+- .hcakey (decryption key for .hca, in HCA Decoder format)
+- .vgmstream/.vgms + .pos (to force FFmpeg formats + loop assist)
+
 Enjoy!
 -hcs

src/coding/ffmpeg_decoder.c

@@ -278,8 +278,8 @@ void seek_ffmpeg(VGMSTREAM *vgmstream, int32_t num_sample) {
     }
 
     /* todo fix this properly */
-    if (data->totalFrames) {
+    if (data->totalSamples) {
-        ts = (int)ts * (data->formatCtx->duration) / data->totalFrames;
+        ts = (int)ts * (data->formatCtx->duration) / data->totalSamples;
     } else {
         data->samplesToDiscard = num_sample;
         ts = 0;

src/meta/ffmpeg.c

@@ -5,7 +5,7 @@
 #ifdef VGM_USE_FFMPEG
 
 /* internal sizes, can be any value */
-#define FFMPEG_DEFAULT_BLOCK_SIZE 2048
+#define FFMPEG_DEFAULT_BUFFER_SIZE 2048
 #define FFMPEG_DEFAULT_IO_BUFFER_SIZE 128 * 1024
 
 static int init_seek(ffmpeg_codec_data * data);
@@ -43,23 +43,52 @@ VGMSTREAM * init_vgmstream_ffmpeg(STREAMFILE *streamFile) {
 
 VGMSTREAM * init_vgmstream_ffmpeg_offset(STREAMFILE *streamFile, uint64_t start, uint64_t size) {
     VGMSTREAM *vgmstream = NULL;
+    int loop_flag = 0;
+    int32_t loop_start = 0, loop_end = 0, num_samples = 0;
 
+    /* init ffmpeg */
     ffmpeg_codec_data *data = init_ffmpeg_offset(streamFile, start, size);
     if (!data) return NULL;
-    
+
-    vgmstream = allocate_vgmstream(data->channels, 0);
+
+    /* try to get .pos data */
+    {
+        uint8_t posbuf[4+4+4];
+
+        if ( read_pos_file(posbuf, 4+4+4, streamFile) ) {
+            loop_start = get_32bitLE(posbuf+0);
+            loop_end = get_32bitLE(posbuf+4);
+            loop_flag = 1; /* incorrect looping will be validated outside */
+            /* FFmpeg can't always determine totalSamples correctly so optionally load it (can be 0/NULL)
+             * won't crash and will output silence if no loop points and bigger than actual stream's samples */
+            num_samples = get_32bitLE(posbuf+8);
+        }
+    }
+
+
+    /* build VGMSTREAM */
+    vgmstream = allocate_vgmstream(data->channels, loop_flag);
     if (!vgmstream) goto fail;
     
-    vgmstream->loop_flag = 0;
+    vgmstream->loop_flag = loop_flag;
     vgmstream->codec_data = data;
     vgmstream->channels = data->channels;
     vgmstream->sample_rate = data->sampleRate;
-    vgmstream->num_samples = data->totalFrames;
     vgmstream->coding_type = coding_FFmpeg;
     vgmstream->layout_type = layout_none;
     vgmstream->meta_type = meta_FFmpeg;
 
-    /* this may happen for some streams */
+    if (!num_samples) {
+        num_samples = data->totalSamples;
+    }
+    vgmstream->num_samples = num_samples;
+
+    if (loop_flag) {
+        vgmstream->loop_start_sample = loop_start;
+        vgmstream->loop_end_sample = loop_end;
+    }
+
+    /* this may happen for some streams if FFmpeg can't determine it */
     if (vgmstream->num_samples <= 0)
         goto fail;
 
@@ -337,13 +366,18 @@ ffmpeg_codec_data * init_ffmpeg_faux_riff(STREAMFILE *streamFile, int64_t fmt_of
 
     /* try to guess frames/samples (duration isn't always set) */
     tb.num = 1; tb.den = data->codecCtx->sample_rate;
-    data->totalFrames = av_rescale_q(stream->duration, stream->time_base, tb);
+    data->totalSamples = av_rescale_q(stream->duration, stream->time_base, tb);
-    if (data->totalFrames < 0)
+    if (data->totalSamples < 0)
-        data->totalFrames = 0; /* caller must consider this */
+        data->totalSamples = 0; /* caller must consider this */
 
+    data->blockAlign = data->codecCtx->block_align;
+    data->frameSize = data->codecCtx->frame_size;
+    if(data->frameSize == 0) /* some formats don't set frame_size but can get on request, and vice versa */
+        data->frameSize = av_get_audio_frame_duration(data->codecCtx,0);
+    
     /* setup decode buffer */
-    data->samplesPerBlock = FFMPEG_DEFAULT_BLOCK_SIZE;
+    data->sampleBufferBlock = FFMPEG_DEFAULT_BUFFER_SIZE;
-    data->sampleBuffer = av_malloc( data->samplesPerBlock * (data->bitsPerSample / 8) * data->channels );
+    data->sampleBuffer = av_malloc( data->sampleBufferBlock * (data->bitsPerSample / 8) * data->channels );
     if (!data->sampleBuffer)
         goto fail;
     
@@ -409,6 +443,7 @@ static int init_seek(ffmpeg_codec_data * data) {
         if (!found_first) { /* first found */
             found_first = 1;
             pos = pkt->pos;
+            ts = pkt->dts;
             continue;
         } else { /* second found */
             size = pkt->pos - pos; /* coded, pkt->size is decoded size */
@@ -416,6 +451,11 @@ static int init_seek(ffmpeg_codec_data * data) {
         }
     }
 
+    /* apparently some (non-audio?) streams start with a DTS before 0, but some read_seeks expect 0, which would disrupt the index
+     *  we may need to keep start_ts around, since avstream/codec/format isn't always set */
+    if (ts != 0)
+        goto fail;
+
     /* add index 0 */
     ret = av_add_index_entry(stream, pos, ts, size, distance, AVINDEX_KEYFRAME);
     if ( ret < 0 )

src/meta/mp4.c

@@ -210,7 +210,7 @@ VGMSTREAM * init_vgmstream_mp4_aac_ffmpeg(STREAMFILE *streamFile) {
     vgmstream = allocate_vgmstream(ffmpeg_data->channels,loop_flag);
     if (!vgmstream) goto fail;
 
-    vgmstream->num_samples = ffmpeg_data->totalFrames; /* todo compare with FFD num_samples*/
+    vgmstream->num_samples = ffmpeg_data->totalSamples; /* todo FFD num_samples is different from this */
     vgmstream->sample_rate = ffmpeg_data->sampleRate;
     vgmstream->channels = ffmpeg_data->channels;
     if (loop_flag) {

src/meta/ps3_msf.c

@@ -133,13 +133,11 @@ VGMSTREAM * init_vgmstream_ps3_msf(STREAMFILE *streamFile) {
             vgmstream->meta_type = meta_FFmpeg;
             vgmstream->codec_data = ffmpeg_data;
 
-            vgmstream->num_samples = ffmpeg_data->totalFrames;
+            vgmstream->num_samples = ffmpeg_data->totalSamples;
-            if (loop_flag) {
+
-                int atrac3_frame = 1024;
+            if (loop_flag && ffmpeg_data->blockAlign > 0) {
-                int block_align = (codec_id == 0x4 ? 96 : codec_id == 0x5 ? 152 : 192) * channel_count;
+                vgmstream->loop_start_sample = (loop_start / ffmpeg_data->blockAlign) * ffmpeg_data->frameSize;
-                /* int block_align = ffmpeg_data->codecCtx->block_align; *//* is this always set? */
+                vgmstream->loop_end_sample = (loop_end / ffmpeg_data->blockAlign) * ffmpeg_data->frameSize;
-                vgmstream->loop_start_sample = (loop_start / block_align) * atrac3_frame;
-                vgmstream->loop_end_sample = (loop_end / block_align) * atrac3_frame;
             }
 
             break;
@@ -155,12 +153,14 @@ VGMSTREAM * init_vgmstream_ps3_msf(STREAMFILE *streamFile) {
             vgmstream->meta_type = meta_FFmpeg;
             vgmstream->codec_data = ffmpeg_data;
 
-            /* todo check CBR better (frame_size=0?) */
+            /* TODO check CBR better (bitrate % X != 0?) */
+            if (ffmpeg_data->bitrate == 0)
+                goto fail;
 
-            /* vgmstream->num_samples = ffmpeg_data->totalFrames; */ /* duration is not set/innacurate for MP3 in FFMpeg */
+            /* vgmstream->num_samples = ffmpeg_data->totalSamples; */ /* duration may not be set/inaccurate */
             vgmstream->num_samples = (int64_t)data_size * ffmpeg_data->sampleRate * 8 / ffmpeg_data->bitrate;
             if (loop_flag) {
-                int frame_size = ffmpeg_data->codecCtx->frame_size;
+                int frame_size = ffmpeg_data->frameSize;
                 vgmstream->loop_start_sample = (int64_t)loop_start * ffmpeg_data->sampleRate * 8 / ffmpeg_data->bitrate;
                 vgmstream->loop_start_sample -= vgmstream->loop_start_sample==frame_size ? frame_size
                         : vgmstream->loop_start_sample % frame_size;

src/meta/riff.c

@@ -402,7 +402,7 @@ VGMSTREAM * init_vgmstream_riff(STREAMFILE *streamFile) {
                 ffmpeg_data = init_ffmpeg_offset(streamFile, 0, streamFile->get_size(streamFile) );
                 if ( !ffmpeg_data ) goto fail;
 
-                sample_count = ffmpeg_data->totalFrames; /* fact_sample_count */
+                sample_count = ffmpeg_data->totalSamples; /* fact_sample_count */
                 /* the encoder introduces some garbage (usually silent) samples to skip before the stream
                  *  loop values include the skip samples but fact_sample_count doesn't; add them back to fix some edge loops */
                 if (fact_sample_skip > 0)
@@ -412,9 +412,12 @@ VGMSTREAM * init_vgmstream_riff(STREAMFILE *streamFile) {
 #endif
 #ifdef VGM_USE_MAIATRAC3PLUS
 		case coding_AT3plus:
-            /* rough total samples, not totally accurate since there are some skipped samples in the beginning
+            /* rough total samples (block_size may be incorrect if not using joint stereo) */
-             * channels shouldn't matter (mono and stereo encoding produces the same number of frames in ATRAC3plus) */
+            sample_count = (data_size / fmt.block_size) * 2048;
-            sample_count = (data_size / fmt.block_size) * 2048; /* number_of_frames by decoded_samples_per_frame */
+            /* favor fact_samples if available (skip isn't correctly handled for now) */
+            if (fact_sample_count > 0 && fact_sample_count + fact_sample_skip < sample_count)
+                sample_count = fact_sample_count + fact_sample_skip;
+
 			break;
 #endif
         default:

src/meta/sqex_scd.c

@@ -295,7 +295,7 @@ VGMSTREAM * init_vgmstream_sqex_scd(STREAMFILE *streamFile) {
                     vgmstream->coding_type = coding_FFmpeg;
                     vgmstream->layout_type = layout_none;
                     
-                    vgmstream->num_samples = ffmpeg_data->totalFrames;
+                    vgmstream->num_samples = ffmpeg_data->totalSamples;
 
                     if (loop_flag) {
                         vgmstream->loop_start_sample = loop_start;

src/streamfile.c

@@ -350,3 +350,44 @@ fail:
     return 0;
 }
 
+/**
+ * open file containing looping data and copy to buffer
+ *
+ * returns true if found and copied
+ */
+int read_pos_file(uint8_t * buf, size_t bufsize, STREAMFILE *streamFile) {
+    char posname[PATH_LIMIT];
+    char filename[PATH_LIMIT];
+    /*size_t bytes_read;*/
+    STREAMFILE * streamFilePos= NULL;
+
+    streamFile->get_name(streamFile,filename,sizeof(filename));
+
+    if (strlen(filename)+4 > sizeof(posname)) goto fail;
+
+    /* try to open a posfile using variations: "(name.ext).pos" */
+    {
+        strcpy(posname, filename);
+        strcat(posname, ".pos");
+        streamFilePos = streamFile->open(streamFile,posname,STREAMFILE_DEFAULT_BUFFER_SIZE);
+        if (streamFilePos) goto found;
+
+        goto fail;
+    }
+
+found:
+    //if (get_streamfile_size(streamFilePos) != bufsize) goto fail;
+
+    /* allow pos files to be of different sizes in case of new features, just fill all we can */
+    memset(buf, 0, bufsize);
+    read_streamfile(buf, 0, bufsize, streamFilePos);
+
+    close_streamfile(streamFilePos);
+
+    return 1;
+
+fail:
+    if (streamFilePos) close_streamfile(streamFilePos);
+
+    return 0;
+}

src/streamfile.h

@@ -151,5 +151,6 @@ size_t get_streamfile_dos_line(int dst_length, char * dst, off_t offset,
 
 int read_key_file(uint8_t * buf, size_t bufsize, STREAMFILE *streamFile);
 
+int read_pos_file(uint8_t * buf, size_t bufsize, STREAMFILE *streamFile);
 
 #endif

src/vgmstream.c

@@ -1072,7 +1072,7 @@ int get_vgmstream_samples_per_frame(VGMSTREAM * vgmstream) {
             ffmpeg_codec_data *data = (ffmpeg_codec_data *) vgmstream->codec_data;
             if (data) { 
 	            /* must know the full block size for edge loops */
-                return data->samplesPerBlock;
+                return data->sampleBufferBlock;
             }
             return 0;
         }

src/vgmstream.h

@@ -868,13 +868,16 @@ typedef struct {
     int bitsPerSample;
     int floatingPoint;
     int sampleRate;
-    int64_t totalFrames; // sample count, or 0 if unknown
     int bitrate;
+    // extra info: 0 if unknown or not fixed
+    int64_t totalSamples; // estimated count (may not be accurate for some demuxers)
+    int64_t blockAlign; // coded block of bytes, counting channels (the block can be joint stereo)
+    int64_t frameSize; // decoded samples per block
     
-    // Intermediate buffer
+    // Intermediate byte buffer
     uint8_t *sampleBuffer;
-    // max samples a block can held (can be less or more than samples per decoded frame)
+    // max samples we can held (can be less or more than frameSize)
-    size_t samplesPerBlock;
+    size_t sampleBufferBlock;
     
     // FFmpeg context used for metadata
     AVCodec *codec;

test/test.c

@@ -14,11 +14,9 @@
 #endif
 
 #ifndef VERSION
-#ifdef _MSC_VER
+/* To include the git version number / commit in test.exe, compile outside of Visual Studio and make sure git / sh is in the current PATH */
-// To include the git version number / commit in test.exe, compile outside of Visual Studio and make sure git is in the current PATH.
 #define VERSION ""
 #endif
-#endif
 
 #define BUFSIZE 4000