Tweak STREAMFILES to read +2GB files

This commit is contained in:
bnnm 2021-09-04 21:57:23 +02:00
parent 19e84f44e5
commit 1d758db7d4
8 changed files with 147 additions and 108 deletions

View File

@ -12,13 +12,13 @@ extern "C" {
typedef struct { typedef struct {
STREAMFILE sf; STREAMFILE sf;
VFSFile *vfsFile; VFSFile *vfsFile;
off_t offset; offv_t offset;
char name[32768]; char name[32768];
} VFS_STREAMFILE; } VFS_STREAMFILE;
static STREAMFILE *open_vfs_by_VFSFILE(VFSFile *file, const char *path); static STREAMFILE *open_vfs_by_VFSFILE(VFSFile *file, const char *path);
static size_t read_vfs(VFS_STREAMFILE *streamfile, uint8_t *dest, off_t offset, size_t length) { static size_t read_vfs(VFS_STREAMFILE *streamfile, uint8_t *dest, offv_t offset, size_t length) {
size_t bytes_read; size_t bytes_read;
if (/*!streamfile->vfsFile ||*/ !dest || length <= 0 || offset < 0) if (/*!streamfile->vfsFile ||*/ !dest || length <= 0 || offset < 0)
@ -73,9 +73,9 @@ STREAMFILE *open_vfs_by_VFSFILE(VFSFile *file, const char *path) {
// success, set our pointers // success, set our pointers
memset(streamfile, 0, sizeof(VFS_STREAMFILE)); memset(streamfile, 0, sizeof(VFS_STREAMFILE));
streamfile->sf.read = (size_t (*)(STREAMFILE *, uint8_t *, off_t, size_t))read_vfs; streamfile->sf.read = (size_t (*)(STREAMFILE *, uint8_t *, offv_t, size_t))read_vfs;
streamfile->sf.get_size = (size_t (*)(STREAMFILE *))get_size_vfs; streamfile->sf.get_size = (size_t (*)(STREAMFILE *))get_size_vfs;
streamfile->sf.get_offset = (off_t (*)(STREAMFILE *))get_offset_vfs; streamfile->sf.get_offset = (offv_t (*)(STREAMFILE *))get_offset_vfs;
streamfile->sf.get_name = (void (*)(STREAMFILE *, char *, size_t))get_name_vfs; streamfile->sf.get_name = (void (*)(STREAMFILE *, char *, size_t))get_name_vfs;
streamfile->sf.open = (STREAMFILE *(*)(STREAMFILE *, const char *, size_t))open_vfs_impl; streamfile->sf.open = (STREAMFILE *(*)(STREAMFILE *, const char *, size_t))open_vfs_impl;
streamfile->sf.close = (void (*)(STREAMFILE *))close_vfs; streamfile->sf.close = (void (*)(STREAMFILE *))close_vfs;

View File

@ -24,8 +24,8 @@ typedef struct {
service_ptr_t<file> m_file; /* foobar IO service */ service_ptr_t<file> m_file; /* foobar IO service */
abort_callback * p_abort; /* foobar error stuff */ abort_callback * p_abort; /* foobar error stuff */
char * name; /* IO filename */ char * name; /* IO filename */
off_t offset; /* last read offset (info) */ offv_t offset; /* last read offset (info) */
off_t buffer_offset; /* current buffer data start */ offv_t buffer_offset; /* current buffer data start */
uint8_t * buffer; /* data buffer */ uint8_t * buffer; /* data buffer */
size_t buffersize; /* max buffer size */ size_t buffersize; /* max buffer size */
size_t validsize; /* current buffer size */ size_t validsize; /* current buffer size */
@ -35,7 +35,7 @@ typedef struct {
static STREAMFILE * open_foo_streamfile_buffer(const char * const filename, size_t buffersize, abort_callback * p_abort, t_filestats * stats); static STREAMFILE * open_foo_streamfile_buffer(const char * const filename, size_t buffersize, abort_callback * p_abort, t_filestats * stats);
static STREAMFILE * open_foo_streamfile_buffer_by_file(service_ptr_t<file> m_file, bool m_file_opened, const char * const filename, size_t buffersize, abort_callback * p_abort); static STREAMFILE * open_foo_streamfile_buffer_by_file(service_ptr_t<file> m_file, bool m_file_opened, const char * const filename, size_t buffersize, abort_callback * p_abort);
static size_t read_foo(FOO_STREAMFILE *streamfile, uint8_t * dest, off_t offset, size_t length) { static size_t read_foo(FOO_STREAMFILE *streamfile, uint8_t * dest, offv_t offset, size_t length) {
size_t length_read_total = 0; size_t length_read_total = 0;
if (!streamfile || !streamfile->m_file_opened || !dest || length <= 0 || offset < 0) if (!streamfile || !streamfile->m_file_opened || !dest || length <= 0 || offset < 0)
@ -44,7 +44,7 @@ static size_t read_foo(FOO_STREAMFILE *streamfile, uint8_t * dest, off_t offset,
/* is the part of the requested length in the buffer? */ /* is the part of the requested length in the buffer? */
if (offset >= streamfile->buffer_offset && offset < streamfile->buffer_offset + streamfile->validsize) { if (offset >= streamfile->buffer_offset && offset < streamfile->buffer_offset + streamfile->validsize) {
size_t length_to_read; size_t length_to_read;
off_t offset_into_buffer = offset - streamfile->buffer_offset; int offset_into_buffer = (offset - streamfile->buffer_offset);
length_to_read = streamfile->validsize - offset_into_buffer; length_to_read = streamfile->validsize - offset_into_buffer;
if (length_to_read > length) if (length_to_read > length)
@ -112,7 +112,7 @@ static size_t read_foo(FOO_STREAMFILE *streamfile, uint8_t * dest, off_t offset,
static size_t get_size_foo(FOO_STREAMFILE * streamfile) { static size_t get_size_foo(FOO_STREAMFILE * streamfile) {
return streamfile->filesize; return streamfile->filesize;
} }
static off_t get_offset_foo(FOO_STREAMFILE *streamfile) { static offv_t get_offset_foo(FOO_STREAMFILE *streamfile) {
return streamfile->offset; return streamfile->offset;
} }
static void get_name_foo(FOO_STREAMFILE *streamfile,char *buffer,size_t length) { static void get_name_foo(FOO_STREAMFILE *streamfile,char *buffer,size_t length) {
@ -165,9 +165,9 @@ static STREAMFILE * open_foo_streamfile_buffer_by_file(service_ptr_t<file> m_fil
streamfile = (FOO_STREAMFILE *) calloc(1,sizeof(FOO_STREAMFILE)); streamfile = (FOO_STREAMFILE *) calloc(1,sizeof(FOO_STREAMFILE));
if (!streamfile) goto fail; if (!streamfile) goto fail;
streamfile->sf.read = (size_t (__cdecl *)(_STREAMFILE *,uint8_t *,off_t,size_t)) read_foo; streamfile->sf.read = (size_t (__cdecl *)(_STREAMFILE *,uint8_t *,offv_t,size_t)) read_foo;
streamfile->sf.get_size = (size_t (__cdecl *)(_STREAMFILE *)) get_size_foo; streamfile->sf.get_size = (size_t (__cdecl *)(_STREAMFILE *)) get_size_foo;
streamfile->sf.get_offset = (off_t (__cdecl *)(_STREAMFILE *)) get_offset_foo; streamfile->sf.get_offset = (offv_t (__cdecl *)(_STREAMFILE *)) get_offset_foo;
streamfile->sf.get_name = (void (__cdecl *)(_STREAMFILE *,char *,size_t)) get_name_foo; streamfile->sf.get_name = (void (__cdecl *)(_STREAMFILE *,char *,size_t)) get_name_foo;
streamfile->sf.open = (_STREAMFILE *(__cdecl *)(_STREAMFILE *,const char *const ,size_t)) open_foo; streamfile->sf.open = (_STREAMFILE *(__cdecl *)(_STREAMFILE *,const char *const ,size_t)) open_foo;
streamfile->sf.close = (void (__cdecl *)(_STREAMFILE *)) close_foo; streamfile->sf.close = (void (__cdecl *)(_STREAMFILE *)) close_foo;

View File

@ -682,6 +682,6 @@ int mpc_get_samples(STREAMFILE* sf, off_t offset, int32_t* p_samples, int32_t* p
/* helper to pass a wrapped, clamped, fake extension-ed, SF to another meta */ /* helper to pass a wrapped, clamped, fake extension-ed, SF to another meta */
STREAMFILE* setup_subfile_streamfile(STREAMFILE* sf, off_t subfile_offset, size_t subfile_size, const char* extension); STREAMFILE* setup_subfile_streamfile(STREAMFILE* sf, offv_t subfile_offset, size_t subfile_size, const char* extension);
#endif /*_CODING_H*/ #endif /*_CODING_H*/

View File

@ -1092,7 +1092,7 @@ fail:
/* CUSTOM STREAMFILES */ /* CUSTOM STREAMFILES */
/* ******************************************** */ /* ******************************************** */
STREAMFILE* setup_subfile_streamfile(STREAMFILE* sf, off_t subfile_offset, size_t subfile_size, const char* extension) { STREAMFILE* setup_subfile_streamfile(STREAMFILE* sf, offv_t subfile_offset, size_t subfile_size, const char* extension) {
STREAMFILE* new_sf = NULL; STREAMFILE* new_sf = NULL;
new_sf = open_wrap_streamfile(sf); new_sf = open_wrap_streamfile(sf);

View File

@ -322,7 +322,7 @@ ffmpeg_codec_data* init_ffmpeg_header_offset_subsong(STREAMFILE* sf, uint8_t* he
goto fail; goto fail;
if (size == 0 || start + size > get_streamfile_size(sf)) { if (size == 0 || start + size > get_streamfile_size(sf)) {
vgm_logi(size != 0, "FFMPEG: wrong start+size found: %x + %x > %x \n", (uint32_t)start, (uint32_t)size, get_streamfile_size(sf)); vgm_asserti(size != 0, "FFMPEG: wrong start+size found: %x + %x > %x \n", (uint32_t)start, (uint32_t)size, get_streamfile_size(sf));
size = get_streamfile_size(sf) - start; size = get_streamfile_size(sf) - start;
} }

View File

@ -6,14 +6,50 @@
#include "vgmstream.h" #include "vgmstream.h"
/* MSVC fixes (though mingw uses MSVCRT but not MSC_VER, maybe use AND?) */
#if defined(__MSVCRT__) || defined(_MSC_VER)
#include <io.h>
#define fseek_v _fseeki64 //fseek
#define ftell_v _ftelli64 //ftell
/*
#ifndef fseeko
#define fseeko fseek
#endif
#ifndef ftello
#define ftello ftell
#endif
*/
#ifdef fileno
#undef fileno
#endif
#define fileno _fileno
#define fdopen _fdopen
#define dup _dup
#ifndef off64_t
#define off_t __int64
#endif
#elif defined(XBMC)
#define fseek_v fseek
#define ftell_v ftell
#else
#define fseek_v fseeko64 //fseeko
#define ftell_v ftello64 //ftelloo
#endif
/* a STREAMFILE that operates via standard IO using a buffer */ /* a STREAMFILE that operates via standard IO using a buffer */
typedef struct { typedef struct {
STREAMFILE sf; /* callbacks */ STREAMFILE sf; /* callbacks */
FILE * infile; /* actual FILE */ FILE * infile; /* actual FILE */
char name[PATH_LIMIT]; /* FILE filename */ char name[PATH_LIMIT]; /* FILE filename */
off_t offset; /* last read offset (info) */ offv_t offset; /* last read offset (info) */
off_t buffer_offset; /* current buffer data start */ offv_t buffer_offset; /* current buffer data start */
uint8_t* buffer; /* data buffer */ uint8_t* buffer; /* data buffer */
size_t buffersize; /* max buffer size */ size_t buffersize; /* max buffer size */
size_t validsize; /* current buffer size */ size_t validsize; /* current buffer size */
@ -23,7 +59,7 @@ typedef struct {
static STREAMFILE* open_stdio_streamfile_buffer(const char * const filename, size_t buffersize); static STREAMFILE* open_stdio_streamfile_buffer(const char * const filename, size_t buffersize);
static STREAMFILE* open_stdio_streamfile_buffer_by_file(FILE *infile, const char * const filename, size_t buffersize); static STREAMFILE* open_stdio_streamfile_buffer_by_file(FILE *infile, const char * const filename, size_t buffersize);
static size_t read_stdio(STDIO_STREAMFILE *streamfile, uint8_t *dst, off_t offset, size_t length) { static size_t read_stdio(STDIO_STREAMFILE *streamfile, uint8_t *dst, offv_t offset, size_t length) {
size_t length_read_total = 0; size_t length_read_total = 0;
if (!streamfile->infile || !dst || length <= 0 || offset < 0) if (!streamfile->infile || !dst || length <= 0 || offset < 0)
@ -34,7 +70,7 @@ static size_t read_stdio(STDIO_STREAMFILE *streamfile, uint8_t *dst, off_t offse
/* is the part of the requested length in the buffer? */ /* is the part of the requested length in the buffer? */
if (offset >= streamfile->buffer_offset && offset < streamfile->buffer_offset + streamfile->validsize) { if (offset >= streamfile->buffer_offset && offset < streamfile->buffer_offset + streamfile->validsize) {
size_t length_to_read; size_t length_to_read;
off_t offset_into_buffer = offset - streamfile->buffer_offset; int offset_into_buffer = (offset - streamfile->buffer_offset);
length_to_read = streamfile->validsize - offset_into_buffer; length_to_read = streamfile->validsize - offset_into_buffer;
if (length_to_read > length) if (length_to_read > length)
@ -69,7 +105,7 @@ static size_t read_stdio(STDIO_STREAMFILE *streamfile, uint8_t *dst, off_t offse
} }
/* position to new offset */ /* position to new offset */
if (fseeko(streamfile->infile,offset,SEEK_SET)) { if (fseek_v(streamfile->infile, offset, SEEK_SET)) {
break; /* this shouldn't happen in our code */ break; /* this shouldn't happen in our code */
} }
@ -78,8 +114,8 @@ static size_t read_stdio(STDIO_STREAMFILE *streamfile, uint8_t *dst, off_t offse
* This bug is deterministic and seemingly appears randomly after seeking. * This bug is deterministic and seemingly appears randomly after seeking.
* It results in fread returning data from the wrong area of the file. * It results in fread returning data from the wrong area of the file.
* HPS is one format that is almost always affected by this. * HPS is one format that is almost always affected by this.
* May be related/same as open_stdio's bug when using dup() */ * May be related/same as open_stdio's fixed bug when using dup(), try disabling */
fseek(streamfile->infile, ftell(streamfile->infile), SEEK_SET); fseek_v(streamfile->infile, ftell_v(streamfile->infile), SEEK_SET);
#endif #endif
/* fill the buffer (offset now is beyond buffer_offset) */ /* fill the buffer (offset now is beyond buffer_offset) */
@ -115,7 +151,7 @@ static size_t read_stdio(STDIO_STREAMFILE *streamfile, uint8_t *dst, off_t offse
static size_t get_size_stdio(STDIO_STREAMFILE *streamfile) { static size_t get_size_stdio(STDIO_STREAMFILE *streamfile) {
return streamfile->filesize; return streamfile->filesize;
} }
static off_t get_offset_stdio(STDIO_STREAMFILE *streamfile) { static offv_t get_offset_stdio(STDIO_STREAMFILE *streamfile) {
return streamfile->offset; return streamfile->offset;
} }
static void get_name_stdio(STDIO_STREAMFILE *streamfile, char *buffer, size_t length) { static void get_name_stdio(STDIO_STREAMFILE *streamfile, char *buffer, size_t length) {
@ -186,18 +222,18 @@ static STREAMFILE* open_stdio_streamfile_buffer_by_file(FILE *infile, const char
/* cache filesize */ /* cache filesize */
if (infile) { if (infile) {
fseeko(streamfile->infile,0,SEEK_END); fseek_v(streamfile->infile, 0x00, SEEK_END);
streamfile->filesize = ftello(streamfile->infile); streamfile->filesize = ftell_v(streamfile->infile);
fseek_v(streamfile->infile, 0x00, SEEK_SET);
} }
else { else {
streamfile->filesize = 0; /* allow virtual, non-existing files */ streamfile->filesize = 0; /* allow virtual, non-existing files */
} }
/* Typically fseek(o)/ftell(o) may only handle up to ~2.14GB, signed 32b = 0x7FFFFFFF /* Typically fseek(o)/ftell(o) may only handle up to ~2.14GB, signed 32b = 0x7FFFFFFF
* (happens in banks like FSB, though rarely). Can be remedied with the * (happens in banks like FSB, though rarely). Should work if configured properly, log otherwise. */
* preprocessor (-D_FILE_OFFSET_BITS=64 in GCC) but it's not well tested. */
if (streamfile->filesize == 0xFFFFFFFF) { /* -1 on error */ if (streamfile->filesize == 0xFFFFFFFF) { /* -1 on error */
VGM_LOG("STREAMFILE: ftell error\n"); vgm_logi("STREAMFILE: file size too big (report)\n");
goto fail; /* can be ignored but may result in strange/unexpected behaviors */ goto fail; /* can be ignored but may result in strange/unexpected behaviors */
} }
@ -242,8 +278,8 @@ typedef struct {
STREAMFILE sf; STREAMFILE sf;
STREAMFILE *inner_sf; STREAMFILE *inner_sf;
off_t offset; /* last read offset (info) */ offv_t offset; /* last read offset (info) */
off_t buffer_offset; /* current buffer data start */ offv_t buffer_offset; /* current buffer data start */
uint8_t* buffer; /* data buffer */ uint8_t* buffer; /* data buffer */
size_t buffersize; /* max buffer size */ size_t buffersize; /* max buffer size */
size_t validsize; /* current buffer size */ size_t validsize; /* current buffer size */
@ -251,7 +287,7 @@ typedef struct {
} BUFFER_STREAMFILE; } BUFFER_STREAMFILE;
static size_t buffer_read(BUFFER_STREAMFILE *streamfile, uint8_t *dst, off_t offset, size_t length) { static size_t buffer_read(BUFFER_STREAMFILE *streamfile, uint8_t *dst, offv_t offset, size_t length) {
size_t length_read_total = 0; size_t length_read_total = 0;
if (!dst || length <= 0 || offset < 0) if (!dst || length <= 0 || offset < 0)
@ -260,7 +296,7 @@ static size_t buffer_read(BUFFER_STREAMFILE *streamfile, uint8_t *dst, off_t off
/* is the part of the requested length in the buffer? */ /* is the part of the requested length in the buffer? */
if (offset >= streamfile->buffer_offset && offset < streamfile->buffer_offset + streamfile->validsize) { if (offset >= streamfile->buffer_offset && offset < streamfile->buffer_offset + streamfile->validsize) {
size_t length_to_read; size_t length_to_read;
off_t offset_into_buffer = offset - streamfile->buffer_offset; int offset_into_buffer = (offset - streamfile->buffer_offset);
length_to_read = streamfile->validsize - offset_into_buffer; length_to_read = streamfile->validsize - offset_into_buffer;
if (length_to_read > length) if (length_to_read > length)
@ -322,7 +358,7 @@ static size_t buffer_read(BUFFER_STREAMFILE *streamfile, uint8_t *dst, off_t off
static size_t buffer_get_size(BUFFER_STREAMFILE *streamfile) { static size_t buffer_get_size(BUFFER_STREAMFILE *streamfile) {
return streamfile->filesize; /* cache */ return streamfile->filesize; /* cache */
} }
static size_t buffer_get_offset(BUFFER_STREAMFILE *streamfile) { static offv_t buffer_get_offset(BUFFER_STREAMFILE *streamfile) {
return streamfile->offset; /* cache */ return streamfile->offset; /* cache */
} }
static void buffer_get_name(BUFFER_STREAMFILE *streamfile, char *buffer, size_t length) { static void buffer_get_name(BUFFER_STREAMFILE *streamfile, char *buffer, size_t length) {
@ -392,13 +428,13 @@ typedef struct {
STREAMFILE *inner_sf; STREAMFILE *inner_sf;
} WRAP_STREAMFILE; } WRAP_STREAMFILE;
static size_t wrap_read(WRAP_STREAMFILE *streamfile, uint8_t *dst, off_t offset, size_t length) { static size_t wrap_read(WRAP_STREAMFILE *streamfile, uint8_t *dst, offv_t offset, size_t length) {
return streamfile->inner_sf->read(streamfile->inner_sf, dst, offset, length); /* default */ return streamfile->inner_sf->read(streamfile->inner_sf, dst, offset, length); /* default */
} }
static size_t wrap_get_size(WRAP_STREAMFILE *streamfile) { static size_t wrap_get_size(WRAP_STREAMFILE *streamfile) {
return streamfile->inner_sf->get_size(streamfile->inner_sf); /* default */ return streamfile->inner_sf->get_size(streamfile->inner_sf); /* default */
} }
static size_t wrap_get_offset(WRAP_STREAMFILE *streamfile) { static offv_t wrap_get_offset(WRAP_STREAMFILE *streamfile) {
return streamfile->inner_sf->get_offset(streamfile->inner_sf); /* default */ return streamfile->inner_sf->get_offset(streamfile->inner_sf); /* default */
} }
static void wrap_get_name(WRAP_STREAMFILE *streamfile, char *buffer, size_t length) { static void wrap_get_name(WRAP_STREAMFILE *streamfile, char *buffer, size_t length) {
@ -446,12 +482,12 @@ typedef struct {
STREAMFILE sf; STREAMFILE sf;
STREAMFILE* inner_sf; STREAMFILE* inner_sf;
off_t start; offv_t start;
size_t size; size_t size;
} CLAMP_STREAMFILE; } CLAMP_STREAMFILE;
static size_t clamp_read(CLAMP_STREAMFILE* streamfile, uint8_t* dst, off_t offset, size_t length) { static size_t clamp_read(CLAMP_STREAMFILE* streamfile, uint8_t* dst, offv_t offset, size_t length) {
off_t inner_offset = streamfile->start + offset; offv_t inner_offset = streamfile->start + offset;
size_t clamp_length = length; size_t clamp_length = length;
if (offset + length > streamfile->size) { if (offset + length > streamfile->size) {
@ -466,7 +502,7 @@ static size_t clamp_read(CLAMP_STREAMFILE* streamfile, uint8_t* dst, off_t offse
static size_t clamp_get_size(CLAMP_STREAMFILE* streamfile) { static size_t clamp_get_size(CLAMP_STREAMFILE* streamfile) {
return streamfile->size; return streamfile->size;
} }
static off_t clamp_get_offset(CLAMP_STREAMFILE* streamfile) { static offv_t clamp_get_offset(CLAMP_STREAMFILE* streamfile) {
return streamfile->inner_sf->get_offset(streamfile->inner_sf) - streamfile->start; return streamfile->inner_sf->get_offset(streamfile->inner_sf) - streamfile->start;
} }
static void clamp_get_name(CLAMP_STREAMFILE* streamfile, char* buffer, size_t length) { static void clamp_get_name(CLAMP_STREAMFILE* streamfile, char* buffer, size_t length) {
@ -491,7 +527,7 @@ static void clamp_close(CLAMP_STREAMFILE* streamfile) {
free(streamfile); free(streamfile);
} }
STREAMFILE* open_clamp_streamfile(STREAMFILE* streamfile, off_t start, size_t size) { STREAMFILE* open_clamp_streamfile(STREAMFILE* streamfile, offv_t start, size_t size) {
CLAMP_STREAMFILE* this_sf = NULL; CLAMP_STREAMFILE* this_sf = NULL;
if (!streamfile || size == 0) return NULL; if (!streamfile || size == 0) return NULL;
@ -515,7 +551,7 @@ STREAMFILE* open_clamp_streamfile(STREAMFILE* streamfile, off_t start, size_t si
return &this_sf->sf; return &this_sf->sf;
} }
STREAMFILE* open_clamp_streamfile_f(STREAMFILE* streamfile, off_t start, size_t size) { STREAMFILE* open_clamp_streamfile_f(STREAMFILE* streamfile, offv_t start, size_t size) {
STREAMFILE* new_sf = open_clamp_streamfile(streamfile, start, size); STREAMFILE* new_sf = open_clamp_streamfile(streamfile, start, size);
if (!new_sf) if (!new_sf)
close_streamfile(streamfile); close_streamfile(streamfile);
@ -530,13 +566,13 @@ typedef struct {
STREAMFILE *inner_sf; STREAMFILE *inner_sf;
void* data; /* state for custom reads, malloc'ed + copied on open (to re-open streamfiles cleanly) */ void* data; /* state for custom reads, malloc'ed + copied on open (to re-open streamfiles cleanly) */
size_t data_size; size_t data_size;
size_t (*read_callback)(STREAMFILE *, uint8_t *, off_t, size_t, void*); /* custom read to modify data before copying into buffer */ size_t (*read_callback)(STREAMFILE *, uint8_t *, offv_t, size_t, void*); /* custom read to modify data before copying into buffer */
size_t (*size_callback)(STREAMFILE *, void*); /* size when custom reads make data smaller/bigger than underlying streamfile */ size_t (*size_callback)(STREAMFILE *, void*); /* size when custom reads make data smaller/bigger than underlying streamfile */
int (*init_callback)(STREAMFILE*, void*); /* init the data struct members somehow, return >= 0 if ok */ int (*init_callback)(STREAMFILE*, void*); /* init the data struct members somehow, return >= 0 if ok */
void (*close_callback)(STREAMFILE*, void*); /* close the data struct members somehow */ void (*close_callback)(STREAMFILE*, void*); /* close the data struct members somehow */
} IO_STREAMFILE; } IO_STREAMFILE;
static size_t io_read(IO_STREAMFILE *streamfile, uint8_t *dst, off_t offset, size_t length) { static size_t io_read(IO_STREAMFILE *streamfile, uint8_t *dst, offv_t offset, size_t length) {
return streamfile->read_callback(streamfile->inner_sf, dst, offset, length, streamfile->data); return streamfile->read_callback(streamfile->inner_sf, dst, offset, length, streamfile->data);
} }
static size_t io_get_size(IO_STREAMFILE *streamfile) { static size_t io_get_size(IO_STREAMFILE *streamfile) {
@ -545,7 +581,7 @@ static size_t io_get_size(IO_STREAMFILE *streamfile) {
else else
return streamfile->inner_sf->get_size(streamfile->inner_sf); /* default */ return streamfile->inner_sf->get_size(streamfile->inner_sf); /* default */
} }
static off_t io_get_offset(IO_STREAMFILE *streamfile) { static offv_t io_get_offset(IO_STREAMFILE *streamfile) {
return streamfile->inner_sf->get_offset(streamfile->inner_sf); /* default */ return streamfile->inner_sf->get_offset(streamfile->inner_sf); /* default */
} }
static void io_get_name(IO_STREAMFILE *streamfile, char *buffer, size_t length) { static void io_get_name(IO_STREAMFILE *streamfile, char *buffer, size_t length) {
@ -629,13 +665,13 @@ typedef struct {
char fakename[PATH_LIMIT]; char fakename[PATH_LIMIT];
} FAKENAME_STREAMFILE; } FAKENAME_STREAMFILE;
static size_t fakename_read(FAKENAME_STREAMFILE *streamfile, uint8_t *dst, off_t offset, size_t length) { static size_t fakename_read(FAKENAME_STREAMFILE *streamfile, uint8_t *dst, offv_t offset, size_t length) {
return streamfile->inner_sf->read(streamfile->inner_sf, dst, offset, length); /* default */ return streamfile->inner_sf->read(streamfile->inner_sf, dst, offset, length); /* default */
} }
static size_t fakename_get_size(FAKENAME_STREAMFILE *streamfile) { static size_t fakename_get_size(FAKENAME_STREAMFILE *streamfile) {
return streamfile->inner_sf->get_size(streamfile->inner_sf); /* default */ return streamfile->inner_sf->get_size(streamfile->inner_sf); /* default */
} }
static size_t fakename_get_offset(FAKENAME_STREAMFILE *streamfile) { static offv_t fakename_get_offset(FAKENAME_STREAMFILE *streamfile) {
return streamfile->inner_sf->get_offset(streamfile->inner_sf); /* default */ return streamfile->inner_sf->get_offset(streamfile->inner_sf); /* default */
} }
static void fakename_get_name(FAKENAME_STREAMFILE *streamfile, char *buffer, size_t length) { static void fakename_get_name(FAKENAME_STREAMFILE *streamfile, char *buffer, size_t length) {
@ -714,13 +750,13 @@ typedef struct {
STREAMFILE **inner_sfs; STREAMFILE **inner_sfs;
size_t inner_sfs_size; size_t inner_sfs_size;
size_t *sizes; size_t *sizes;
off_t size; offv_t size;
off_t offset; offv_t offset;
} MULTIFILE_STREAMFILE; } MULTIFILE_STREAMFILE;
static size_t multifile_read(MULTIFILE_STREAMFILE *streamfile, uint8_t *dst, off_t offset, size_t length) { static size_t multifile_read(MULTIFILE_STREAMFILE *streamfile, uint8_t *dst, offv_t offset, size_t length) {
int i, segment = 0; int i, segment = 0;
off_t segment_offset = 0; offv_t segment_offset = 0;
size_t done = 0; size_t done = 0;
if (offset > streamfile->size) { if (offset > streamfile->size) {
@ -757,7 +793,7 @@ static size_t multifile_read(MULTIFILE_STREAMFILE *streamfile, uint8_t *dst, off
static size_t multifile_get_size(MULTIFILE_STREAMFILE *streamfile) { static size_t multifile_get_size(MULTIFILE_STREAMFILE *streamfile) {
return streamfile->size; return streamfile->size;
} }
static size_t multifile_get_offset(MULTIFILE_STREAMFILE * streamfile) { static offv_t multifile_get_offset(MULTIFILE_STREAMFILE * streamfile) {
return streamfile->offset; return streamfile->offset;
} }
static void multifile_get_name(MULTIFILE_STREAMFILE *streamfile, char *buffer, size_t length) { static void multifile_get_name(MULTIFILE_STREAMFILE *streamfile, char *buffer, size_t length) {
@ -1410,15 +1446,15 @@ void get_streamfile_ext(STREAMFILE* sf, char* buffer, size_t size) {
/* debug util, mainly for custom IO testing */ /* debug util, mainly for custom IO testing */
void dump_streamfile(STREAMFILE* sf, int num) { void dump_streamfile(STREAMFILE* sf, int num) {
#ifdef VGM_DEBUG_OUTPUT #ifdef VGM_DEBUG_OUTPUT
off_t offset = 0; offv_t offset = 0;
FILE* f = NULL; FILE* f = NULL;
if (num >= 0) { if (num >= 0) {
char filename[PATH_LIMIT]; char filename[PATH_LIMIT];
char dumpname[PATH_LIMIT]; char dumpname[PATH_LIMIT];
get_streamfile_filename(sf, filename, PATH_LIMIT); get_streamfile_filename(sf, filename, sizeof(filename));
snprintf(dumpname,PATH_LIMIT, "%s_%02i.dump", filename, num); snprintf(dumpname, sizeof(dumpname), "%s_%02i.dump", filename, num);
f = fopen(dumpname,"wb"); f = fopen(dumpname,"wb");
if (!f) return; if (!f) return;
@ -1426,20 +1462,20 @@ void dump_streamfile(STREAMFILE* sf, int num) {
VGM_LOG("dump streamfile: size %x\n", get_streamfile_size(sf)); VGM_LOG("dump streamfile: size %x\n", get_streamfile_size(sf));
while (offset < get_streamfile_size(sf)) { while (offset < get_streamfile_size(sf)) {
uint8_t buffer[0x8000]; uint8_t buf[0x8000];
size_t read; size_t bytes;
read = read_streamfile(buffer,offset,0x8000,sf); bytes = read_streamfile(buf, offset, sizeof(buf), sf);
if(!read) { if(!bytes) {
VGM_LOG("dump streamfile: can't read at %lx\n", offset); VGM_LOG("dump streamfile: can't read at %lx\n", offset);
break; break;
} }
if (f) if (f)
fwrite(buffer,sizeof(uint8_t),read, f); fwrite(buf, sizeof(uint8_t), bytes, f);
else else
VGM_LOGB(buffer,read,0); VGM_LOGB(buf, bytes, 0);
offset += read; offset += bytes;
} }
if (f) { if (f) {

View File

@ -21,28 +21,9 @@
#if defined(__MSVCRT__) || defined(_MSC_VER) #if defined(__MSVCRT__) || defined(_MSC_VER)
#include <io.h> #include <io.h>
#ifndef fseeko #ifndef off64_t
#define fseeko fseek #define off_t __int64
#endif #endif
#ifndef ftello
#define ftello ftell
#endif
#define dup _dup
#ifdef fileno
#undef fileno
#endif
#define fileno _fileno
#define fdopen _fdopen
// #ifndef off64_t
// #define off_t __int64
// #endif
#endif
#if defined(XBMC)
#define fseeko fseek
#endif #endif
#ifndef DIR_SEPARATOR #ifndef DIR_SEPARATOR
@ -53,6 +34,19 @@
#endif #endif
#endif #endif
/* 64-bit offset is needed for banks that hit +2.5GB (like .fsb or .ktsl2stbin).
* Leave as typedef to toggle since it's theoretically slower when compiled as 32-bit.
* ATM it's only used in choice places until more performance tests are done.
* uint32_t could be an option but needs to test when/how neg offsets are used.
*
* On POSIX 32-bit off_t can become off64_t by passing -D_FILE_OFFSET_BITS=64,
* but not on MSVC as it doesn't have proper POSIX support, so a custom type is needed.
* fseeks/tells also need to be adjusted for 64-bit support.
*/
typedef int64_t offv_t; //off64_t
//typedef int64_t sizev_t; // size_t int64_t off64_t
/* Streamfiles normally use an internal buffer to increase performance, configurable /* Streamfiles normally use an internal buffer to increase performance, configurable
* but usually of this size. Lower increases the number of freads/system calls (slower). * but usually of this size. Lower increases the number of freads/system calls (slower).
* However some formats need to jump around causing more buffer trashing than usual, * However some formats need to jump around causing more buffer trashing than usual,
@ -65,17 +59,26 @@
* to do file operations, as plugins may need to provide their own callbacks. * to do file operations, as plugins may need to provide their own callbacks.
* Reads from arbitrary offsets, meaning internally may need fseek equivalents during reads. */ * Reads from arbitrary offsets, meaning internally may need fseek equivalents during reads. */
typedef struct _STREAMFILE { typedef struct _STREAMFILE {
size_t (*read)(struct _STREAMFILE*, uint8_t* dst, off_t offset, size_t length); /* read 'length' data at 'offset' to 'dst' */
size_t (*get_size)(struct _STREAMFILE*); size_t (*read)(struct _STREAMFILE* sf, uint8_t* dst, offv_t offset, size_t length);
off_t (*get_offset)(struct _STREAMFILE*); //todo: DO NOT USE, NOT RESET PROPERLY (remove?)
/* for dual-file support */ /* get max offset */
void (*get_name)(struct _STREAMFILE*, char* name, size_t length); size_t (*get_size)(struct _STREAMFILE* sf);
struct _STREAMFILE* (*open)(struct _STREAMFILE*, const char* const filename, size_t buffersize);
//todo: DO NOT USE, NOT RESET PROPERLY (remove?)
offv_t (*get_offset)(struct _STREAMFILE*);
/* copy current filename to name buf */
void (*get_name)(struct _STREAMFILE* sf, char* name, size_t name_size);
/* open another streamfile from filename */
struct _STREAMFILE* (*open)(struct _STREAMFILE* sf, const char* const filename, size_t buffer_size);
/* free current STREAMFILE */
void (*close)(struct _STREAMFILE*); void (*close)(struct _STREAMFILE*);
/* Substream selection for formats with subsongs.
/* Substream selection for files with subsongs. Manually used in metas if supported. * Not ideal here, but it was the simplest way to pass to all init_vgmstream_x functions. */
* Not ideal here, but it's the simplest way to pass to all init_vgmstream_x functions. */
int stream_index; /* 0=default/auto (first), 1=first, N=Nth */ int stream_index; /* 0=default/auto (first), 1=first, N=Nth */
} STREAMFILE; } STREAMFILE;
@ -105,8 +108,8 @@ STREAMFILE* open_wrap_streamfile_f(STREAMFILE* sf);
/* Opens a STREAMFILE that clamps reads to a section of a larger streamfile. /* Opens a STREAMFILE that clamps reads to a section of a larger streamfile.
* Can be used with subfiles inside a bigger file (to fool metas, or to simplify custom IO). */ * Can be used with subfiles inside a bigger file (to fool metas, or to simplify custom IO). */
STREAMFILE* open_clamp_streamfile(STREAMFILE* sf, off_t start, size_t size); STREAMFILE* open_clamp_streamfile(STREAMFILE* sf, offv_t start, size_t size);
STREAMFILE* open_clamp_streamfile_f(STREAMFILE* sf, off_t start, size_t size); STREAMFILE* open_clamp_streamfile_f(STREAMFILE* sf, offv_t start, size_t size);
/* Opens a STREAMFILE that uses custom IO for streamfile reads. /* Opens a STREAMFILE that uses custom IO for streamfile reads.
* Can be used to modify data on the fly (ex. decryption), or even transform it from a format to another. * Can be used to modify data on the fly (ex. decryption), or even transform it from a format to another.
@ -156,7 +159,7 @@ static inline void close_streamfile(STREAMFILE* sf) {
} }
/* read from a file, returns number of bytes read */ /* read from a file, returns number of bytes read */
static inline size_t read_streamfile(uint8_t *dst, off_t offset, size_t length, STREAMFILE* sf) { static inline size_t read_streamfile(uint8_t* dst, offv_t offset, size_t length, STREAMFILE* sf) {
return sf->read(sf, dst, offset, length); return sf->read(sf, dst, offset, length);
} }

View File

@ -42,15 +42,15 @@ typedef struct {
static STREAMFILE *open_winamp_streamfile_by_file(FILE *infile, const char * path); static STREAMFILE *open_winamp_streamfile_by_file(FILE *infile, const char * path);
//static STREAMFILE *open_winamp_streamfile_by_ipath(const in_char *wpath); //static STREAMFILE *open_winamp_streamfile_by_ipath(const in_char *wpath);
static size_t wasf_read(WINAMP_STREAMFILE* sf, uint8_t* dest, off_t offset, size_t length) { static size_t wasf_read(WINAMP_STREAMFILE* sf, uint8_t* dest, offv_t offset, size_t length) {
return sf->stdiosf->read(sf->stdiosf, dest, offset, length); return sf->stdiosf->read(sf->stdiosf, dest, offset, length);
} }
static off_t wasf_get_size(WINAMP_STREAMFILE* sf) { static size_t wasf_get_size(WINAMP_STREAMFILE* sf) {
return sf->stdiosf->get_size(sf->stdiosf); return sf->stdiosf->get_size(sf->stdiosf);
} }
static off_t wasf_get_offset(WINAMP_STREAMFILE* sf) { static offv_t wasf_get_offset(WINAMP_STREAMFILE* sf) {
return sf->stdiosf->get_offset(sf->stdiosf); return sf->stdiosf->get_offset(sf->stdiosf);
} }