Cool_Tools/vgmstream
1
0
Fork 0
mirror of https://github.com/vgmstream/vgmstream.git synced 2024-11-28 00:20:47 +01:00
Code Issues Packages Projects Releases Wiki Activity

Optimize STDIO/FOO streamfiles slightly

Browse Source 
Tries to simplify code, and buffer is not emptied unless we really read
more data (useful for edge reads). get_offset is now reliable (though
never used).
This commit is contained in:
bnnm 2018-08-24 18:48:42 +02:00
parent 6c553882f3
commit b4884522d8
2 changed files with 134 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

232
fb2k/foo_streamfile.cpp
View File

@ -15,80 +15,73 @@ extern "C" {
} }
#include "foo_vgmstream.h" #include "foo_vgmstream.h"
/* On EOF reads we can return length 0, or ignore and return the requested length + 0-set the buffer.
* Some decoders don't check for EOF and may decode garbage if returned 0, as read_Nbit() funcs return -1.
* Only matters for metas that get num_samples wrong (bigger than total data). */
#define STREAMFILE_IGNORE_EOF 0
/* a STREAMFILE that operates via foobar's file service using a buffer */ /* a STREAMFILE that operates via foobar's file service using a buffer */
typedef struct _FOO_STREAMFILE { typedef struct {
struct _STREAMFILE sf; STREAMFILE sf; /* callbacks */
abort_callback * p_abort;
service_ptr_t<file> m_file; service_ptr_t<file> m_file; /* foobar IO service */
char * name; abort_callback * p_abort; /* foobar error stuff */
off_t offset; char * name; /* IO filename */
size_t validsize; off_t offset; /* last read offset (info) */
uint8_t * buffer; off_t buffer_offset; /* current buffer data start */
size_t buffersize; uint8_t * buffer; /* data buffer */
size_t filesize; size_t buffersize; /* max buffer size */
} FOO_STREAMFILE; size_t validsize; /* current buffer size */
size_t filesize; /* buffered file size */
} FOO_STREAMFILE;
static STREAMFILE * open_foo_streamfile_buffer_by_file(service_ptr_t<file> m_file,const char * const filename, size_t buffersize, abort_callback * p_abort);
static STREAMFILE * open_foo_streamfile_buffer(const char * const filename, size_t buffersize, abort_callback * p_abort, t_filestats * stats); static STREAMFILE * open_foo_streamfile_buffer(const char * const filename, size_t buffersize, abort_callback * p_abort, t_filestats * stats);
static STREAMFILE * open_foo_streamfile_buffer_by_file(service_ptr_t<file> m_file,const char * const filename, size_t buffersize, abort_callback * p_abort);
static size_t read_the_rest_foo(uint8_t * dest, off_t offset, size_t length, FOO_STREAMFILE * streamfile) { static size_t read_foo(FOO_STREAMFILE *streamfile, uint8_t * dest, off_t offset, size_t length) {
size_t length_read_total=0; size_t length_read_total = 0;
if (!streamfile || !dest || length <= 0 || offset < 0)
return 0;
/* is the part of the requested length in the buffer? */ /* is the part of the requested length in the buffer? */
if (offset >= streamfile->offset && offset < streamfile->offset + streamfile->validsize) { if (offset >= streamfile->buffer_offset && offset < streamfile->buffer_offset + streamfile->validsize) {
size_t length_read; size_t length_to_read;
off_t offset_into_buffer = offset - streamfile->offset; off_t offset_into_buffer = offset - streamfile->buffer_offset;
length_read = streamfile->validsize - offset_into_buffer;
memcpy(dest,streamfile->buffer + offset_into_buffer,length_read); length_to_read = streamfile->validsize - offset_into_buffer;
length_read_total += length_read; if (length_to_read > length)
length -= length_read; length_to_read = length;
offset += length_read;
dest += length_read; memcpy(dest,streamfile->buffer + offset_into_buffer,length_to_read);
length_read_total += length_to_read;
length -= length_to_read;
offset += length_to_read;
dest += length_to_read;
} }
/* What would make more sense here is to read the whole request
* at once into the dest buffer, as it must be large enough, and then
* copy some part of that into our own buffer.
* The destination buffer is supposed to be much smaller than the
* STREAMFILE buffer, though. Maybe we should only ever return up
* to the buffer size to avoid having to deal with things like this
* which are outside of my intended use.
*/
/* read the rest of the requested length */ /* read the rest of the requested length */
while (length > 0) { while (length > 0) {
size_t length_to_read; size_t length_to_read;
size_t length_read;
streamfile->validsize = 0; /* buffer is empty now */
/* request outside file: ignore to avoid seek/read */ /* ignore requests at EOF */
if (offset > streamfile->filesize) { if (offset >= streamfile->filesize) {
streamfile->offset = streamfile->filesize; //offset = streamfile->filesize; /* seems fseek doesn't clamp offset */
//VGM_ASSERT_ONCE(offset > streamfile->filesize, "STDIO: reading over filesize 0x%x @ 0x%lx + 0x%x\n", streamfile->filesize, offset, length);
#if STREAMFILE_IGNORE_EOF break;
memset(dest,0,length); /* dest is already shifted */
return length_read_total + length; /* partially-read + 0-set buffer */
#else
return length_read_total; /* partially-read buffer */
#endif
} }
/* position to new offset */ /* position to new offset */
try { try {
streamfile->m_file->seek(offset,*streamfile->p_abort); streamfile->m_file->seek(offset,*streamfile->p_abort);
//if (streamfile->m_file->is_eof(*streamfile->p_abort)) /* allow edge case of offset=filesize */
// return length_read;
} catch (...) { } catch (...) {
streamfile->offset = streamfile->filesize; break; /* this shouldn't happen in our code */
return 0; /* fail miserably (fseek shouldn't fail and reach this) */ }
/* fill the buffer (offset now is beyond buffer_offset) */
try {
streamfile->buffer_offset = offset;
streamfile->validsize = streamfile->m_file->read(streamfile->buffer,streamfile->buffersize,*streamfile->p_abort);
} catch(...) {
break; /* improbable? */
} }
streamfile->offset = offset;
/* decide how much must be read this time */ /* decide how much must be read this time */
if (length > streamfile->buffersize) if (length > streamfile->buffersize)
@ -96,70 +89,45 @@ static size_t read_the_rest_foo(uint8_t * dest, off_t offset, size_t length, FOO
else else
length_to_read = length; length_to_read = length;
/* fill the buffer */ /* give up on partial reads (EOF) */
try { if (streamfile->validsize < length_to_read) {
length_read = streamfile->m_file->read(streamfile->buffer,streamfile->buffersize,*streamfile->p_abort); memcpy(dest,streamfile->buffer,streamfile->validsize);
} catch(...) { offset += streamfile->validsize;
return 0; /* fail miserably */ length_read_total += streamfile->validsize;
} break;
streamfile->validsize = length_read;
/* if we can't get enough to satisfy the request (EOF) we give up */
if (length_read < length_to_read) {
memcpy(dest,streamfile->buffer,length_read);
#if STREAMFILE_IGNORE_EOF
memset(dest+length_read,0,length-length_read);
return length_read_total + length; /* partially-read + 0-set buffer */
#else
return length_read_total + length_read; /* partially-read buffer */
#endif
} }
/* use the new buffer */ /* use the new buffer */
memcpy(dest,streamfile->buffer,length_to_read); memcpy(dest,streamfile->buffer,length_to_read);
offset += length_to_read;
length_read_total += length_to_read; length_read_total += length_to_read;
length -= length_to_read; length -= length_to_read;
dest += length_to_read; dest += length_to_read;
offset += length_to_read;
} }
streamfile->offset = offset; /* last fread offset */
return length_read_total; return length_read_total;
} }
static size_t get_size_foo(FOO_STREAMFILE * streamfile) {
static size_t read_foo(FOO_STREAMFILE *streamfile, uint8_t * dest, off_t offset, size_t length) { return streamfile->filesize;
if (!streamfile || !dest || length<=0)
return 0;
/* request outside file: ignore to avoid seek/read in read_the_rest_foo() */
if (offset > streamfile->filesize) {
streamfile->offset = streamfile->filesize;
#if STREAMFILE_IGNORE_EOF
memset(dest,0,length);
return length; /* 0-set buffer */
#else
return 0; /* nothing to read */
#endif
}
/* just copy if entire request is within the buffer */
if (offset >= streamfile->offset && offset + length <= streamfile->offset + streamfile->validsize) {
off_t offset_into_buffer = offset - streamfile->offset;
memcpy(dest,streamfile->buffer + offset_into_buffer,length);
return length;
}
/* request outside buffer: new fread */
{
return read_the_rest_foo(dest,offset,length,streamfile);
}
} }
static off_t get_offset_foo(FOO_STREAMFILE *streamfile) {
STREAMFILE * open_foo_streamfile(const char * const filename, abort_callback * p_abort, t_filestats * stats) { return streamfile->offset;
return open_foo_streamfile_buffer(filename,STREAMFILE_DEFAULT_BUFFER_SIZE, p_abort, stats); }
static void get_name_foo(FOO_STREAMFILE *streamfile,char *buffer,size_t length) {
/* Most crap only cares about the filename itself */
size_t ourlen = strlen(streamfile->name);
if (ourlen > length) {
if (length) strcpy(buffer, streamfile->name + ourlen - length + 1);
} else {
strcpy(buffer, streamfile->name);
}
}
static void close_foo(FOO_STREAMFILE * streamfile) {
streamfile->m_file.release();
free(streamfile->name);
free(streamfile->buffer);
free(streamfile);
} }
static STREAMFILE *open_foo(FOO_STREAMFILE *streamFile,const char * const filename,size_t buffersize) { static STREAMFILE *open_foo(FOO_STREAMFILE *streamFile,const char * const filename,size_t buffersize) {
@ -173,7 +141,7 @@ static STREAMFILE *open_foo(FOO_STREAMFILE *streamFile,const char * const filena
// if same name, duplicate the file pointer we already have open // if same name, duplicate the file pointer we already have open
if (!strcmp(streamFile->name,filename)) { if (!strcmp(streamFile->name,filename)) {
m_file = streamFile->m_file; m_file = streamFile->m_file;
if (1) { {
newstreamFile = open_foo_streamfile_buffer_by_file(m_file,filename,buffersize,streamFile->p_abort); newstreamFile = open_foo_streamfile_buffer_by_file(m_file,filename,buffersize,streamFile->p_abort);
if (newstreamFile) { if (newstreamFile) {
return newstreamFile; return newstreamFile;
@ -186,45 +154,15 @@ static STREAMFILE *open_foo(FOO_STREAMFILE *streamFile,const char * const filena
return open_foo_streamfile_buffer(filename,buffersize,streamFile->p_abort,NULL); return open_foo_streamfile_buffer(filename,buffersize,streamFile->p_abort,NULL);
} }
static size_t get_size_foo(FOO_STREAMFILE * streamfile) {
return streamfile->filesize;
}
static off_t get_offset_foo(FOO_STREAMFILE *streamFile) {
return streamFile->offset;
}
static void close_foo(FOO_STREAMFILE * streamfile) {
streamfile->m_file.release();
free(streamfile->name);
free(streamfile->buffer);
free(streamfile);
}
static void get_name_foo(FOO_STREAMFILE *streamfile,char *buffer,size_t length) {
/* Most crap only cares about the filename itself */
size_t ourlen = strlen(streamfile->name);
if (ourlen > length) {
if (length) strcpy(buffer, streamfile->name + ourlen - length + 1);
} else {
strcpy(buffer, streamfile->name);
}
}
static STREAMFILE * open_foo_streamfile_buffer_by_file(service_ptr_t<file> m_file,const char * const filename, size_t buffersize, abort_callback * p_abort) { static STREAMFILE * open_foo_streamfile_buffer_by_file(service_ptr_t<file> m_file,const char * const filename, size_t buffersize, abort_callback * p_abort) {
uint8_t * buffer; uint8_t * buffer;
FOO_STREAMFILE * streamfile; FOO_STREAMFILE * streamfile;
buffer = (uint8_t *) calloc(buffersize,1); buffer = (uint8_t *) calloc(buffersize,1);
if (!buffer) { if (!buffer) goto fail;
return NULL;
}
streamfile = (FOO_STREAMFILE *) calloc(1,sizeof(FOO_STREAMFILE)); streamfile = (FOO_STREAMFILE *) calloc(1,sizeof(FOO_STREAMFILE));
if (!streamfile) { if (!streamfile) goto fail;
free(buffer);
return NULL;
}
streamfile->sf.read = (size_t (__cdecl *)(_STREAMFILE *,uint8_t *,off_t,size_t)) read_foo; streamfile->sf.read = (size_t (__cdecl *)(_STREAMFILE *,uint8_t *,off_t,size_t)) read_foo;
streamfile->sf.get_size = (size_t (__cdecl *)(_STREAMFILE *)) get_size_foo; streamfile->sf.get_size = (size_t (__cdecl *)(_STREAMFILE *)) get_size_foo;
@ -234,22 +172,22 @@ static STREAMFILE * open_foo_streamfile_buffer_by_file(service_ptr_t<file> m_fil
streamfile->sf.close = (void (__cdecl *)(_STREAMFILE *)) close_foo; streamfile->sf.close = (void (__cdecl *)(_STREAMFILE *)) close_foo;
streamfile->m_file = m_file; streamfile->m_file = m_file;
streamfile->p_abort = p_abort;
streamfile->buffersize = buffersize; streamfile->buffersize = buffersize;
streamfile->buffer = buffer; streamfile->buffer = buffer;
streamfile->p_abort = p_abort;
streamfile->name = strdup(filename); streamfile->name = strdup(filename);
if (!streamfile->name) { if (!streamfile->name) goto fail;
free(streamfile);
free(buffer);
return NULL;
}
/* cache filesize */ /* cache filesize */
streamfile->filesize = streamfile->m_file->get_size(*streamfile->p_abort); streamfile->filesize = streamfile->m_file->get_size(*streamfile->p_abort);
return &streamfile->sf; return &streamfile->sf;
fail:
free(buffer);
free(streamfile);
return NULL;
} }
static STREAMFILE * open_foo_streamfile_buffer(const char * const filename, size_t buffersize, abort_callback * p_abort, t_filestats * stats) { static STREAMFILE * open_foo_streamfile_buffer(const char * const filename, size_t buffersize, abort_callback * p_abort, t_filestats * stats) {
@ -269,3 +207,7 @@ static STREAMFILE * open_foo_streamfile_buffer(const char * const filename, size
return streamFile; return streamFile;
} }
STREAMFILE * open_foo_streamfile(const char * const filename, abort_callback * p_abort, t_filestats * stats) {
return open_foo_streamfile_buffer(filename,STREAMFILE_DEFAULT_BUFFER_SIZE, p_abort, stats);
}

106
src/streamfile.c
View File

@ -9,13 +9,15 @@
/* a STREAMFILE that operates via standard IO using a buffer */ /* a STREAMFILE that operates via standard IO using a buffer */
typedef struct { typedef struct {
STREAMFILE sf; /* callbacks */ STREAMFILE sf; /* callbacks */
FILE * infile; /* actual FILE */ FILE * infile; /* actual FILE */
char name[PATH_LIMIT]; char name[PATH_LIMIT]; /* FILE filename */
off_t offset; /* current offset */ off_t offset; /* last read offset (info) */
size_t validsize; /* current buffer size */ off_t buffer_offset; /* current buffer data start */
uint8_t * buffer; /* data buffer */ uint8_t * buffer; /* data buffer */
size_t buffersize; /* max buffer size */ size_t buffersize; /* max buffer size */
size_t filesize; /* cached file size (max offset) */ size_t validsize; /* current buffer size */
size_t filesize; /* buffered file size */
} STDIOSTREAMFILE; } STDIOSTREAMFILE;
static STREAMFILE * open_stdio_streamfile_buffer(const char * const filename, size_t buffersize); static STREAMFILE * open_stdio_streamfile_buffer(const char * const filename, size_t buffersize);
@ -28,9 +30,9 @@ static size_t read_stdio(STDIOSTREAMFILE *streamfile,uint8_t * dest, off_t offse
return 0; return 0;
/* is the part of the requested length in the buffer? */ /* is the part of the requested length in the buffer? */
if (offset >= streamfile->offset && offset < streamfile->offset + streamfile->validsize) { if (offset >= streamfile->buffer_offset && offset < streamfile->buffer_offset + streamfile->validsize) {
size_t length_to_read; size_t length_to_read;
off_t offset_into_buffer = offset - streamfile->offset; off_t offset_into_buffer = offset - streamfile->buffer_offset;
length_to_read = streamfile->validsize - offset_into_buffer; length_to_read = streamfile->validsize - offset_into_buffer;
if (length_to_read > length) if (length_to_read > length)
@ -43,32 +45,34 @@ static size_t read_stdio(STDIOSTREAMFILE *streamfile,uint8_t * dest, off_t offse
dest += length_to_read; dest += length_to_read;
} }
/* What would make more sense here is to read the whole request
* at once into the dest buffer, as it must be large enough, and then
* copy some part of that into our own buffer.
* The destination buffer is supposed to be much smaller than the
* STREAMFILE buffer, though. Maybe we should only ever return up
* to the buffer size to avoid having to deal with things like this
* which are outside of my intended use. */
/* read the rest of the requested length */ /* read the rest of the requested length */
while (length > 0) { while (length > 0) {
size_t length_to_read, length_read; size_t length_to_read;
streamfile->validsize = 0; /* buffer is empty now */
/* request outside file: ignore to avoid seek/read */ /* ignore requests at EOF */
if (offset > streamfile->filesize) { if (offset >= streamfile->filesize) {
streamfile->offset = streamfile->filesize; //offset = streamfile->filesize; /* seems fseek doesn't clamp offset */
VGM_LOG_ONCE("ERROR: reading over filesize 0x%x @ 0x%lx + 0x%x (buggy meta?)\n", streamfile->filesize, offset, length); VGM_ASSERT_ONCE(offset > streamfile->filesize, "STDIO: reading over filesize 0x%x @ 0x%lx + 0x%x\n", streamfile->filesize, offset, length);
return length_read_total; /* partially-read buffer */ break;
} }
/* position to new offset */ /* position to new offset */
if (fseeko(streamfile->infile,offset,SEEK_SET)) { if (fseeko(streamfile->infile,offset,SEEK_SET)) {
streamfile->offset = streamfile->filesize; break; /* this shouldn't happen in our code */
return 0; /* fail miserably (fseek shouldn't fail and reach this) */
} }
streamfile->offset = offset;
#ifdef _MSC_VER
/* Workaround a bug that appears when compiling with MSVC (later versions).
* This bug is deterministic and seemingly appears randomly after seeking.
* It results in fread returning data from the wrong area of the file.
* HPS is one format that is almost always affected by this. */
fseek(streamfile->infile, ftell(streamfile->infile), SEEK_SET);
#endif
/* fill the buffer (offset now is beyond buffer_offset) */
streamfile->buffer_offset = offset;
streamfile->validsize = fread(streamfile->buffer,sizeof(uint8_t),streamfile->buffersize,streamfile->infile);
/* decide how much must be read this time */ /* decide how much must be read this time */
if (length > streamfile->buffersize) if (length > streamfile->buffersize)
@ -76,57 +80,41 @@ static size_t read_stdio(STDIOSTREAMFILE *streamfile,uint8_t * dest, off_t offse
else else
length_to_read = length; length_to_read = length;
#ifdef _MSC_VER /* give up on partial reads (EOF) */
/* Workaround a bug that appears when compiling witn MSVC. if (streamfile->validsize < length_to_read) {
* This bug is dertiministic and seemingly appears randomly memcpy(dest,streamfile->buffer,streamfile->validsize);
* after seeking. offset += streamfile->validsize;
* It results in fread returning data from the wrong length_read_total += streamfile->validsize;
* area of the file. break;
* HPS is one format that is almost always affected
* by this. */
fseek(streamfile->infile, ftell(streamfile->infile), SEEK_SET);
#endif
/* fill the buffer */
length_read = fread(streamfile->buffer,sizeof(uint8_t),streamfile->buffersize,streamfile->infile);
streamfile->validsize = length_read;
/* if we can't get enough to satisfy the request (EOF) we give up */
if (length_read < length_to_read) {
memcpy(dest,streamfile->buffer,length_read);
return length_read_total + length_read; /* partially-read buffer */
} }
/* use the new buffer */ /* use the new buffer */
memcpy(dest,streamfile->buffer,length_to_read); memcpy(dest,streamfile->buffer,length_to_read);
offset += length_to_read;
length_read_total += length_to_read; length_read_total += length_to_read;
length -= length_to_read; length -= length_to_read;
dest += length_to_read; dest += length_to_read;
offset += length_to_read;
} }
streamfile->offset = offset; /* last fread offset */
return length_read_total; return length_read_total;
} }
static size_t get_size_stdio(STDIOSTREAMFILE * streamfile) {
return streamfile->filesize;
}
static off_t get_offset_stdio(STDIOSTREAMFILE *streamfile) {
return streamfile->offset;
}
static void get_name_stdio(STDIOSTREAMFILE *streamfile,char *buffer,size_t length) {
strncpy(buffer,streamfile->name,length);
buffer[length-1]='\0';
}
static void close_stdio(STDIOSTREAMFILE * streamfile) { static void close_stdio(STDIOSTREAMFILE * streamfile) {
fclose(streamfile->infile); fclose(streamfile->infile);
free(streamfile->buffer); free(streamfile->buffer);
free(streamfile); free(streamfile);
} }
static size_t get_size_stdio(STDIOSTREAMFILE * streamfile) {
return streamfile->filesize;
}
static off_t get_offset_stdio(STDIOSTREAMFILE *streamfile) {
return streamfile->offset;
}
static void get_name_stdio(STDIOSTREAMFILE *streamfile,char *buffer,size_t length) {
strncpy(buffer,streamfile->name,length);
buffer[length-1]='\0';
}
static STREAMFILE *open_stdio(STDIOSTREAMFILE *streamFile,const char * const filename,size_t buffersize) { static STREAMFILE *open_stdio(STDIOSTREAMFILE *streamFile,const char * const filename,size_t buffersize) {
int newfd; int newfd;
FILE *newfile; FILE *newfile;
@ -264,12 +252,12 @@ static size_t buffer_read(BUFFER_STREAMFILE *streamfile, uint8_t * dest, off_t o
/* ignore requests at EOF */ /* ignore requests at EOF */
if (offset >= streamfile->filesize) { if (offset >= streamfile->filesize) {
//offset = streamfile->filesize; /* seems fseek doesn't clamp offset */ //todo once //offset = streamfile->filesize; /* seems fseek doesn't clamp offset */
VGM_ASSERT_ONCE(offset > streamfile->filesize, "BUFFER: reading over filesize 0x%x @ 0x%lx + 0x%x\n", streamfile->filesize, offset, length); VGM_ASSERT_ONCE(offset > streamfile->filesize, "BUFFER: reading over filesize 0x%x @ 0x%lx + 0x%x\n", streamfile->filesize, offset, length);
break; break;
} }
/* fill the buffer (being here means offset is outside buffer thus empty) */ /* fill the buffer (offset now is beyond buffer_offset) */
streamfile->buffer_offset = offset; streamfile->buffer_offset = offset;
streamfile->validsize = streamfile->inner_sf->read(streamfile->inner_sf, streamfile->buffer, streamfile->buffer_offset, streamfile->buffersize); streamfile->validsize = streamfile->inner_sf->read(streamfile->inner_sf, streamfile->buffer, streamfile->buffer_offset, streamfile->buffersize);