From d644d2f9f77302bcd6ea1336837893e5cbb58e49 Mon Sep 17 00:00:00 2001 From: bnnm Date: Sun, 17 Sep 2017 03:41:36 +0200 Subject: [PATCH] Add HCA key detection and key list Updated the meta a bit so it's closer to others and optimized some mallocs, as it was a bit hard to understand for me so I hope it's for the better --- src/libvgmstream.vcproj | 4 + src/libvgmstream.vcxproj | 1 + src/libvgmstream.vcxproj.filters | 3 + src/meta/hca.c | 217 +++++++++++++++++++------------ src/meta/hca_keys.h | 106 +++++++++++++++ src/vgmstream.h | 7 +- 6 files changed, 254 insertions(+), 84 deletions(-) create mode 100644 src/meta/hca_keys.h diff --git a/src/libvgmstream.vcproj b/src/libvgmstream.vcproj index e10e1534..f908ca2e 100644 --- a/src/libvgmstream.vcproj +++ b/src/libvgmstream.vcproj @@ -200,6 +200,10 @@ RelativePath=".\meta\meta.h" > + + + diff --git a/src/libvgmstream.vcxproj.filters b/src/libvgmstream.vcxproj.filters index e2ad59cd..d2673803 100644 --- a/src/libvgmstream.vcxproj.filters +++ b/src/libvgmstream.vcxproj.filters @@ -65,6 +65,9 @@ meta\Header Files + + meta\Header Files + coding\Header Files diff --git a/src/meta/hca.c b/src/meta/hca.c index 8a16e7b4..7417c225 100644 --- a/src/meta/hca.c +++ b/src/meta/hca.c @@ -1,116 +1,171 @@ -#include "../vgmstream.h" #include "meta.h" -#include "../util.h" +#include "hca_keys.h" +#include "../coding/coding.h" -static VGMSTREAM * init_vgmstream_hca_offset(STREAMFILE *streamFile, uint64_t start, uint64_t size); +#define HCA_KEY_MAX_TEST_CLIPS 400 /* hopefully nobody masters files with more that a handful... */ +#define HCA_KEY_MAX_TEST_FRAMES 100 /* ~102400 samples */ +#define HCA_KEY_MAX_TEST_SAMPLES 10240 /* ~10 frames of non-blank samples */ + +static void find_hca_key(hca_codec_data * hca_data, clHCA * hca, uint8_t * buffer, int header_size, unsigned int * out_key1, unsigned int * out_key2); VGMSTREAM * init_vgmstream_hca(STREAMFILE *streamFile) { + VGMSTREAM * vgmstream = NULL; + uint8_t buffer[0x8000]; /* hca header buffer data (probably max ~0x400) */ + char filename[PATH_LIMIT]; + off_t start = 0; + size_t file_size = streamFile->get_size(streamFile); + + int header_size; + hca_codec_data * hca_data = NULL; /* vgmstream HCA context */ + clHCA * hca; /* HCA_Decoder context */ + unsigned int ciphKey1, ciphKey2; /* check extension, case insensitive */ if ( !check_extensions(streamFile, "hca")) return NULL; - return init_vgmstream_hca_offset( streamFile, 0, streamFile->get_size(streamFile) ); -} -static VGMSTREAM * init_vgmstream_hca_offset(STREAMFILE *streamFile, uint64_t start, uint64_t size) { - unsigned int ciphKey1; - unsigned int ciphKey2; + /* test/init header (find real header size first) */ + if ( file_size < 8 ) goto fail; + if ( read_streamfile(buffer, start, 8, streamFile) != 8 ) goto fail; - char filename[PATH_LIMIT]; - - VGMSTREAM * vgmstream = NULL; + header_size = clHCA_isOurFile0(buffer); + if ( header_size < 0 || header_size > 0x8000 ) goto fail; - hca_codec_data * hca_file = ( hca_codec_data * ) calloc(1, sizeof(hca_codec_data) + clHCA_sizeof()); - void * hca_data = NULL; - clHCA * hca; + if ( read_streamfile(buffer, start, header_size, streamFile) != header_size ) goto fail; + if ( clHCA_isOurFile1(buffer, header_size) < 0 ) goto fail; - uint8_t header[8]; - int header_size; + /* init vgmstream context */ + hca_data = (hca_codec_data *) calloc(1, sizeof(hca_codec_data) + clHCA_sizeof()); + if (!hca_data) goto fail; + //hca_data->size = file_size; + hca_data->start = 0; + hca_data->sample_ptr = clHCA_samplesPerBlock; - if ( !hca_file ) goto fail; + /* HCA_Decoder context memory goes right after our codec data (reserved in alloc'ed) */ + hca = (clHCA *)(hca_data + 1); - if ( size < 8 ) goto fail; + /* pre-load streamfile so the hca_data is ready before key detection */ + streamFile->get_name( streamFile, filename, sizeof(filename) ); + hca_data->streamfile = streamFile->open(streamFile, filename, STREAMFILE_DEFAULT_BUFFER_SIZE); + if (!hca_data->streamfile) goto fail; - hca_file->streamfile = streamFile; - hca_file->start = start; - hca_file->size = size; - - if ( read_streamfile( header, start, 8, streamFile) != 8 ) goto fail; - header_size = clHCA_isOurFile0( header ); - - if ( header_size < 0 ) goto fail; - - hca_data = malloc( header_size ); - - if ( !hca_data ) goto fail; - - memcpy( hca_data, header, 8 ); - - if ( read_streamfile( ((uint8_t*)hca_data) + 8, start + 8, header_size - 8, streamFile ) != header_size - 8 ) goto fail; - - if ( clHCA_isOurFile1( hca_data, header_size ) < 0 ) goto fail; - - hca = (clHCA *)(hca_file + 1); - - /* try to find key in external file */ + /* find decryption key in external file or preloaded list */ { uint8_t keybuf[8]; - if ( read_key_file(keybuf, 8, streamFile) ) { ciphKey2 = get_32bitBE(keybuf+0); ciphKey1 = get_32bitBE(keybuf+4); } else { - /* PSO2 */ - ciphKey2=0xCC554639; - ciphKey1=0x30DBE1AB; + find_hca_key(hca_data, hca, buffer, header_size, &ciphKey1, &ciphKey2); } } - clHCA_clear(hca, ciphKey1, ciphKey2); - - if (clHCA_Decode(hca, hca_data, header_size, 0) < 0) goto fail; + /* init decoder with key */ + clHCA_clear(hca, ciphKey1, ciphKey2); + if ( clHCA_Decode(hca, buffer, header_size, 0) < 0 ) goto fail; + if ( clHCA_getInfo(hca, &hca_data->info) < 0 ) goto fail; - free( hca_data ); - hca_data = NULL; - - if (clHCA_getInfo(hca, &hca_file->info) < 0) goto fail; - hca_file->sample_ptr = clHCA_samplesPerBlock; - hca_file->samples_discard = 0; + /* build the VGMSTREAM */ + vgmstream = allocate_vgmstream(hca_data->info.channelCount, hca_data->info.loopEnabled); + if (!vgmstream) goto fail; - streamFile->get_name( streamFile, filename, sizeof(filename) ); + vgmstream->sample_rate = hca_data->info.samplingRate; + vgmstream->num_samples = hca_data->info.blockCount * clHCA_samplesPerBlock; + vgmstream->loop_start_sample = hca_data->info.loopStart * clHCA_samplesPerBlock; + vgmstream->loop_end_sample = hca_data->info.loopEnd * clHCA_samplesPerBlock; - hca_file->streamfile = streamFile->open(streamFile, filename, STREAMFILE_DEFAULT_BUFFER_SIZE); - if (!hca_file->streamfile) goto fail; - - vgmstream = allocate_vgmstream( hca_file->info.channelCount, 1 ); - if (!vgmstream) goto fail; + vgmstream->coding_type = coding_CRI_HCA; + vgmstream->layout_type = layout_none; + vgmstream->meta_type = meta_HCA; - vgmstream->loop_flag = hca_file->info.loopEnabled; - vgmstream->loop_start_sample = hca_file->info.loopStart * clHCA_samplesPerBlock; - vgmstream->loop_end_sample = hca_file->info.loopEnd * clHCA_samplesPerBlock; + vgmstream->codec_data = hca_data; - vgmstream->codec_data = hca_file; - - vgmstream->channels = hca_file->info.channelCount; - vgmstream->sample_rate = hca_file->info.samplingRate; - - vgmstream->num_samples = hca_file->info.blockCount * clHCA_samplesPerBlock; - - vgmstream->coding_type = coding_CRI_HCA; - vgmstream->layout_type = layout_none; - vgmstream->meta_type = meta_HCA; - - return vgmstream; + return vgmstream; fail: - if ( hca_data ) { - free( hca_data ); - } - if ( hca_file ) { - free( hca_file ); - } - return NULL; + free(hca_data); + return NULL; +} + + +/* Tries to find the decryption key from a list. Simply decodes a few frames and checks if there aren't too many + * clipped samples, as it's common for invalid keys (though possible with valid keys in poorly mastered files). */ +static void find_hca_key(hca_codec_data * hca_data, clHCA * hca, uint8_t * buffer, int header_size, unsigned int * out_key1, unsigned int * out_key2) { + sample testbuf[clHCA_samplesPerBlock]; + int i; + size_t keys_length = sizeof(hcakey_list) / sizeof(hcakey_info); + + int min_clip_count = -1; + /* defaults to PSO2 key, most common */ + unsigned int best_key2 = 0xCC554639; + unsigned int best_key1 = 0x30DBE1AB; + + + /* find a candidate key */ + for (i = 0; i < keys_length; i++) { + int clip_count = 0, sample_count = 0; + int f = 0, s; + + unsigned int key1, key2; + uint64_t key = hcakey_list[i].key; + key2 = (key >> 32) & 0xFFFFFFFF; + key1 = (key >> 0) & 0xFFFFFFFF; + + + /* re-init HCA with the current key as buffer becomes invalid (probably can be simplified) */ + hca_data->curblock = 0; + hca_data->sample_ptr = clHCA_samplesPerBlock; + if ( read_streamfile(buffer, hca_data->start, header_size, hca_data->streamfile) != header_size ) continue; + + clHCA_clear(hca, key1, key2); + if (clHCA_Decode(hca, buffer, header_size, 0) < 0) continue; + if (clHCA_getInfo(hca, &hca_data->info) < 0) continue; + + + /* test enough frames, but not too many */ + while (f < HCA_KEY_MAX_TEST_FRAMES && f < hca_data->info.blockCount) { + decode_hca(hca_data, testbuf, clHCA_samplesPerBlock, hca_data->info.channelCount); + + for (s = 0; s < clHCA_samplesPerBlock; s++) { + if (testbuf[s] != 0x0000 && testbuf[s] != 0xFFFF) + sample_count++; /* ignore upper/lower blank samples */ + + if (testbuf[s] == 0x7FFF || testbuf[s] == 0x8000) + clip_count++; /* upper/lower clip */ + } + + if (clip_count > HCA_KEY_MAX_TEST_CLIPS) + break; /* too many, don't bother */ + if (sample_count >= HCA_KEY_MAX_TEST_SAMPLES) + break; /* enough non-blank samples tested */ + + f++; + } + + //;VGM_LOG("HCA: key %08x%08x clip_count=%i\n", ciphKey2,ciphKey1, clip_count); + if (min_clip_count < 0 || clip_count < min_clip_count) { + min_clip_count = clip_count; + best_key2 = key2; + best_key1 = key1; + } + + if (min_clip_count == 0) + break; /* can't get better than this */ + + /* a few clips is normal, but some invalid keys may give low numbers too */ + //if (clip_count < 10) + // break; + } + + /* reset HCA */ + hca_data->curblock = 0; + hca_data->sample_ptr = clHCA_samplesPerBlock; + read_streamfile(buffer, hca_data->start, header_size, hca_data->streamfile); + + VGM_LOG("HCA: best key=%08x%08x (clips=%i)\n", best_key2,best_key1, min_clip_count); + *out_key2 = best_key2; + *out_key1 = best_key1; } diff --git a/src/meta/hca_keys.h b/src/meta/hca_keys.h new file mode 100644 index 00000000..e3012873 --- /dev/null +++ b/src/meta/hca_keys.h @@ -0,0 +1,106 @@ +#ifndef _HCA_KEYS_H_ +#define _HCA_KEYS_H_ + +typedef struct { + uint64_t key; +} hcakey_info; + +/* CRI's tools expect an unsigned 64 bit number, but keys are commonly found online in hex form */ +static const hcakey_info hcakey_list[] = { + + // HCA Decoder default + {9621963164387704}, // CF222F1FE0748978 + + // Phantasy Star Online 2 (multi?) + // used by most console games + {0xCC55463930DBE1AB}, // CC55463930DBE1AB / 14723751768204501419 + // variation from VGAudio, but some 2ch poster says the above works with CRI's tools; seems to decode the same + {24002584467202475}, // 0055463930DBE1AB + + // Old Phantasy Star Online 2 (multi?) + {61891147883431481}, // 30DBE1ABCC554639 + + // Jojo All Star Battle (PS3) + {19700307}, // 00000000012C9A53 + + // Ro-Kyu-Bu! Himitsu no Otoshimono (PSP) + {2012082716}, // 0000000077EDF21C + + // Ro-Kyu-Bu! Naisho no Shutter Chance (PSV) + {1234253142}, // 0000000049913556 + + // Idolm@ster Cinderella Stage (iOS/Android) + // Shadowverse (iOS/Android) + {59751358413602}, // 00003657F27E3B22 + + // Grimoire ~Shiritsu Grimoire Mahou Gakuen~ (iOS/Android) + {5027916581011272}, // 0011DCDD0DC57F48 + + // Idol Connect (iOS/Android) + {2424}, // 0000000000000978 + + // Kamen Rider Battle Rush (iOS/Android) + {29423500797988784}, // 00688884A11CCFB0 + + // SD Gundam Strikers (iOS/Android) + {30260840980773}, // 00001B85A6AD6125 + + // Sonic Runners (iOS/Android) + {19910623}, // 00000000012FCFDF + + // Fate/Grand Order (iOS/Android) base assets + {12345}, // 0000000000003039 + + // Fate/Grand Order (iOS/Android) download assets *unconfirmed + {9117927877783581796}, // 7E89631892EBF464 + + // Raramagi (iOS/Android) + {45719322}, // 0000000002B99F1A + + // Idolm@ster Million Live (iOS/Android) + {765765765765765}, // 0002B875BC731A85 + + // Kurokishi to Shiro no Maou (iOS/Android) + {3003875739822025258}, // 29AFE911F5816A2A + + // Puella Magi Madoka Magica Side Story: Magia Record (iOS/Android) + // Hortensia Saga (iOS/Android) + {20536401}, // 0000000001395C51 + + // The Tower of Princess (iOS/Android) + {9101518402445063}, // 002055C8634B5F07 + + // Fallen Princess (iOS/Android) + {145552191146490718}, // 02051AF25990FB5E + + // Diss World (iOS/Android) + {9001712656335836006}, // 7CEC81F7C3091366 + + // Ikemen Vampire - Ijin-tachi to Koi no Yuuwaku (iOS/Android) + {45152594117267709}, // 00A06A0B8D0C10FD + + // Super Robot Wars X-Omega (iOS/Android) + {165521992944278}, // 0000968A97978A96 + + // BanG Dream! Girls Band Party! (iOS/Android) + {8910}, // 00000000000022CE + + // Tokyo 7th Sisters (iOS/Android) *unconfirmed + {0xFDAE531AAB414BA1}, // FDAE531AAB414BA1 + + // One Piece Dance Battle (iOS/Android) + {1905818}, // 00000000001D149A + + // Derby Stallion Masters (iOS/Android) + {19840202}, // 00000000012EBCCA + + // World Chain (iOS/Android) + {4892292804961027794}, // 43E4EA62B8E6C6D2 + + // Yuyuyui (iOS/Android) *unconfirmed + {4867249871962584729}, // 438BF1F883653699 + +}; + + +#endif/*_HCA_KEYS_H_*/ diff --git a/src/vgmstream.h b/src/vgmstream.h index 939cca88..950df11a 100644 --- a/src/vgmstream.h +++ b/src/vgmstream.h @@ -1014,12 +1014,13 @@ typedef struct { typedef struct { STREAMFILE *streamfile; uint64_t start; - uint64_t size; + //uint64_t size; clHCA_stInfo info; unsigned int curblock; - unsigned int sample_ptr, samples_discard; + unsigned int sample_ptr; + unsigned int samples_discard; signed short sample_buffer[clHCA_samplesPerBlock * 16]; - // clHCA exists here + //clHCA * hca exists here (pre-alloc'ed) } hca_codec_data; #ifdef VGM_USE_FFMPEG