Add HCA key detection and key list

Updated the meta a bit so it's closer to others and optimized some
mallocs, as it was a bit hard to understand for me so I hope it's for
the better

src/libvgmstream.vcproj

@@ -200,6 +200,10 @@
 					RelativePath=".\meta\meta.h"
 					>
 				</File>
+                <File
+                    RelativePath=".\meta\hca_keys.h"
+                    >
+                </File>
 			</Filter>
 			<Filter
 				Name="Source Files"

src/libvgmstream.vcxproj

@@ -115,6 +115,7 @@
     <ClInclude Include="util.h" />
     <ClInclude Include="vgmstream.h" />
     <ClInclude Include="meta\meta.h" />
+    <ClInclude Include="meta\hca_keys.h" />
     <ClInclude Include="coding\acm_decoder.h" />
     <ClInclude Include="coding\coding.h" />
     <ClInclude Include="coding\fsb_vorbis_data.h" />

src/libvgmstream.vcxproj.filters

@@ -65,6 +65,9 @@
     <ClInclude Include="meta\meta.h">
       <Filter>meta\Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="meta\hca_keys.h">
+      <Filter>meta\Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="coding\acm_decoder.h">
       <Filter>coding\Header Files</Filter>
     </ClInclude>

src/meta/hca.c

@@ -1,116 +1,171 @@
-#include "../vgmstream.h"
 #include "meta.h"
-#include "../util.h"
+#include "hca_keys.h"
+#include "../coding/coding.h"
 
-static VGMSTREAM * init_vgmstream_hca_offset(STREAMFILE *streamFile, uint64_t start, uint64_t size);
+#define HCA_KEY_MAX_TEST_CLIPS   400   /* hopefully nobody masters files with more that a handful... */
+#define HCA_KEY_MAX_TEST_FRAMES  100   /* ~102400 samples */
+#define HCA_KEY_MAX_TEST_SAMPLES 10240 /* ~10 frames of non-blank samples */
+
+static void find_hca_key(hca_codec_data * hca_data, clHCA * hca, uint8_t * buffer, int header_size, unsigned int * out_key1, unsigned int * out_key2);
 
 VGMSTREAM * init_vgmstream_hca(STREAMFILE *streamFile) {
+    VGMSTREAM * vgmstream = NULL;
+    uint8_t buffer[0x8000]; /* hca header buffer data (probably max ~0x400) */
+    char filename[PATH_LIMIT];
+    off_t start = 0;
+    size_t file_size = streamFile->get_size(streamFile);
+
+    int header_size;
+    hca_codec_data * hca_data = NULL; /* vgmstream HCA context */
+    clHCA * hca; /* HCA_Decoder context */
+    unsigned int ciphKey1, ciphKey2;
 
     /* check extension, case insensitive */
     if ( !check_extensions(streamFile, "hca")) return NULL;
 
-	return init_vgmstream_hca_offset( streamFile, 0, streamFile->get_size(streamFile) );
-}
 
-static VGMSTREAM * init_vgmstream_hca_offset(STREAMFILE *streamFile, uint64_t start, uint64_t size) {
+    /* test/init header (find real header size first) */
-	unsigned int ciphKey1;
+    if ( file_size < 8 ) goto fail;
-	unsigned int ciphKey2;
+    if ( read_streamfile(buffer, start, 8, streamFile) != 8 ) goto fail;
 
-	char filename[PATH_LIMIT];
+    header_size = clHCA_isOurFile0(buffer);
+    if ( header_size < 0 || header_size > 0x8000 ) goto fail;
 
-	VGMSTREAM * vgmstream = NULL;
+    if ( read_streamfile(buffer, start, header_size, streamFile) != header_size ) goto fail;
+    if ( clHCA_isOurFile1(buffer, header_size) < 0 ) goto fail;
 
-	hca_codec_data * hca_file = ( hca_codec_data * ) calloc(1, sizeof(hca_codec_data) + clHCA_sizeof());
-	void * hca_data = NULL;
-	clHCA * hca;
-
-	uint8_t header[8];
-
-	int header_size;
-
-	if ( !hca_file ) goto fail;
-
-	if ( size < 8 ) goto fail;
-
-	hca_file->streamfile = streamFile;
-	hca_file->start = start;
-	hca_file->size = size;
-    
-	if ( read_streamfile( header, start, 8, streamFile) != 8 ) goto fail;
-
-	header_size = clHCA_isOurFile0( header );
-    
-	if ( header_size < 0 ) goto fail;
-
-	hca_data = malloc( header_size );
 
+    /* init vgmstream context */
+    hca_data = (hca_codec_data *) calloc(1, sizeof(hca_codec_data) + clHCA_sizeof());
     if (!hca_data) goto fail;
+    //hca_data->size = file_size;
+    hca_data->start = 0;
+    hca_data->sample_ptr = clHCA_samplesPerBlock;
 
-	memcpy( hca_data, header, 8 );
+    /* HCA_Decoder context memory goes right after our codec data (reserved in alloc'ed) */
+    hca = (clHCA *)(hca_data + 1);
 
-	if ( read_streamfile( ((uint8_t*)hca_data) + 8, start + 8, header_size - 8, streamFile ) != header_size - 8 ) goto fail;
+    /* pre-load streamfile so the hca_data is ready before key detection */
+    streamFile->get_name( streamFile, filename, sizeof(filename) );
+    hca_data->streamfile = streamFile->open(streamFile, filename, STREAMFILE_DEFAULT_BUFFER_SIZE);
+    if (!hca_data->streamfile) goto fail;
 
-	if ( clHCA_isOurFile1( hca_data, header_size ) < 0 ) goto fail;
 
-	hca = (clHCA *)(hca_file + 1);
+    /* find decryption key in external file or preloaded list */
-
-    /* try to find key in external file */
     {
         uint8_t keybuf[8];
-
         if ( read_key_file(keybuf, 8, streamFile) ) {
             ciphKey2 = get_32bitBE(keybuf+0);
             ciphKey1 = get_32bitBE(keybuf+4);
         } else {
-            /* PSO2 */
+            find_hca_key(hca_data, hca, buffer, header_size, &ciphKey1, &ciphKey2);
-            ciphKey2=0xCC554639;
-            ciphKey1=0x30DBE1AB;
         }
     }
 
+    /* init decoder with key */
     clHCA_clear(hca, ciphKey1, ciphKey2);
+    if ( clHCA_Decode(hca, buffer, header_size, 0) < 0 ) goto fail;
+    if ( clHCA_getInfo(hca, &hca_data->info) < 0 ) goto fail;
 
-	if (clHCA_Decode(hca, hca_data, header_size, 0) < 0) goto fail;
 
-	free( hca_data );
+    /* build the VGMSTREAM */
-	hca_data = NULL;
+    vgmstream = allocate_vgmstream(hca_data->info.channelCount, hca_data->info.loopEnabled);
-    
-	if (clHCA_getInfo(hca, &hca_file->info) < 0) goto fail;
-
-	hca_file->sample_ptr = clHCA_samplesPerBlock;
-	hca_file->samples_discard = 0;
-
-	streamFile->get_name( streamFile, filename, sizeof(filename) );
-
-	hca_file->streamfile = streamFile->open(streamFile, filename, STREAMFILE_DEFAULT_BUFFER_SIZE);
-	if (!hca_file->streamfile) goto fail;
-    
-	vgmstream = allocate_vgmstream( hca_file->info.channelCount, 1 );
     if (!vgmstream) goto fail;
 
-	vgmstream->loop_flag = hca_file->info.loopEnabled;
+    vgmstream->sample_rate = hca_data->info.samplingRate;
-	vgmstream->loop_start_sample = hca_file->info.loopStart * clHCA_samplesPerBlock;
+    vgmstream->num_samples = hca_data->info.blockCount * clHCA_samplesPerBlock;
-	vgmstream->loop_end_sample = hca_file->info.loopEnd * clHCA_samplesPerBlock;
+    vgmstream->loop_start_sample = hca_data->info.loopStart * clHCA_samplesPerBlock;
-
+    vgmstream->loop_end_sample = hca_data->info.loopEnd * clHCA_samplesPerBlock;
-	vgmstream->codec_data = hca_file;
-
-	vgmstream->channels = hca_file->info.channelCount;
-	vgmstream->sample_rate = hca_file->info.samplingRate;
-
-	vgmstream->num_samples = hca_file->info.blockCount * clHCA_samplesPerBlock;
 
     vgmstream->coding_type = coding_CRI_HCA;
     vgmstream->layout_type = layout_none;
     vgmstream->meta_type = meta_HCA;
 
+    vgmstream->codec_data = hca_data;
+
     return vgmstream;
 
 fail:
-	if ( hca_data ) {
     free(hca_data);
-	}
-	if ( hca_file ) {
-		free( hca_file );
-	}
     return NULL;
 }
+
+
+/* Tries to find the decryption key from a list. Simply decodes a few frames and checks if there aren't too many
+ * clipped samples, as it's common for invalid keys (though possible with valid keys in poorly mastered files). */
+static void find_hca_key(hca_codec_data * hca_data, clHCA * hca, uint8_t * buffer, int header_size, unsigned int * out_key1, unsigned int * out_key2) {
+    sample testbuf[clHCA_samplesPerBlock];
+    int i;
+    size_t keys_length = sizeof(hcakey_list) / sizeof(hcakey_info);
+
+    int min_clip_count = -1;
+    /* defaults to PSO2 key, most common */
+    unsigned int best_key2 = 0xCC554639;
+    unsigned int best_key1 = 0x30DBE1AB;
+
+
+    /* find a candidate key */
+    for (i = 0; i < keys_length; i++) {
+        int clip_count = 0, sample_count = 0;
+        int f = 0, s;
+
+        unsigned int key1, key2;
+        uint64_t key = hcakey_list[i].key;
+        key2 = (key >> 32) & 0xFFFFFFFF;
+        key1 = (key >>  0) & 0xFFFFFFFF;
+
+
+        /* re-init HCA with the current key as buffer becomes invalid (probably can be simplified) */
+        hca_data->curblock = 0;
+        hca_data->sample_ptr = clHCA_samplesPerBlock;
+        if ( read_streamfile(buffer, hca_data->start, header_size, hca_data->streamfile) != header_size ) continue;
+
+        clHCA_clear(hca, key1, key2);
+        if (clHCA_Decode(hca, buffer, header_size, 0) < 0) continue;
+        if (clHCA_getInfo(hca, &hca_data->info) < 0) continue;
+
+
+        /* test enough frames, but not too many */
+        while (f < HCA_KEY_MAX_TEST_FRAMES && f < hca_data->info.blockCount) {
+            decode_hca(hca_data, testbuf, clHCA_samplesPerBlock, hca_data->info.channelCount);
+
+            for (s = 0; s < clHCA_samplesPerBlock; s++) {
+                if (testbuf[s] != 0x0000 && testbuf[s] != 0xFFFF)
+                    sample_count++; /* ignore upper/lower blank samples */
+
+                if (testbuf[s] == 0x7FFF || testbuf[s] == 0x8000)
+                    clip_count++; /* upper/lower clip */
+            }
+
+            if (clip_count > HCA_KEY_MAX_TEST_CLIPS)
+                break; /* too many, don't bother */
+            if (sample_count >= HCA_KEY_MAX_TEST_SAMPLES)
+                break; /* enough non-blank samples tested */
+
+            f++;
+        }
+
+        //;VGM_LOG("HCA: key %08x%08x clip_count=%i\n", ciphKey2,ciphKey1, clip_count);
+        if (min_clip_count < 0 || clip_count < min_clip_count) {
+            min_clip_count = clip_count;
+            best_key2 = key2;
+            best_key1 = key1;
+        }
+
+        if (min_clip_count == 0)
+            break; /* can't get better than this */
+
+        /* a few clips is normal, but some invalid keys may give low numbers too */
+        //if (clip_count < 10)
+        //    break;
+    }
+
+    /* reset HCA */
+    hca_data->curblock = 0;
+    hca_data->sample_ptr = clHCA_samplesPerBlock;
+    read_streamfile(buffer, hca_data->start, header_size, hca_data->streamfile);
+
+    VGM_LOG("HCA: best key=%08x%08x (clips=%i)\n", best_key2,best_key1, min_clip_count);
+    *out_key2 = best_key2;
+    *out_key1 = best_key1;
+}

src/meta/hca_keys.h

@@ -0,0 +1,106 @@
+#ifndef _HCA_KEYS_H_
+#define _HCA_KEYS_H_
+
+typedef struct {
+    uint64_t key;
+} hcakey_info;
+
+/* CRI's tools expect an unsigned 64 bit number, but keys are commonly found online in hex form */
+static const hcakey_info hcakey_list[] = {
+
+        // HCA Decoder default
+        {9621963164387704},         // CF222F1FE0748978
+
+        // Phantasy Star Online 2 (multi?)
+        // used by most console games
+        {0xCC55463930DBE1AB},       // CC55463930DBE1AB / 14723751768204501419
+        // variation from VGAudio, but some 2ch poster says the above works with CRI's tools; seems to decode the same
+        {24002584467202475},        // 0055463930DBE1AB
+
+        // Old Phantasy Star Online 2 (multi?)
+        {61891147883431481},        // 30DBE1ABCC554639
+
+        // Jojo All Star Battle (PS3)
+        {19700307},                 // 00000000012C9A53
+
+        // Ro-Kyu-Bu! Himitsu no Otoshimono (PSP)
+        {2012082716},               // 0000000077EDF21C
+
+        // Ro-Kyu-Bu! Naisho no Shutter Chance (PSV)
+        {1234253142},               // 0000000049913556
+
+        // Idolm@ster Cinderella Stage (iOS/Android)
+        // Shadowverse (iOS/Android)
+        {59751358413602},           // 00003657F27E3B22
+
+        // Grimoire ~Shiritsu Grimoire Mahou Gakuen~ (iOS/Android)
+        {5027916581011272},         // 0011DCDD0DC57F48
+
+        // Idol Connect (iOS/Android)
+        {2424},                     // 0000000000000978
+
+        // Kamen Rider Battle Rush (iOS/Android)
+        {29423500797988784},        // 00688884A11CCFB0
+
+        // SD Gundam Strikers (iOS/Android)
+        {30260840980773},           // 00001B85A6AD6125
+
+        // Sonic Runners (iOS/Android)
+        {19910623},                 // 00000000012FCFDF
+
+        // Fate/Grand Order (iOS/Android) base assets
+        {12345},                    // 0000000000003039
+
+        // Fate/Grand Order (iOS/Android) download assets *unconfirmed
+        {9117927877783581796},      // 7E89631892EBF464
+
+        // Raramagi (iOS/Android)
+        {45719322},                 // 0000000002B99F1A
+
+        // Idolm@ster Million Live (iOS/Android)
+        {765765765765765},          // 0002B875BC731A85
+
+        // Kurokishi to Shiro no Maou (iOS/Android)
+        {3003875739822025258},      // 29AFE911F5816A2A
+
+        // Puella Magi Madoka Magica Side Story: Magia Record (iOS/Android)
+        // Hortensia Saga (iOS/Android)
+        {20536401},                 // 0000000001395C51
+
+        // The Tower of Princess (iOS/Android)
+        {9101518402445063},         // 002055C8634B5F07
+
+        // Fallen Princess (iOS/Android)
+        {145552191146490718},       // 02051AF25990FB5E
+
+        // Diss World (iOS/Android)
+        {9001712656335836006},      // 7CEC81F7C3091366
+
+        // Ikemen Vampire - Ijin-tachi to Koi no Yuuwaku (iOS/Android)
+        {45152594117267709},        // 00A06A0B8D0C10FD
+
+        // Super Robot Wars X-Omega (iOS/Android)
+        {165521992944278},          // 0000968A97978A96
+
+        // BanG Dream! Girls Band Party! (iOS/Android)
+        {8910},                     // 00000000000022CE
+
+        // Tokyo 7th Sisters (iOS/Android) *unconfirmed
+        {0xFDAE531AAB414BA1},       // FDAE531AAB414BA1
+
+        // One Piece Dance Battle (iOS/Android)
+        {1905818},                  // 00000000001D149A
+
+        // Derby Stallion Masters (iOS/Android)
+        {19840202},                 // 00000000012EBCCA
+
+        // World Chain (iOS/Android)
+        {4892292804961027794},      // 43E4EA62B8E6C6D2
+
+        // Yuyuyui (iOS/Android) *unconfirmed
+        {4867249871962584729},      // 438BF1F883653699
+
+};
+
+
+#endif/*_HCA_KEYS_H_*/

src/vgmstream.h

@@ -1014,12 +1014,13 @@ typedef struct {
 typedef struct {
     STREAMFILE *streamfile;
     uint64_t start;
-    uint64_t size;
+    //uint64_t size;
     clHCA_stInfo info;
     unsigned int curblock;
-    unsigned int sample_ptr, samples_discard;
+    unsigned int sample_ptr;
+    unsigned int samples_discard;
     signed short sample_buffer[clHCA_samplesPerBlock * 16];
-    // clHCA exists here
+    //clHCA * hca exists here (pre-alloc'ed)
 } hca_codec_data;
 
 #ifdef VGM_USE_FFMPEG