Improve HCA/clHCA key detection

- Only test keys when file is encrypted
- Add clHCA_TestBlock to test a frame with the current key
- Move some key detection code to hca_decoder.c and simplify hca.c
detection
- Decrease number of test frames due to increased accuracy of
clHCA_TestBlock
This commit is contained in:
bnnm 2018-09-02 16:00:58 +02:00
parent fd52fe0e95
commit 9c0db7cae3
5 changed files with 194 additions and 74 deletions

View File

@ -36,16 +36,16 @@ typedef struct clHCA_stInfo {
unsigned int channelCount;
unsigned int blockSize;
unsigned int blockCount;
unsigned int encoderDelay; /* samples appended to the beginning */
unsigned int encoderPadding; /* samples appended to the end */
unsigned int loopEnabled;
unsigned int loopStartBlock;
unsigned int loopEndBlock;
unsigned int loopStartDelay; /* samples in block before loop starts */
unsigned int loopEndPadding; /* samples in block after loop ends */
unsigned int encoderDelay; /* samples appended to the beginning */
unsigned int encoderPadding; /* samples appended to the end */
unsigned int samplesPerBlock; /* should be 1024 */
const char *comment;
unsigned int encryptionEnabled; /* requires keycode */
/* Derived sample formulas:
* - sample count: blockCount*samplesPerBlock - encoderDelay - encoderPadding;
@ -59,11 +59,6 @@ typedef struct clHCA_stInfo {
* Returns 0 on success, <0 on failure. */
int clHCA_getInfo(clHCA *, clHCA_stInfo *out);
/* Sets a 64 bit encryption key, to properly decode blocks. This may be called
* multiple times to change the key, before or after clHCA_DecodeHeader.
* Key is ignored if the file is not encrypted. */
void clHCA_SetKey(clHCA *, unsigned long long keycode);
/* Decodes a single frame, from data after headerSize. Should be called after
* clHCA_DecodeHeader and size must be at least blockSize long.
* Returns 0 on success, <0 on failure. */
@ -74,6 +69,17 @@ int clHCA_DecodeBlock(clHCA *, void *data, unsigned int size);
* next decode. Buffer must be at least (samplesPerBlock*channels) long. */
void clHCA_ReadSamples16(clHCA *, signed short * outSamples);
/* Sets a 64 bit encryption key, to properly decode blocks. This may be called
* multiple times to change the key, before or after clHCA_DecodeHeader.
* Key is ignored if the file is not encrypted. */
void clHCA_SetKey(clHCA *, unsigned long long keycode);
/* Tests a single frame for validity, mainly to test if current key is correct.
* Returns <0 on incorrect block (wrong key), 0 on silent block (not useful to determine)
* and >0 if block is correct (the closer to 1 the more likely).
* Incorrect keys may give a few valid frames, so it's best to test a number of them
* and select the key with scores closer to 1. */
int clHCA_TestBlock(clHCA *hca, void *data, unsigned int size);
#ifdef __cplusplus
}

View File

@ -12,7 +12,7 @@
/* TODO:
* - improve portability on types and float casts, sizeof(int) isn't necessarily sizeof(float)
* - check "packed_noise_level" vs VGAudio (CriHcaPacking.UnpackFrameHeader), weird behaviour
* - check "delta decode" vs VGAudio (CriHcaPacking.DeltaDecode), may be setting wrong values on bad data
* - check "delta scalefactors" vs VGAudio (CriHcaPacking.DeltaDecode), may be setting wrong values on bad data
* - check "read intensity" vs VGAudio (CriHcaPacking.ReadIntensity), skips intensities if first is 15
* - simplify DCT4 code
* - add extra validations: encoder_delay/padding < sample_count, bands/totals (max: 128?), track count==1, etc
@ -42,9 +42,9 @@ typedef struct stChannel {
/* HCA channel config */
int type; /* discrete / stereo-primary / stereo-secondary */
unsigned int coded_scalefactor_count; /* scalefactors used (depending on channel type) */
unsigned char *hfr_scales; /* high frequency scales, pointing to higher scalefactors (simplification) */
/* subframe state */
unsigned char *hfr_scales; /* high frequency scales, pointing to higher scalefactors (simplification) */
unsigned char intensity[HCA_SUBFRAMES_PER_FRAME]; /* intensity indexes (value max: 15 / 4b) */
unsigned char scalefactors[HCA_SAMPLES_PER_SUBFRAME]; /* scale indexes (value max: 64 / 6b)*/
unsigned char resolution[HCA_SAMPLES_PER_SUBFRAME]; /* resolution indexes (value max: 15 / 4b) */
@ -262,15 +262,16 @@ int clHCA_getInfo(clHCA *hca, clHCA_stInfo *info) {
info->channelCount = hca->channels;
info->blockSize = hca->frame_size;
info->blockCount = hca->frame_count;
info->encoderDelay = hca->encoder_delay;
info->encoderPadding = hca->encoder_padding;
info->loopEnabled = hca->loop_flag;
info->loopStartBlock = hca->loop_start_frame;
info->loopEndBlock = hca->loop_end_frame;
info->loopStartDelay = hca->loop_start_delay;
info->loopEndPadding = hca->loop_end_padding;
info->encoderDelay = hca->encoder_delay;
info->encoderPadding = hca->encoder_padding;
info->samplesPerBlock = HCA_SAMPLES_PER_FRAME;
info->comment = hca->comment;
info->encryptionEnabled = hca->ciph_type == 56; /* keycode encryption */
return 0;
}
@ -692,7 +693,7 @@ int clHCA_DecodeHeader(clHCA *hca, void *data, unsigned int size) {
hca->ath_type = bitreader_read(&br, 16);
}
else {
/* removed in v2.0, default in v1.x (maybe only ever used in v1.1) */
/* removed in v2.0, default in v1.x (maybe only used in v1.1, as v1.2/v1.3 set ath_type = 0) */
hca->ath_type = (hca->version < 0x200) ? 1 : 0;
}
@ -912,10 +913,74 @@ void clHCA_SetKey(clHCA *hca, unsigned long long keycode) {
}
}
int clHCA_TestBlock(clHCA *hca, void *data, unsigned int size) {
const float scale = 32768.0f;
unsigned int ch, sf, s;
int status;
int clips = 0, blanks = 0;
float fsample;
signed int psample;
/* return if decode fails (happens sometimes with wrong keys) */
status = clHCA_DecodeBlock(hca, data, size);
if (status < 0)
return -1;
/* check decode results */
for (ch = 0; ch < hca->channels; ch++) {
for (sf = 0; sf < HCA_SUBFRAMES_PER_FRAME; sf++) {
for (s = 0; s < HCA_SAMPLES_PER_SUBFRAME; s++) {
fsample = hca->channel[ch].wave[sf][s];
psample = (signed int) (fsample * scale);
if (fsample > 1.0f || fsample < -1.0f)
clips++;
else if (psample == 0 || psample == -1)
blanks++;
}
}
}
/* the more clips the less likely block was correctly decrypted */
if (clips > 0)
return clips;
/* if block is silent result is not useful */
if (blanks == hca->channels * HCA_SUBFRAMES_PER_FRAME * HCA_SAMPLES_PER_SUBFRAME)
return 0;
/* block may be correct (but wrong keys can get this too) */
return 1;
}
#if 0
// it'd seem like resetting IMDCT (others get overwritten) would matter when restarting the
// stream from 0, but doesn't seem any different, maybe because the first frame acts as setup/empty
void clHCA_DecodeReset(clHCA * hca) {
unsigned int i;
if (!hca || !hca->is_valid)
return;
for (i = 0; i < hca->channels; i++) {
stChannel *ch = &hca->channel[i];
memset(ch->intensity, 0, sizeof(ch->intensity[0]) * HCA_SUBFRAMES_PER_FRAME);
memset(ch->scalefactors, 0, sizeof(ch->scalefactors[0]) * HCA_SAMPLES_PER_SUBFRAME);
memset(ch->resolution, 0, sizeof(ch->resolution[0]) * HCA_SAMPLES_PER_SUBFRAME);
memset(ch->gain, 0, sizeof(ch->gain[0]) * HCA_SAMPLES_PER_SUBFRAME);
memset(ch->spectra, 0, sizeof(ch->spectra[0]) * HCA_SAMPLES_PER_SUBFRAME);
memset(ch->temp, 0, sizeof(ch->temp[0]) * HCA_SAMPLES_PER_SUBFRAME);
memset(ch->dct, 0, sizeof(ch->dct[0]) * HCA_SAMPLES_PER_SUBFRAME);
memset(ch->imdct_previous, 0, sizeof(ch->imdct_previous[0]) * HCA_SAMPLES_PER_SUBFRAME);
memset(ch->wave, 0, sizeof(ch->wave[0][0]) * HCA_SUBFRAMES_PER_FRAME * HCA_SUBFRAMES_PER_FRAME);
}
}
#endif
//--------------------------------------------------
// Decode
//--------------------------------------------------
static void decode1_unpack_channel(stChannel *ch, clData *br,
static int decode1_unpack_channel(stChannel *ch, clData *br,
unsigned int hfr_group_count, unsigned int packed_noise_level, const unsigned char *ath_curve);
static void decode2_dequantize_coefficients(stChannel *ch, clData *br);
@ -960,8 +1025,10 @@ int clHCA_DecodeBlock(clHCA *hca, void *data, unsigned int size) {
unsigned int packed_noise_level = (frame_acceptable_noise_level << 8) - frame_evaluation_boundary;
for (ch = 0; ch < hca->channels; ch++) {
decode1_unpack_channel(&hca->channel[ch], &br,
int unpack = decode1_unpack_channel(&hca->channel[ch], &br,
hca->hfr_group_count, packed_noise_level, hca->ath_curve);
if (unpack < 0)
return -1;
}
}
@ -991,6 +1058,11 @@ int clHCA_DecodeBlock(clHCA *hca, void *data, unsigned int size) {
}
}
/* should read all frame sans checksum at most */
if (br.bit > br.size - 16) {
return -1;
}
return 0;
}
@ -1029,7 +1101,7 @@ static const unsigned int decode1_quantizer_step_size_int[16] = {
};
static const float *decode1_quantizer_step_size = (const float *)decode1_quantizer_step_size_int;
static void decode1_unpack_channel(stChannel *ch, clData *br,
static int decode1_unpack_channel(stChannel *ch, clData *br,
unsigned int hfr_group_count, unsigned int packed_noise_level, const unsigned char *ath_curve) {
unsigned int i;
const unsigned int csf_count = ch->coded_scalefactor_count;
@ -1046,16 +1118,23 @@ static void decode1_unpack_channel(stChannel *ch, clData *br,
}
}
else if (scalefactor_delta_bits > 0) {
/* delta decode */
unsigned char expected_delta = (1 << scalefactor_delta_bits) - 1;
unsigned char extra_delta = expected_delta >> 1;
/* delta scalefactors */
const unsigned char expected_delta = (1 << scalefactor_delta_bits) - 1;
const unsigned char extra_delta = expected_delta >> 1;
unsigned char scalefactor_prev = bitreader_read(br, 6);
ch->scalefactors[0] = scalefactor_prev;
for (i = 1; i < csf_count; i++) {
unsigned char delta = bitreader_read(br, scalefactor_delta_bits);
if (delta != expected_delta) { //??? may give negative escalefactors on wrong values???
scalefactor_prev += delta - extra_delta; /* delta scalefactors */
if (delta != expected_delta) {
/* may happen with bad keycodes, scalefactors must be 6b indexes */
int scalefactor_test = (int)scalefactor_prev + ((int)delta - (int)extra_delta);
if (scalefactor_test < 0 || scalefactor_test > 64) {
return -1;
}
scalefactor_prev += delta - extra_delta;
} else {
scalefactor_prev = bitreader_read(br, 6);
}
@ -1073,11 +1152,15 @@ static void decode1_unpack_channel(stChannel *ch, clData *br,
unsigned char intensity_value = bitreader_peek(br, 4);
ch->intensity[0] = intensity_value;
if (intensity_value < 15) { /* 15 may be an invalid value? */
if (intensity_value < 15) {
for (i = 0; i < HCA_SUBFRAMES_PER_FRAME; i++) {
ch->intensity[i] = bitreader_read(br, 4);
}
}
/* 15 may be an invalid value? */
//else {
// return -1;
//}
}
else {
/* read hfr scalefactors */
@ -1120,6 +1203,8 @@ static void decode1_unpack_channel(stChannel *ch, clData *br,
ch->gain[i] = scalefactor_scale * resolution_scale;
}
}
return 0;
}
//--------------------------------------------------

View File

@ -175,6 +175,7 @@ void decode_hca(hca_codec_data * data, sample * outbuf, int32_t samples_to_do);
void reset_hca(hca_codec_data * data);
void loop_hca(hca_codec_data * data);
void free_hca(hca_codec_data * data);
int test_hca_key(hca_codec_data * data, unsigned long long keycode);
#ifdef VGM_USE_VORBIS
/* ogg_vorbis_decoder */

View File

@ -106,7 +106,7 @@ void decode_hca(hca_codec_data * data, sample * outbuf, int32_t samples_to_do) {
/* decode frame */
status = clHCA_DecodeBlock(data->handle, (void*)(data->data_buffer), blockSize);
if (status < 0) {
VGM_LOG("HCA: decode fail at %lx", offset);
VGM_LOG("HCA: decode fail at %lx\n", offset);
break;
}
@ -120,7 +120,6 @@ void decode_hca(hca_codec_data * data, sample * outbuf, int32_t samples_to_do) {
}
}
void reset_hca(hca_codec_data * data) {
if (!data) return;
@ -149,3 +148,61 @@ void free_hca(hca_codec_data * data) {
free(data->sample_buffer);
free(data);
}
#define HCA_KEY_MAX_BLANK_FRAMES 15 /* ignored up to N blank frames (not uncommon to have ~10, if more something is off) */
#define HCA_KEY_MAX_TEST_FRAMES 10 /* 5~15 should be enough, but mostly silent or badly mastered files may need more */
#define HCA_KEY_MAX_ACCEPTABLE_SCORE 300 /* unlikely to work correctly, 10~30 may be ok */
/* Test a number of frames if key decrypts correctly.
* Returns score: <0: error/wrong, 0: unknown/silent file, >0: good (the closest to 1 the better) */
int test_hca_key(hca_codec_data * data, unsigned long long keycode) {
size_t test_frame = 0, current_frame = 0, blank_frames = 0;
int total_score = 0;
const unsigned int blockSize = data->info.blockSize;
clHCA_SetKey(data->handle, keycode);
while (test_frame < HCA_KEY_MAX_TEST_FRAMES && current_frame < data->info.blockCount) {
off_t offset = data->info.headerSize + current_frame * blockSize;
int score;
size_t bytes;
/* read frame */
bytes = read_streamfile(data->data_buffer, offset, blockSize, data->streamfile);
if (bytes != blockSize) {
total_score = -1;
break;
}
/* test frame */
score = clHCA_TestBlock(data->handle, (void*)(data->data_buffer), blockSize);
if (score < 0) {
total_score = -1;
break;
}
current_frame++;
/* skip blank block at the beginning */
if (score == 0 && blank_frames < HCA_KEY_MAX_BLANK_FRAMES) {
blank_frames++;
continue;
}
test_frame++;
total_score += score;
/* too far, don't bother checking more frames */
if (total_score > HCA_KEY_MAX_ACCEPTABLE_SCORE)
break;
}
/* signal best possible score */
if (total_score > 0 && total_score <= HCA_KEY_MAX_TEST_FRAMES) {
total_score = 1;
}
return total_score;
}

View File

@ -20,7 +20,7 @@ VGMSTREAM * init_vgmstream_hca(STREAMFILE *streamFile) {
if (!hca_data) goto fail;
/* find decryption key in external file or preloaded list */
{
if (hca_data->info.encryptionEnabled) {
uint8_t keybuf[8];
if (read_key_file(keybuf, 8, streamFile) == 8) {
keycode = (uint64_t)get_64bitBE(keybuf+0x00);
@ -63,75 +63,46 @@ fail:
}
#define HCA_KEY_MAX_TEST_CLIPS 400 /* hopefully nobody masters files with more that a handful... */
#define HCA_KEY_MAX_TEST_FRAMES 100 /* ~102400 samples */
#define HCA_KEY_MAX_TEST_SAMPLES 10240 /* ~10 frames of non-blank samples */
/* Tries to find the decryption key from a list. Simply decodes a few frames and checks if there aren't too many
* clipped samples, as it's common for invalid keys (though possible with valid keys in poorly mastered files). */
/* Try to find the decryption key from a list. */
static void find_hca_key(hca_codec_data * hca_data, unsigned long long * out_keycode) {
const size_t keys_length = sizeof(hcakey_list) / sizeof(hcakey_info);
sample *test_samples = NULL;
size_t buffer_samples = hca_data->info.samplesPerBlock * hca_data->info.channelCount;
unsigned long long best_keycode;
int best_score = -1;
int i;
int min_clip_count = -1;
test_samples = malloc(sizeof(sample) * buffer_samples);
if (!test_samples)
return; /* ??? */
best_keycode = 0xCC55463930DBE1AB; /* defaults to PSO2 key, most common */
/* find a candidate key */
for (i = 0; i < keys_length; i++) {
int clip_count = 0, sample_count = 0;
int frame = 0, s;
int score;
unsigned long long keycode = (unsigned long long)hcakey_list[i].key;
clHCA_SetKey(hca_data->handle, keycode);
reset_hca(hca_data);
score = test_hca_key(hca_data, keycode);
/* test enough frames, but not too many */
while (frame < HCA_KEY_MAX_TEST_FRAMES && frame < hca_data->info.blockCount) {
decode_hca(hca_data, test_samples, hca_data->info.samplesPerBlock);
//;VGM_LOG("HCA: test key=%08x%08x, score=%i\n",
// (uint32_t)((keycode >> 32) & 0xFFFFFFFF), (uint32_t)(keycode & 0xFFFFFFFF), score);
for (s = 0; s < buffer_samples; s++) {
if (test_samples[s] != 0 && test_samples[s] != -1)
sample_count++; /* ignore upper/lower blank samples */
/* wrong key */
if (score < 0)
continue;
if (test_samples[s] == 32767 || test_samples[s] == -32768)
clip_count++; /* upper/lower clip */
}
if (clip_count > HCA_KEY_MAX_TEST_CLIPS)
break; /* too many, don't bother */
if (sample_count >= HCA_KEY_MAX_TEST_SAMPLES)
break; /* enough non-blank samples tested */
frame++;
}
if (min_clip_count < 0 || clip_count < min_clip_count) {
min_clip_count = clip_count;
/* score 0 is not trustable, update too if something better is found */
if (best_score < 0 || score < best_score || (best_score == 0 && score == 1)) {
best_score = score;
best_keycode = keycode;
}
if (min_clip_count == 0)
break; /* can't get better than this */
/* a few clips is normal, but some invalid keys may give low numbers too */
//if (clip_count < 10)
// break;
/* best possible score */
if (score == 1) {
break;
}
}
VGM_ASSERT(min_clip_count > 0,
"HCA: best key=%08x%08x (clips=%i)\n",
(uint32_t)((best_keycode >> 32) & 0xFFFFFFFF), (uint32_t)(best_keycode & 0xFFFFFFFF), min_clip_count);
//;VGM_LOG("HCA: best key=%08x%08x (score=%i)\n",
// (uint32_t)((best_keycode >> 32) & 0xFFFFFFFF), (uint32_t)(best_keycode & 0xFFFFFFFF), best_score);
reset_hca(hca_data);
VGM_ASSERT(best_score > 1, "HCA: best key=%08x%08x (score=%i)\n",
(uint32_t)((best_keycode >> 32) & 0xFFFFFFFF), (uint32_t)(best_keycode & 0xFFFFFFFF), best_score);
*out_keycode = best_keycode;
free(test_samples);
}