1
0
mirror of synced 2025-02-21 12:29:47 +01:00

feat: Added Digram and Layered Distribution plots to information view

This commit is contained in:
WerWolv 2022-12-27 22:50:37 +01:00
parent f1aeec309e
commit 4807ca0057
5 changed files with 283 additions and 164 deletions

View File

@ -0,0 +1,224 @@
#pragma once
#include <hex.hpp>
#include <imgui.h>
#define IMGUI_DEFINE_MATH_OPERATORS
#include <imgui_internal.h>
#include <random>
namespace hex {
namespace {
std::vector<u8> getSampleSelection(prv::Provider *provider, u64 address, size_t size, size_t sampleSize) {
const size_t sequenceCount = std::ceil(std::sqrt(sampleSize));
std::vector<u8> buffer;
if (size < sampleSize) {
buffer.resize(size);
provider->read(address, buffer.data(), size);
} else {
std::random_device randomDevice;
std::mt19937_64 random(randomDevice());
std::map<u64, std::vector<u8>> orderedData;
for (u32 i = 0; i < sequenceCount; i++) {
ssize_t offset = random() % size;
std::vector<u8> sequence;
sequence.resize(std::min<size_t>(sequenceCount, size - offset));
provider->read(address + offset, sequence.data(), sequence.size());
orderedData.insert({ offset, sequence });
}
buffer.reserve(sampleSize);
u64 lastEnd = 0x00;
for (const auto &[offset, sequence] : orderedData) {
if (offset < lastEnd)
buffer.resize(buffer.size() - (lastEnd - offset));
std::copy(sequence.begin(), sequence.end(), std::back_inserter(buffer));
lastEnd = offset + sequence.size();
}
}
return buffer;
}
std::vector<u8> getSampleSelection(const std::vector<u8> &inputBuffer, size_t sampleSize) {
const size_t sequenceCount = std::ceil(std::sqrt(sampleSize));
std::vector<u8> buffer;
if (inputBuffer.size() < sampleSize) {
buffer = inputBuffer;
} else {
std::random_device randomDevice;
std::mt19937_64 random(randomDevice());
std::map<u64, std::vector<u8>> orderedData;
for (u32 i = 0; i < sequenceCount; i++) {
ssize_t offset = random() % inputBuffer.size();
std::vector<u8> sequence;
sequence.reserve(sampleSize);
std::copy(inputBuffer.begin() + offset, inputBuffer.begin() + offset + std::min<size_t>(sequenceCount, inputBuffer.size() - offset), std::back_inserter(sequence));
orderedData.insert({ offset, sequence });
}
buffer.reserve(sampleSize);
u64 lastEnd = 0x00;
for (const auto &[offset, sequence] : orderedData) {
if (offset < lastEnd)
buffer.resize(buffer.size() - (lastEnd - offset));
std::copy(sequence.begin(), sequence.end(), std::back_inserter(buffer));
lastEnd = offset + sequence.size();
}
}
return buffer;
}
}
class DiagramDigram {
public:
DiagramDigram(size_t sampleSize = 0x9000) : m_sampleSize(sampleSize) { }
void draw(ImVec2 size) {
ImGui::PushStyleColor(ImGuiCol_ChildBg, ImU32(ImColor(0, 0, 0)));
if (ImGui::BeginChild("##digram", size, true)) {
auto drawList = ImGui::GetWindowDrawList();
float xStep = (size.x * 0.95F) / 0xFF;
float yStep = (size.y * 0.95F) / 0xFF;
if (!this->m_processing)
for (size_t i = 0; i < (this->m_buffer.empty() ? 0 : this->m_buffer.size() - 1); i++) {
const auto &[x, y] = std::pair { this->m_buffer[i] * xStep, this->m_buffer[i + 1] * yStep };
auto color = ImLerp(ImColor(0xFF, 0x6D, 0x01).Value, ImColor(0x01, 0x93, 0xFF).Value, float(i) / this->m_buffer.size());
color.w = this->m_opacityBuffer[i];
auto pos = ImGui::GetWindowPos() + ImVec2(size.x * 0.025F, size.y * 0.025F) + ImVec2(x, y);
drawList->AddRectFilled(pos, pos + ImVec2(xStep, yStep), ImColor(color));
}
}
ImGui::EndChild();
ImGui::PopStyleColor();
}
void process(prv::Provider *provider, u64 address, size_t size) {
this->m_processing = true;
this->m_buffer = getSampleSelection(provider, address, size, this->m_sampleSize);
processImpl();
this->m_processing = false;
}
void process(const std::vector<u8> &buffer) {
this->m_processing = true;
this->m_buffer = getSampleSelection(buffer, this->m_sampleSize);
processImpl();
this->m_processing = false;
}
private:
void processImpl() {
this->m_opacityBuffer.resize(this->m_buffer.size());
std::map<u64, size_t> heatMap;
for (size_t i = 0; i < (this->m_buffer.empty() ? 0 : this->m_buffer.size() - 1); i++) {
auto count = ++heatMap[this->m_buffer[i] << 8 | heatMap[i + 1]];
this->m_highestCount = std::max(this->m_highestCount, count);
}
for (size_t i = 0; i < (this->m_buffer.empty() ? 0 : this->m_buffer.size() - 1); i++) {
this->m_opacityBuffer[i] = std::min(0.2F + (float(heatMap[this->m_buffer[i] << 8 | this->m_buffer[i + 1]]) / float(this->m_highestCount / 1000)), 1.0F);
}
}
private:
size_t m_sampleSize;
std::vector<u8> m_buffer;
std::vector<float> m_opacityBuffer;
size_t m_highestCount = 0;
std::atomic<bool> m_processing = false;
};
class DiagramLayeredDistribution {
public:
DiagramLayeredDistribution(size_t sampleSize = 0x9000) : m_sampleSize(sampleSize) { }
void draw(ImVec2 size) {
ImGui::PushStyleColor(ImGuiCol_ChildBg, ImU32(ImColor(0, 0, 0)));
if (ImGui::BeginChild("##layered_distribution", size, true)) {
auto drawList = ImGui::GetWindowDrawList();
float xStep = (size.x * 0.95F) / 0xFF;
float yStep = (size.y * 0.95F) / 0xFF;
if (!this->m_processing)
for (size_t i = 0; i < (this->m_buffer.empty() ? 0 : this->m_buffer.size()); i++) {
const auto &[x, y] = std::pair { this->m_buffer[i] * xStep, yStep * ((float(i) / this->m_buffer.size()) * 0xFF) };
auto color = ImLerp(ImColor(0xFF, 0x6D, 0x01).Value, ImColor(0x01, 0x93, 0xFF).Value, float(i) / this->m_buffer.size());
color.w = this->m_opacityBuffer[i];
auto pos = ImGui::GetWindowPos() + ImVec2(size.x * 0.025F, size.y * 0.025F) + ImVec2(x, y);
drawList->AddRectFilled(pos, pos + ImVec2(xStep, yStep), ImColor(color));
}
}
ImGui::EndChild();
ImGui::PopStyleColor();
}
void process(prv::Provider *provider, u64 address, size_t size) {
this->m_processing = true;
this->m_buffer = getSampleSelection(provider, address, size, this->m_sampleSize);
processImpl();
this->m_processing = false;
}
void process(const std::vector<u8> &buffer) {
this->m_processing = true;
this->m_buffer = getSampleSelection(buffer, this->m_sampleSize);
processImpl();
this->m_processing = false;
}
private:
void processImpl() {
this->m_opacityBuffer.resize(this->m_buffer.size());
std::map<u64, size_t> heatMap;
for (size_t i = 0; i < (this->m_buffer.empty() ? 0 : this->m_buffer.size() - 1); i++) {
auto count = ++heatMap[this->m_buffer[i] << 8 | heatMap[i + 1]];
this->m_highestCount = std::max(this->m_highestCount, count);
}
for (size_t i = 0; i < (this->m_buffer.empty() ? 0 : this->m_buffer.size() - 1); i++) {
this->m_opacityBuffer[i] = std::min(0.2F + (float(heatMap[this->m_buffer[i] << 8 | this->m_buffer[i + 1]]) / float(this->m_highestCount / 1000)), 1.0F);
}
}
private:
size_t m_sampleSize;
std::vector<u8> m_buffer;
std::vector<float> m_opacityBuffer;
size_t m_highestCount = 0;
std::atomic<bool> m_processing = false;
};
}

View File

@ -3,6 +3,8 @@
#include <hex/ui/view.hpp>
#include <hex/api/task.hpp>
#include "content/helpers/diagrams.hpp"
#include <array>
#include <atomic>
#include <cstdio>
@ -25,9 +27,9 @@ namespace hex::plugin::builtin {
float m_highestBlockEntropy = 0;
std::vector<float> m_blockEntropy;
std::array<std::vector<float>, 12> m_blockTypeDistributions;
u64 m_blockEntropyProcessedCount = 0;
u64 m_processedBlockCount = 0;
double m_entropyHandlePosition;
double m_diagramHandlePosition = 0.0;
std::array<ImU64, 256> m_valueCounts = { 0 };
TaskHolder m_analyzerTask;
@ -37,6 +39,9 @@ namespace hex::plugin::builtin {
std::string m_dataDescription;
std::string m_dataMimeType;
DiagramDigram m_digram;
DiagramLayeredDistribution m_layeredDistribution;
void analyze();
};

View File

@ -684,12 +684,14 @@
"hex.builtin.view.information.byte_types": "Byte types",
"hex.builtin.view.information.control": "Control",
"hex.builtin.view.information.description": "Description:",
"hex.builtin.view.information.digram": "Digram",
"hex.builtin.view.information.distribution": "Byte distribution",
"hex.builtin.view.information.encrypted": "This data is most likely encrypted or compressed!",
"hex.builtin.view.information.entropy": "Entropy",
"hex.builtin.view.information.file_entropy": "File entropy",
"hex.builtin.view.information.highest_entropy": "Highest entropy block",
"hex.builtin.view.information.info_analysis": "Information analysis",
"hex.builtin.view.information.layered_distribution": "Layered distribution",
"hex.builtin.view.information.magic": "Magic information",
"hex.builtin.view.information.magic_db_added": "Magic database added!",
"hex.builtin.view.information.mime": "MIME Type:",

View File

@ -8,6 +8,7 @@
#include <hex/providers/provider.hpp>
#include <content/helpers/provider_extra_data.hpp>
#include <content/helpers/diagrams.hpp>
#include <cctype>
#include <random>
@ -951,91 +952,21 @@ namespace hex::plugin::builtin {
NodeVisualizerDigram() : Node("hex.builtin.nodes.visualizer.digram.header", { dp::Attribute(dp::Attribute::IOType::In, dp::Attribute::Type::Buffer, "hex.builtin.nodes.common.input") }) { }
void drawNode() override {
drawDigram(scaled({ 200, 200 }));
this->m_digram.draw(scaled({ 200, 200 }));
if (ImGui::IsItemHovered() && ImGui::IsKeyDown(ImGuiKey_LeftShift)) {
ImGui::BeginTooltip();
drawDigram(scaled({ 600, 600 }));
this->m_digram.draw(scaled({ 600, 600 }));
ImGui::EndTooltip();
}
}
void drawDigram(const ImVec2 &viewSize) {
ImGui::PushStyleColor(ImGuiCol_ChildBg, ImU32(ImColor(0, 0, 0)));
if (ImGui::BeginChild("##visualizer", viewSize, true)) {
auto drawList = ImGui::GetWindowDrawList();
float xStep = (viewSize.x * 0.95F) / 0xFF;
float yStep = (viewSize.y * 0.95F) / 0xFF;
for (size_t i = 0; i < (this->m_buffer.empty() ? 0 : this->m_buffer.size() - 1); i++) {
const auto &[x, y] = std::pair { this->m_buffer[i] * xStep, this->m_buffer[i + 1] * yStep };
auto color = ImLerp(ImColor(0xFF, 0x6D, 0x01).Value, ImColor(0x01, 0x93, 0xFF).Value, float(i) / this->m_buffer.size());
color.w = this->m_opacityBuffer[i];
auto pos = ImGui::GetWindowPos() + ImVec2(viewSize.x * 0.025F, viewSize.y * 0.025F) + ImVec2(x, y);
drawList->AddRectFilled(pos, pos + ImVec2(xStep, yStep), ImColor(color));
}
}
ImGui::EndChild();
ImGui::PopStyleColor();
}
void process() override {
constexpr static auto SampleSize = 0x9000;
const static size_t SequenceCount = std::ceil(std::sqrt(SampleSize));
this->m_buffer.clear();
auto buffer = this->getBufferOnInput(0);
if (buffer.size() < SampleSize)
this->m_buffer = buffer;
else {
std::random_device randomDevice;
std::mt19937_64 random(randomDevice());
std::map<u64, std::vector<u8>> orderedData;
for (u32 i = 0; i < SequenceCount; i++) {
ssize_t offset = random() % buffer.size();
std::vector<u8> sequence;
sequence.reserve(SampleSize);
std::copy(buffer.begin() + offset, buffer.begin() + offset + std::min<size_t>(SequenceCount, buffer.size() - offset), std::back_inserter(sequence));
orderedData.insert({ offset, sequence });
}
this->m_buffer.reserve(SampleSize);
u64 lastEnd = 0x00;
for (const auto &[offset, sequence] : orderedData) {
if (offset < lastEnd)
this->m_buffer.resize(this->m_buffer.size() - (lastEnd - offset));
std::copy(sequence.begin(), sequence.end(), std::back_inserter(this->m_buffer));
lastEnd = offset + sequence.size();
}
}
this->m_opacityBuffer.resize(this->m_buffer.size());
std::map<u64, size_t> heatMap;
for (size_t i = 0; i < (this->m_buffer.empty() ? 0 : this->m_buffer.size() - 1); i++) {
auto count = ++heatMap[this->m_buffer[i] << 8 | heatMap[i + 1]];
this->m_highestCount = std::max(this->m_highestCount, count);
}
for (size_t i = 0; i < (this->m_buffer.empty() ? 0 : this->m_buffer.size() - 1); i++) {
this->m_opacityBuffer[i] = std::min(0.2F + (float(heatMap[this->m_buffer[i] << 8 | this->m_buffer[i + 1]]) / float(this->m_highestCount / 1000)), 1.0F);
}
this->m_digram.process(this->getBufferOnInput(0));
}
private:
std::vector<u8> m_buffer;
std::vector<float> m_opacityBuffer;
size_t m_highestCount = 0;
DiagramDigram m_digram;
};
class NodeVisualizerLayeredDistribution : public dp::Node {
@ -1043,90 +974,20 @@ namespace hex::plugin::builtin {
NodeVisualizerLayeredDistribution() : Node("hex.builtin.nodes.visualizer.layered_dist.header", { dp::Attribute(dp::Attribute::IOType::In, dp::Attribute::Type::Buffer, "hex.builtin.nodes.common.input") }) { }
void drawNode() override {
drawLayeredDistribution(scaled({ 200, 200 }));
this->m_layeredDistribution.draw(scaled({ 200, 200 }));
if (ImGui::IsItemHovered() && ImGui::IsKeyDown(ImGuiKey_LeftShift)) {
ImGui::BeginTooltip();
drawLayeredDistribution(scaled({ 600, 600 }));
this->m_layeredDistribution.draw(scaled({ 600, 600 }));
ImGui::EndTooltip();
}
}
void drawLayeredDistribution(const ImVec2 &viewSize) {
ImGui::PushStyleColor(ImGuiCol_ChildBg, ImU32(ImColor(0, 0, 0)));
if (ImGui::BeginChild("##visualizer", viewSize, true)) {
auto drawList = ImGui::GetWindowDrawList();
float xStep = (viewSize.x * 0.95F) / 0xFF;
float yStep = (viewSize.y * 0.95F) / 0xFF;
for (size_t i = 0; i < (this->m_buffer.empty() ? 0 : this->m_buffer.size()); i++) {
const auto &[x, y] = std::pair { this->m_buffer[i] * xStep, yStep * ((float(i) / this->m_buffer.size()) * 0xFF) };
auto color = ImLerp(ImColor(0xFF, 0x6D, 0x01).Value, ImColor(0x01, 0x93, 0xFF).Value, float(i) / this->m_buffer.size());
color.w = this->m_opacityBuffer[i];
auto pos = ImGui::GetWindowPos() + ImVec2(viewSize.x * 0.025F, viewSize.y * 0.025F) + ImVec2(x, y);
drawList->AddRectFilled(pos, pos + ImVec2(xStep, yStep), ImColor(color));
}
}
ImGui::EndChild();
ImGui::PopStyleColor();
}
void process() override {
constexpr static auto SampleSize = 0x9000;
const static size_t SequenceCount = std::ceil(std::sqrt(SampleSize));
this->m_buffer.clear();
auto buffer = this->getBufferOnInput(0);
if (buffer.size() < SampleSize)
this->m_buffer = buffer;
else {
std::random_device randomDevice;
std::mt19937_64 random(randomDevice());
std::map<u64, std::vector<u8>> orderedData;
for (u32 i = 0; i < SequenceCount; i++) {
ssize_t offset = random() % buffer.size();
std::vector<u8> sequence;
sequence.reserve(SampleSize);
std::copy(buffer.begin() + offset, buffer.begin() + offset + std::min<size_t>(SequenceCount, buffer.size() - offset), std::back_inserter(sequence));
orderedData.insert({ offset, sequence });
}
this->m_buffer.reserve(SampleSize);
u64 lastEnd = 0x00;
for (const auto &[offset, sequence] : orderedData) {
if (offset < lastEnd)
this->m_buffer.resize(this->m_buffer.size() - (lastEnd - offset));
std::copy(sequence.begin(), sequence.end(), std::back_inserter(this->m_buffer));
lastEnd = offset + sequence.size();
}
}
this->m_opacityBuffer.resize(this->m_buffer.size());
std::map<u64, size_t> heatMap;
for (size_t i = 0; i < (this->m_buffer.empty() ? 0 : this->m_buffer.size() - 1); i++) {
auto count = ++heatMap[this->m_buffer[i] << 8 | heatMap[i + 1]];
this->m_highestCount = std::max(this->m_highestCount, count);
}
for (size_t i = 0; i < (this->m_buffer.empty() ? 0 : this->m_buffer.size() - 1); i++) {
this->m_opacityBuffer[i] = std::min(0.2F + (float(heatMap[this->m_buffer[i] << 8 | this->m_buffer[i + 1]]) / float(this->m_highestCount / 1000)), 1.0F);
}
this->m_layeredDistribution.process(this->getBufferOnInput(0));
}
private:
std::vector<u8> m_buffer;
std::vector<float> m_opacityBuffer;
size_t m_highestCount = 0;
DiagramLayeredDistribution m_layeredDistribution;
};
class NodeVisualizerImage : public dp::Node {

View File

@ -35,7 +35,7 @@ namespace hex::plugin::builtin {
EventManager::subscribe<EventRegionSelected>(this, [this](Region region) {
if (this->m_blockSize != 0)
this->m_entropyHandlePosition = region.getStartAddress() / double(this->m_blockSize);
this->m_diagramHandlePosition = region.getStartAddress() / double(this->m_blockSize);
});
EventManager::subscribe<EventProviderDeleted>(this, [this](const auto*) {
@ -139,11 +139,19 @@ namespace hex::plugin::builtin {
std::array<ImU64, 256> blockValueCounts = { 0 };
const auto blockCount = provider->getSize() / this->m_blockSize;
this->m_blockTypeDistributions.fill({});
this->m_blockEntropy.clear();
this->m_blockEntropy.resize(provider->getSize() / this->m_blockSize);
this->m_blockEntropy.resize(blockCount);
for (auto &blockDistribution : this->m_blockTypeDistributions)
blockDistribution.resize(blockCount);
this->m_valueCounts.fill(0);
this->m_blockEntropyProcessedCount = 0;
this->m_processedBlockCount = 0;
this->m_digram.process(provider, this->m_analyzedRegion.getStartAddress(), this->m_analyzedRegion.getSize());
this->m_layeredDistribution.process(provider, this->m_analyzedRegion.getStartAddress(), this->m_analyzedRegion.getSize());
auto reader = prv::BufferedReader(provider);
reader.setEndAddress(provider->getBaseAddress() + provider->getSize());
@ -155,15 +163,15 @@ namespace hex::plugin::builtin {
count++;
if ((count % this->m_blockSize) == 0) [[unlikely]] {
this->m_blockEntropy[this->m_blockEntropyProcessedCount] = calculateEntropy(blockValueCounts, this->m_blockSize);
this->m_blockEntropy[this->m_processedBlockCount] = calculateEntropy(blockValueCounts, this->m_blockSize);
{
auto typeDist = calculateTypeDistribution(blockValueCounts, this->m_blockSize);
for (u8 i = 0; i < typeDist.size(); i++)
this->m_blockTypeDistributions[i].push_back(typeDist[i]);
this->m_blockTypeDistributions[i][this->m_processedBlockCount] = typeDist[i];
}
this->m_blockEntropyProcessedCount += 1;
this->m_processedBlockCount += 1;
blockValueCounts = { 0 };
task.update(count);
}
@ -286,11 +294,11 @@ namespace hex::plugin::builtin {
constexpr static std::array Names = { "iscntrl", "isprint", "isspace", "isblank", "isgraph", "ispunct", "isalnum", "isalpha", "isupper", "islower", "isdigit", "isxdigit" };
for (u32 i = 0; i < 12; i++) {
ImPlot::PlotLine(Names[i], this->m_blockTypeDistributions[i].data(), this->m_blockTypeDistributions[i].size());
ImPlot::PlotLine(Names[i], this->m_blockTypeDistributions[i].data(), this->m_processedBlockCount);
}
if (ImPlot::DragLineX(1, &this->m_entropyHandlePosition, ImGui::GetStyleColorVec4(ImGuiCol_Text))) {
u64 address = u64(std::max<double>(this->m_entropyHandlePosition, 0) * this->m_blockSize) + provider->getBaseAddress();
if (ImPlot::DragLineX(1, &this->m_diagramHandlePosition, ImGui::GetStyleColorVec4(ImGuiCol_Text))) {
u64 address = u64(std::max<double>(this->m_diagramHandlePosition, 0) * this->m_blockSize) + provider->getBaseAddress();
address = std::min(address, provider->getBaseAddress() + provider->getSize() - 1);
ImHexApi::HexEditor::setSelection(address, 1);
}
@ -306,21 +314,25 @@ namespace hex::plugin::builtin {
ImPlot::SetupAxes("hex.builtin.common.address"_lang, "hex.builtin.view.information.entropy"_lang, ImPlotAxisFlags_Lock, ImPlotAxisFlags_Lock);
ImPlot::SetupAxesLimits(0, this->m_blockEntropy.size(), -0.1F, 1.1F, ImGuiCond_Always);
ImPlot::PlotLine("##entropy_line", this->m_blockEntropy.data(), this->m_blockEntropyProcessedCount);
ImPlot::PlotLine("##entropy_line", this->m_blockEntropy.data(), this->m_processedBlockCount);
if (ImPlot::DragLineX(1, &this->m_entropyHandlePosition, ImGui::GetStyleColorVec4(ImGuiCol_Text))) {
u64 address = u64(std::max<double>(this->m_entropyHandlePosition, 0) * this->m_blockSize) + provider->getBaseAddress();
if (ImPlot::DragLineX(1, &this->m_diagramHandlePosition, ImGui::GetStyleColorVec4(ImGuiCol_Text))) {
u64 address = u64(std::max<double>(this->m_diagramHandlePosition, 0) * this->m_blockSize) + provider->getBaseAddress();
address = std::min(address, provider->getBaseAddress() + provider->getSize() - 1);
ImHexApi::HexEditor::setSelection(address, 1);
}
ImPlot::EndPlot();
}
ImPlot::PopStyleColor();
ImGui::PopStyleColor();
ImGui::NewLine();
this->m_diagramHandlePosition = std::clamp<double>(
this->m_diagramHandlePosition,
this->m_analyzedRegion.getStartAddress() / double(this->m_blockSize),
this->m_analyzedRegion.getEndAddress() / double(this->m_blockSize));
}
// Entropy information
@ -352,7 +364,22 @@ namespace hex::plugin::builtin {
ImGui::NewLine();
ImGui::TextFormattedColored(ImVec4(0.92F, 0.25F, 0.2F, 1.0F), "{}", "hex.builtin.view.information.encrypted"_lang);
}
}
ImGui::BeginGroup();
{
ImGui::Header("hex.builtin.view.information.digram"_lang);
this->m_digram.draw(ImVec2(300, 300));
}
ImGui::EndGroup();
ImGui::SameLine();
ImGui::BeginGroup();
{
ImGui::Header("hex.builtin.view.information.layered_distribution"_lang);
this->m_layeredDistribution.draw(ImVec2(300, 300));
}
ImGui::EndGroup(); }
}
}
ImGui::EndChild();