Refactored advanced entropy operation into entropy operation
This commit is contained in:
mshwed
parent
b99af58636
commit
1b161f997b
41
package-lock.json
generated
41
package-lock.json
generated
@ -5353,8 +5353,7 @@
|
|||||||
"ansi-regex": {
|
"ansi-regex": {
|
||||||
"version": "2.1.1",
|
"version": "2.1.1",
|
||||||
"bundled": true,
|
"bundled": true,
|
||||||
"dev": true,
|
"dev": true
|
||||||
"optional": true
|
|
||||||
},
|
},
|
||||||
"aproba": {
|
"aproba": {
|
||||||
"version": "1.2.0",
|
"version": "1.2.0",
|
||||||
@ -5375,14 +5374,12 @@
|
|||||||
"balanced-match": {
|
"balanced-match": {
|
||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"bundled": true,
|
"bundled": true,
|
||||||
"dev": true,
|
"dev": true
|
||||||
"optional": true
|
|
||||||
},
|
},
|
||||||
"brace-expansion": {
|
"brace-expansion": {
|
||||||
"version": "1.1.11",
|
"version": "1.1.11",
|
||||||
"bundled": true,
|
"bundled": true,
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"optional": true,
|
|
||||||
"requires": {
|
"requires": {
|
||||||
"balanced-match": "^1.0.0",
|
"balanced-match": "^1.0.0",
|
||||||
"concat-map": "0.0.1"
|
"concat-map": "0.0.1"
|
||||||
@ -5397,20 +5394,17 @@
|
|||||||
"code-point-at": {
|
"code-point-at": {
|
||||||
"version": "1.1.0",
|
"version": "1.1.0",
|
||||||
"bundled": true,
|
"bundled": true,
|
||||||
"dev": true,
|
"dev": true
|
||||||
"optional": true
|
|
||||||
},
|
},
|
||||||
"concat-map": {
|
"concat-map": {
|
||||||
"version": "0.0.1",
|
"version": "0.0.1",
|
||||||
"bundled": true,
|
"bundled": true,
|
||||||
"dev": true,
|
"dev": true
|
||||||
"optional": true
|
|
||||||
},
|
},
|
||||||
"console-control-strings": {
|
"console-control-strings": {
|
||||||
"version": "1.1.0",
|
"version": "1.1.0",
|
||||||
"bundled": true,
|
"bundled": true,
|
||||||
"dev": true,
|
"dev": true
|
||||||
"optional": true
|
|
||||||
},
|
},
|
||||||
"core-util-is": {
|
"core-util-is": {
|
||||||
"version": "1.0.2",
|
"version": "1.0.2",
|
||||||
@ -5527,8 +5521,7 @@
|
|||||||
"inherits": {
|
"inherits": {
|
||||||
"version": "2.0.3",
|
"version": "2.0.3",
|
||||||
"bundled": true,
|
"bundled": true,
|
||||||
"dev": true,
|
"dev": true
|
||||||
"optional": true
|
|
||||||
},
|
},
|
||||||
"ini": {
|
"ini": {
|
||||||
"version": "1.3.5",
|
"version": "1.3.5",
|
||||||
@ -5540,7 +5533,6 @@
|
|||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"bundled": true,
|
"bundled": true,
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"optional": true,
|
|
||||||
"requires": {
|
"requires": {
|
||||||
"number-is-nan": "^1.0.0"
|
"number-is-nan": "^1.0.0"
|
||||||
}
|
}
|
||||||
@ -5555,7 +5547,6 @@
|
|||||||
"version": "3.0.4",
|
"version": "3.0.4",
|
||||||
"bundled": true,
|
"bundled": true,
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"optional": true,
|
|
||||||
"requires": {
|
"requires": {
|
||||||
"brace-expansion": "^1.1.7"
|
"brace-expansion": "^1.1.7"
|
||||||
}
|
}
|
||||||
@ -5563,14 +5554,12 @@
|
|||||||
"minimist": {
|
"minimist": {
|
||||||
"version": "0.0.8",
|
"version": "0.0.8",
|
||||||
"bundled": true,
|
"bundled": true,
|
||||||
"dev": true,
|
"dev": true
|
||||||
"optional": true
|
|
||||||
},
|
},
|
||||||
"minipass": {
|
"minipass": {
|
||||||
"version": "2.3.5",
|
"version": "2.3.5",
|
||||||
"bundled": true,
|
"bundled": true,
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"optional": true,
|
|
||||||
"requires": {
|
"requires": {
|
||||||
"safe-buffer": "^5.1.2",
|
"safe-buffer": "^5.1.2",
|
||||||
"yallist": "^3.0.0"
|
"yallist": "^3.0.0"
|
||||||
@ -5589,7 +5578,6 @@
|
|||||||
"version": "0.5.1",
|
"version": "0.5.1",
|
||||||
"bundled": true,
|
"bundled": true,
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"optional": true,
|
|
||||||
"requires": {
|
"requires": {
|
||||||
"minimist": "0.0.8"
|
"minimist": "0.0.8"
|
||||||
}
|
}
|
||||||
@ -5670,8 +5658,7 @@
|
|||||||
"number-is-nan": {
|
"number-is-nan": {
|
||||||
"version": "1.0.1",
|
"version": "1.0.1",
|
||||||
"bundled": true,
|
"bundled": true,
|
||||||
"dev": true,
|
"dev": true
|
||||||
"optional": true
|
|
||||||
},
|
},
|
||||||
"object-assign": {
|
"object-assign": {
|
||||||
"version": "4.1.1",
|
"version": "4.1.1",
|
||||||
@ -5683,7 +5670,6 @@
|
|||||||
"version": "1.4.0",
|
"version": "1.4.0",
|
||||||
"bundled": true,
|
"bundled": true,
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"optional": true,
|
|
||||||
"requires": {
|
"requires": {
|
||||||
"wrappy": "1"
|
"wrappy": "1"
|
||||||
}
|
}
|
||||||
@ -5769,8 +5755,7 @@
|
|||||||
"safe-buffer": {
|
"safe-buffer": {
|
||||||
"version": "5.1.2",
|
"version": "5.1.2",
|
||||||
"bundled": true,
|
"bundled": true,
|
||||||
"dev": true,
|
"dev": true
|
||||||
"optional": true
|
|
||||||
},
|
},
|
||||||
"safer-buffer": {
|
"safer-buffer": {
|
||||||
"version": "2.1.2",
|
"version": "2.1.2",
|
||||||
@ -5806,7 +5791,6 @@
|
|||||||
"version": "1.0.2",
|
"version": "1.0.2",
|
||||||
"bundled": true,
|
"bundled": true,
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"optional": true,
|
|
||||||
"requires": {
|
"requires": {
|
||||||
"code-point-at": "^1.0.0",
|
"code-point-at": "^1.0.0",
|
||||||
"is-fullwidth-code-point": "^1.0.0",
|
"is-fullwidth-code-point": "^1.0.0",
|
||||||
@ -5826,7 +5810,6 @@
|
|||||||
"version": "3.0.1",
|
"version": "3.0.1",
|
||||||
"bundled": true,
|
"bundled": true,
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"optional": true,
|
|
||||||
"requires": {
|
"requires": {
|
||||||
"ansi-regex": "^2.0.0"
|
"ansi-regex": "^2.0.0"
|
||||||
}
|
}
|
||||||
@ -5870,14 +5853,12 @@
|
|||||||
"wrappy": {
|
"wrappy": {
|
||||||
"version": "1.0.2",
|
"version": "1.0.2",
|
||||||
"bundled": true,
|
"bundled": true,
|
||||||
"dev": true,
|
"dev": true
|
||||||
"optional": true
|
|
||||||
},
|
},
|
||||||
"yallist": {
|
"yallist": {
|
||||||
"version": "3.0.3",
|
"version": "3.0.3",
|
||||||
"bundled": true,
|
"bundled": true,
|
||||||
"dev": true,
|
"dev": true
|
||||||
"optional": true
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|||||||
@ -1,361 +0,0 @@
|
|||||||
/**
|
|
||||||
* @author mshwed [m@ttshwed.com]
|
|
||||||
* @copyright Crown Copyright 2019
|
|
||||||
* @license Apache-2.0
|
|
||||||
*/
|
|
||||||
|
|
||||||
import * as d3temp from "d3";
|
|
||||||
import * as nodomtemp from "nodom";
|
|
||||||
|
|
||||||
import Operation from "../Operation";
|
|
||||||
import Utils from "../Utils";
|
|
||||||
|
|
||||||
const d3 = d3temp.default ? d3temp.default : d3temp;
|
|
||||||
const nodom = nodomtemp.default ? nodomtemp.default: nodomtemp;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Advanced Entropy operation
|
|
||||||
*/
|
|
||||||
class AdvancedEntropy extends Operation {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* AdvancedEntropy constructor
|
|
||||||
*/
|
|
||||||
constructor() {
|
|
||||||
super();
|
|
||||||
|
|
||||||
this.name = "Advanced Entropy";
|
|
||||||
this.module = "Default";
|
|
||||||
this.description = "Shannon Entropy, in the context of information theory, is a measure of the rate at which information is produced by a source of data. It can be used, in a broad sense, to detect whether data is likely to be structured or unstructured. 8 is the maximum, representing highly unstructured, 'random' data. English language text usually falls somewhere between 3.5 and 5. Properly encrypted or compressed data should have an entropy of over 7.5.";
|
|
||||||
this.infoURL = "https://wikipedia.org/wiki/Entropy_(information_theory)";
|
|
||||||
this.inputType = "byteArray";
|
|
||||||
this.outputType = "html";
|
|
||||||
this.args = [
|
|
||||||
{
|
|
||||||
"name": "Visualization",
|
|
||||||
"type": "option",
|
|
||||||
"value": ["Histogram (Bar)", "Histogram (Line)", "Curve", "Image"]
|
|
||||||
}
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Calculates the frequency of bytes in the input.
|
|
||||||
*
|
|
||||||
* @param {byteArray} input
|
|
||||||
* @returns {frequency}
|
|
||||||
*/
|
|
||||||
calculateShannonEntropy(input) {
|
|
||||||
const prob = [],
|
|
||||||
uniques = input.unique(),
|
|
||||||
str = Utils.byteArrayToChars(input);
|
|
||||||
|
|
||||||
let i;
|
|
||||||
for (i = 0; i < uniques.length; i++) {
|
|
||||||
prob.push(str.count(Utils.chr(uniques[i])) / input.length);
|
|
||||||
}
|
|
||||||
|
|
||||||
let entropy = 0,
|
|
||||||
p;
|
|
||||||
|
|
||||||
for (i = 0; i < prob.length; i++) {
|
|
||||||
p = prob[i];
|
|
||||||
entropy += p * Math.log(p) / Math.log(2);
|
|
||||||
}
|
|
||||||
|
|
||||||
return -entropy;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
* @param inputBytes
|
|
||||||
* @returns {entropyData}
|
|
||||||
*/
|
|
||||||
calculateScanningEntropy(inputBytes, binWidth) {
|
|
||||||
const entropyData = [];
|
|
||||||
|
|
||||||
if (inputBytes.length < 256) binWidth = 8;
|
|
||||||
else binWidth = 256;
|
|
||||||
|
|
||||||
for (let bytePos = 0; bytePos < inputBytes.length; bytePos+=binWidth) {
|
|
||||||
const block = inputBytes.slice(bytePos, bytePos+binWidth);
|
|
||||||
entropyData.push(this.calculateShannonEntropy(block));
|
|
||||||
}
|
|
||||||
|
|
||||||
return { entropyData, binWidth };
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Calculates the frequency of bytes in the input.
|
|
||||||
*
|
|
||||||
* @param {object} svg
|
|
||||||
* @param {function} xScale
|
|
||||||
* @param {function} yScale
|
|
||||||
* @param {integer} svgHeight
|
|
||||||
* @param {integer} svgWidth
|
|
||||||
* @param {object} margins
|
|
||||||
* @param {string} xTitle
|
|
||||||
* @param {string} yTitle
|
|
||||||
* @returns {undefined}
|
|
||||||
*/
|
|
||||||
createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, title, xTitle, yTitle) {
|
|
||||||
// Axes
|
|
||||||
const yAxis = d3.axisLeft()
|
|
||||||
.scale(yScale);
|
|
||||||
|
|
||||||
const xAxis = d3.axisBottom()
|
|
||||||
.scale(xScale);
|
|
||||||
|
|
||||||
svg.append("g")
|
|
||||||
.attr("transform", `translate(0, ${svgHeight - margins.bottom})`)
|
|
||||||
.call(xAxis);
|
|
||||||
|
|
||||||
svg.append("g")
|
|
||||||
.attr("transform", `translate(${margins.left},0)`)
|
|
||||||
.call(yAxis);
|
|
||||||
|
|
||||||
// Axes labels
|
|
||||||
svg.append("text")
|
|
||||||
.attr("transform", "rotate(-90)")
|
|
||||||
.attr("y", 0 - margins.left)
|
|
||||||
.attr("x", 0 - (svgHeight / 2))
|
|
||||||
.attr("dy", "1em")
|
|
||||||
.style("text-anchor", "middle")
|
|
||||||
.text(yTitle);
|
|
||||||
|
|
||||||
svg.append("text")
|
|
||||||
.attr("transform", `translate(${svgWidth / 2}, ${svgHeight - margins.bottom + 40})`)
|
|
||||||
.style("text-anchor", "middle")
|
|
||||||
.text(xTitle);
|
|
||||||
|
|
||||||
// Add title
|
|
||||||
svg.append("text")
|
|
||||||
.attr("transform", `translate(${svgWidth / 2}, ${margins.top - 10})`)
|
|
||||||
.style("text-anchor", "middle")
|
|
||||||
.text(title);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Calculates the frequency of bytes in the input.
|
|
||||||
*
|
|
||||||
* @param {byteArray} inputBytes
|
|
||||||
* @returns {frequency}
|
|
||||||
*/
|
|
||||||
calculateByteFrequency(inputBytes) {
|
|
||||||
const byteFrequency = [];
|
|
||||||
for (let i = 0; i < 256; i++) {
|
|
||||||
if (inputBytes.length > 0) {
|
|
||||||
let count = 0;
|
|
||||||
for (const byte of inputBytes) {
|
|
||||||
if (byte === i) count++;
|
|
||||||
}
|
|
||||||
byteFrequency.push(count / inputBytes.length);
|
|
||||||
} else {
|
|
||||||
byteFrequency.push(0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return byteFrequency;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Calculates the frequency of bytes in the input.
|
|
||||||
*
|
|
||||||
* @param {byteArray} byteFrequency
|
|
||||||
* @returns {frequency}
|
|
||||||
*/
|
|
||||||
createByteFrequencyLineHistogram(byteFrequency) {
|
|
||||||
const margins = { top: 30, right: 20, bottom: 50, left: 30 };
|
|
||||||
|
|
||||||
const svgWidth = 500,
|
|
||||||
svgHeight = 500;
|
|
||||||
|
|
||||||
const document = new nodom.Document();
|
|
||||||
let svg = document.createElement("svg");
|
|
||||||
|
|
||||||
svg = d3.select(svg)
|
|
||||||
.attr("width", "100%")
|
|
||||||
.attr("height", "100%")
|
|
||||||
.attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`);
|
|
||||||
|
|
||||||
const yScale = d3.scaleLinear()
|
|
||||||
.domain([0, d3.max(byteFrequency, d => d)])
|
|
||||||
.range([svgHeight - margins.bottom, margins.top]);
|
|
||||||
|
|
||||||
const xScale = d3.scaleLinear()
|
|
||||||
.domain([0, byteFrequency.length - 1])
|
|
||||||
.range([margins.left, svgWidth - margins.right]);
|
|
||||||
|
|
||||||
const line = d3.line()
|
|
||||||
.x((_, i) => xScale(i))
|
|
||||||
.y(d => yScale(d))
|
|
||||||
.curve(d3.curveMonotoneX);
|
|
||||||
|
|
||||||
svg.append("path")
|
|
||||||
.datum(byteFrequency)
|
|
||||||
.attr("fill", "none")
|
|
||||||
.attr("stroke", "steelblue")
|
|
||||||
.attr("d", line);
|
|
||||||
|
|
||||||
this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "", "Byte", "Byte Frequency");
|
|
||||||
|
|
||||||
return svg._groups[0][0].outerHTML;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a byte frequency histogram
|
|
||||||
*
|
|
||||||
* @param {byteArray} byteFrequency
|
|
||||||
* @returns {HTML}
|
|
||||||
*/
|
|
||||||
createByteFrequencyBarHistogram(byteFrequency) {
|
|
||||||
const margins = { top: 30, right: 20, bottom: 50, left: 30 };
|
|
||||||
|
|
||||||
const svgWidth = 500,
|
|
||||||
svgHeight = 500,
|
|
||||||
binWidth = 1;
|
|
||||||
|
|
||||||
const document = new nodom.Document();
|
|
||||||
let svg = document.createElement("svg");
|
|
||||||
svg = d3.select(svg)
|
|
||||||
.attr("width", "100%")
|
|
||||||
.attr("height", "100%")
|
|
||||||
.attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`);
|
|
||||||
|
|
||||||
const yExtent = d3.extent(byteFrequency, d => d);
|
|
||||||
const yScale = d3.scaleLinear()
|
|
||||||
.domain(yExtent)
|
|
||||||
.range([svgHeight - margins.bottom, margins.top]);
|
|
||||||
|
|
||||||
const xScale = d3.scaleLinear()
|
|
||||||
.domain([0, byteFrequency.length - 1])
|
|
||||||
.range([margins.left - binWidth, svgWidth - margins.right]);
|
|
||||||
|
|
||||||
svg.selectAll("rect")
|
|
||||||
.data(byteFrequency)
|
|
||||||
.enter().append("rect")
|
|
||||||
.attr("x", (_, i) => xScale(i) + binWidth)
|
|
||||||
.attr("y", dataPoint => yScale(dataPoint))
|
|
||||||
.attr("width", binWidth)
|
|
||||||
.attr("height", dataPoint => yScale(yExtent[0]) - yScale(dataPoint))
|
|
||||||
.attr("fill", "blue");
|
|
||||||
|
|
||||||
this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "", "Byte", "Byte Frequency");
|
|
||||||
|
|
||||||
return svg._groups[0][0].outerHTML;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a byte frequency histogram
|
|
||||||
*
|
|
||||||
* @param {byteArray} input
|
|
||||||
* @param {number} blockSize
|
|
||||||
* @returns {HTML}
|
|
||||||
*/
|
|
||||||
createEntropyCurve(entropyData) {
|
|
||||||
const margins = { top: 30, right: 20, bottom: 50, left: 30 };
|
|
||||||
|
|
||||||
const svgWidth = 500,
|
|
||||||
svgHeight = 500;
|
|
||||||
|
|
||||||
const document = new nodom.Document();
|
|
||||||
let svg = document.createElement("svg");
|
|
||||||
svg = d3.select(svg)
|
|
||||||
.attr("width", "100%")
|
|
||||||
.attr("height", "100%")
|
|
||||||
.attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`);
|
|
||||||
|
|
||||||
const yScale = d3.scaleLinear()
|
|
||||||
.domain([0, d3.max(entropyData, d => d)])
|
|
||||||
.range([svgHeight - margins.bottom, margins.top]);
|
|
||||||
|
|
||||||
const xScale = d3.scaleLinear()
|
|
||||||
.domain([0, entropyData.length])
|
|
||||||
.range([margins.left, svgWidth - margins.right]);
|
|
||||||
|
|
||||||
const line = d3.line()
|
|
||||||
.x((_, i) => xScale(i))
|
|
||||||
.y(d => yScale(d))
|
|
||||||
.curve(d3.curveMonotoneX);
|
|
||||||
|
|
||||||
if (entropyData.length > 0) {
|
|
||||||
svg.append("path")
|
|
||||||
.datum(entropyData)
|
|
||||||
.attr("d", line);
|
|
||||||
|
|
||||||
svg.selectAll("path").attr("fill", "none").attr("stroke", "steelblue");
|
|
||||||
}
|
|
||||||
|
|
||||||
this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "Scanning Entropy", "Block", "Entropy");
|
|
||||||
|
|
||||||
return svg._groups[0][0].outerHTML;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates an image representation of the entropy
|
|
||||||
*
|
|
||||||
* @param {byteArray} input
|
|
||||||
* @param {number} blockSize
|
|
||||||
* @returns {HTML}
|
|
||||||
*/
|
|
||||||
createEntropyImage(entropyData) {
|
|
||||||
const svgHeight = 100,
|
|
||||||
svgWidth = 100,
|
|
||||||
cellSize = 1,
|
|
||||||
nodes = [];
|
|
||||||
|
|
||||||
for (let i = 0; i < entropyData.length; i++) {
|
|
||||||
nodes.push({
|
|
||||||
x: i % svgWidth,
|
|
||||||
y: Math.floor(i / svgWidth),
|
|
||||||
entropy: entropyData[i]
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const document = new nodom.Document();
|
|
||||||
let svg = document.createElement("svg");
|
|
||||||
svg = d3.select(svg)
|
|
||||||
.attr("width", "100%")
|
|
||||||
.attr("height", "100%")
|
|
||||||
.attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`);
|
|
||||||
|
|
||||||
const greyScale = d3.scaleLinear()
|
|
||||||
.domain([0, d3.max(entropyData, d => d)])
|
|
||||||
.range(["#000000", "#FFFFFF"])
|
|
||||||
.interpolate(d3.interpolateRgb);
|
|
||||||
|
|
||||||
svg
|
|
||||||
.selectAll("rect")
|
|
||||||
.data(nodes)
|
|
||||||
.enter().append("rect")
|
|
||||||
.attr("x", d => d.x * cellSize)
|
|
||||||
.attr("y", d => d.y * cellSize)
|
|
||||||
.attr("width", cellSize)
|
|
||||||
.attr("height", cellSize)
|
|
||||||
.style("fill", d => greyScale(d.entropy));
|
|
||||||
|
|
||||||
return svg._groups[0][0].outerHTML;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {byteArray} input
|
|
||||||
* @param {Object[]} args
|
|
||||||
* @returns {html}
|
|
||||||
*/
|
|
||||||
run(input, args) {
|
|
||||||
const visualizationType = args[0];
|
|
||||||
|
|
||||||
if (visualizationType === "Histogram (Bar)") {
|
|
||||||
return this.createByteFrequencyBarHistogram(this.calculateByteFrequency(input));
|
|
||||||
} else if (visualizationType === "Histogram (Line)") {
|
|
||||||
return this.createByteFrequencyLineHistogram(this.calculateByteFrequency(input));
|
|
||||||
} else if (visualizationType === "Curve") {
|
|
||||||
return this.createEntropyCurve(this.calculateScanningEntropy(input).entropyData);
|
|
||||||
} else if (visualizationType === "Image") {
|
|
||||||
return this.createEntropyImage(this.calculateScanningEntropy(input).entropyData);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export default AdvancedEntropy;
|
|
||||||
@ -4,9 +4,15 @@
|
|||||||
* @license Apache-2.0
|
* @license Apache-2.0
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import * as d3temp from "d3";
|
||||||
|
import * as nodomtemp from "nodom";
|
||||||
|
|
||||||
import Operation from "../Operation";
|
import Operation from "../Operation";
|
||||||
import Utils from "../Utils";
|
import Utils from "../Utils";
|
||||||
|
|
||||||
|
const d3 = d3temp.default ? d3temp.default : d3temp;
|
||||||
|
const nodom = nodomtemp.default ? nodomtemp.default: nodomtemp;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Entropy operation
|
* Entropy operation
|
||||||
*/
|
*/
|
||||||
@ -23,22 +29,28 @@ class Entropy extends Operation {
|
|||||||
this.description = "Shannon Entropy, in the context of information theory, is a measure of the rate at which information is produced by a source of data. It can be used, in a broad sense, to detect whether data is likely to be structured or unstructured. 8 is the maximum, representing highly unstructured, 'random' data. English language text usually falls somewhere between 3.5 and 5. Properly encrypted or compressed data should have an entropy of over 7.5.";
|
this.description = "Shannon Entropy, in the context of information theory, is a measure of the rate at which information is produced by a source of data. It can be used, in a broad sense, to detect whether data is likely to be structured or unstructured. 8 is the maximum, representing highly unstructured, 'random' data. English language text usually falls somewhere between 3.5 and 5. Properly encrypted or compressed data should have an entropy of over 7.5.";
|
||||||
this.infoURL = "https://wikipedia.org/wiki/Entropy_(information_theory)";
|
this.infoURL = "https://wikipedia.org/wiki/Entropy_(information_theory)";
|
||||||
this.inputType = "byteArray";
|
this.inputType = "byteArray";
|
||||||
this.outputType = "number";
|
this.outputType = "html";
|
||||||
this.presentType = "html";
|
this.args = [
|
||||||
this.args = [];
|
{
|
||||||
|
"name": "Visualisation",
|
||||||
|
"type": "option",
|
||||||
|
"value": ["Shannon", "Histogram (Bar)", "Histogram (Line)", "Curve", "Image"]
|
||||||
|
}
|
||||||
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
* Calculates the frequency of bytes in the input.
|
||||||
|
*
|
||||||
* @param {byteArray} input
|
* @param {byteArray} input
|
||||||
* @param {Object[]} args
|
* @returns {frequency}
|
||||||
* @returns {number}
|
|
||||||
*/
|
*/
|
||||||
run(input, args) {
|
calculateShannonEntropy(input) {
|
||||||
const prob = [],
|
const prob = [],
|
||||||
uniques = input.unique(),
|
uniques = input.unique(),
|
||||||
str = Utils.byteArrayToChars(input);
|
str = Utils.byteArrayToChars(input);
|
||||||
let i;
|
|
||||||
|
|
||||||
|
let i;
|
||||||
for (i = 0; i < uniques.length; i++) {
|
for (i = 0; i < uniques.length; i++) {
|
||||||
prob.push(str.count(Utils.chr(uniques[i])) / input.length);
|
prob.push(str.count(Utils.chr(uniques[i])) / input.length);
|
||||||
}
|
}
|
||||||
@ -54,13 +66,284 @@ class Entropy extends Operation {
|
|||||||
return -entropy;
|
return -entropy;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param inputBytes
|
||||||
|
* @returns {entropyData}
|
||||||
|
*/
|
||||||
|
calculateScanningEntropy(inputBytes, binWidth) {
|
||||||
|
const entropyData = [];
|
||||||
|
|
||||||
|
if (inputBytes.length < 256) binWidth = 8;
|
||||||
|
else binWidth = 256;
|
||||||
|
|
||||||
|
for (let bytePos = 0; bytePos < inputBytes.length; bytePos+=binWidth) {
|
||||||
|
const block = inputBytes.slice(bytePos, bytePos+binWidth);
|
||||||
|
entropyData.push(this.calculateShannonEntropy(block));
|
||||||
|
}
|
||||||
|
|
||||||
|
return { entropyData, binWidth };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculates the frequency of bytes in the input.
|
||||||
|
*
|
||||||
|
* @param {object} svg
|
||||||
|
* @param {function} xScale
|
||||||
|
* @param {function} yScale
|
||||||
|
* @param {integer} svgHeight
|
||||||
|
* @param {integer} svgWidth
|
||||||
|
* @param {object} margins
|
||||||
|
* @param {string} xTitle
|
||||||
|
* @param {string} yTitle
|
||||||
|
* @returns {undefined}
|
||||||
|
*/
|
||||||
|
createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, title, xTitle, yTitle) {
|
||||||
|
// Axes
|
||||||
|
const yAxis = d3.axisLeft()
|
||||||
|
.scale(yScale);
|
||||||
|
|
||||||
|
const xAxis = d3.axisBottom()
|
||||||
|
.scale(xScale);
|
||||||
|
|
||||||
|
svg.append("g")
|
||||||
|
.attr("transform", `translate(0, ${svgHeight - margins.bottom})`)
|
||||||
|
.call(xAxis);
|
||||||
|
|
||||||
|
svg.append("g")
|
||||||
|
.attr("transform", `translate(${margins.left},0)`)
|
||||||
|
.call(yAxis);
|
||||||
|
|
||||||
|
// Axes labels
|
||||||
|
svg.append("text")
|
||||||
|
.attr("transform", "rotate(-90)")
|
||||||
|
.attr("y", 0 - margins.left)
|
||||||
|
.attr("x", 0 - (svgHeight / 2))
|
||||||
|
.attr("dy", "1em")
|
||||||
|
.style("text-anchor", "middle")
|
||||||
|
.text(yTitle);
|
||||||
|
|
||||||
|
svg.append("text")
|
||||||
|
.attr("transform", `translate(${svgWidth / 2}, ${svgHeight - margins.bottom + 40})`)
|
||||||
|
.style("text-anchor", "middle")
|
||||||
|
.text(xTitle);
|
||||||
|
|
||||||
|
// Add title
|
||||||
|
svg.append("text")
|
||||||
|
.attr("transform", `translate(${svgWidth / 2}, ${margins.top - 10})`)
|
||||||
|
.style("text-anchor", "middle")
|
||||||
|
.text(title);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculates the frequency of bytes in the input.
|
||||||
|
*
|
||||||
|
* @param {byteArray} inputBytes
|
||||||
|
* @returns {frequency}
|
||||||
|
*/
|
||||||
|
calculateByteFrequency(inputBytes) {
|
||||||
|
const byteFrequency = [];
|
||||||
|
for (let i = 0; i < 256; i++) {
|
||||||
|
if (inputBytes.length > 0) {
|
||||||
|
let count = 0;
|
||||||
|
for (const byte of inputBytes) {
|
||||||
|
if (byte === i) count++;
|
||||||
|
}
|
||||||
|
byteFrequency.push(count / inputBytes.length);
|
||||||
|
} else {
|
||||||
|
byteFrequency.push(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return byteFrequency;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculates the frequency of bytes in the input.
|
||||||
|
*
|
||||||
|
* @param {byteArray} byteFrequency
|
||||||
|
* @returns {frequency}
|
||||||
|
*/
|
||||||
|
createByteFrequencyLineHistogram(byteFrequency) {
|
||||||
|
const margins = { top: 30, right: 20, bottom: 50, left: 30 };
|
||||||
|
|
||||||
|
const svgWidth = 500,
|
||||||
|
svgHeight = 500;
|
||||||
|
|
||||||
|
const document = new nodom.Document();
|
||||||
|
let svg = document.createElement("svg");
|
||||||
|
|
||||||
|
svg = d3.select(svg)
|
||||||
|
.attr("width", "100%")
|
||||||
|
.attr("height", "100%")
|
||||||
|
.attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`);
|
||||||
|
|
||||||
|
const yScale = d3.scaleLinear()
|
||||||
|
.domain([0, d3.max(byteFrequency, d => d)])
|
||||||
|
.range([svgHeight - margins.bottom, margins.top]);
|
||||||
|
|
||||||
|
const xScale = d3.scaleLinear()
|
||||||
|
.domain([0, byteFrequency.length - 1])
|
||||||
|
.range([margins.left, svgWidth - margins.right]);
|
||||||
|
|
||||||
|
const line = d3.line()
|
||||||
|
.x((_, i) => xScale(i))
|
||||||
|
.y(d => yScale(d))
|
||||||
|
.curve(d3.curveMonotoneX);
|
||||||
|
|
||||||
|
svg.append("path")
|
||||||
|
.datum(byteFrequency)
|
||||||
|
.attr("fill", "none")
|
||||||
|
.attr("stroke", "steelblue")
|
||||||
|
.attr("d", line);
|
||||||
|
|
||||||
|
this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "", "Byte", "Byte Frequency");
|
||||||
|
|
||||||
|
return svg._groups[0][0].outerHTML;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a byte frequency histogram
|
||||||
|
*
|
||||||
|
* @param {byteArray} byteFrequency
|
||||||
|
* @returns {HTML}
|
||||||
|
*/
|
||||||
|
createByteFrequencyBarHistogram(byteFrequency) {
|
||||||
|
const margins = { top: 30, right: 20, bottom: 50, left: 30 };
|
||||||
|
|
||||||
|
const svgWidth = 500,
|
||||||
|
svgHeight = 500,
|
||||||
|
binWidth = 1;
|
||||||
|
|
||||||
|
const document = new nodom.Document();
|
||||||
|
let svg = document.createElement("svg");
|
||||||
|
svg = d3.select(svg)
|
||||||
|
.attr("width", "100%")
|
||||||
|
.attr("height", "100%")
|
||||||
|
.attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`);
|
||||||
|
|
||||||
|
const yExtent = d3.extent(byteFrequency, d => d);
|
||||||
|
const yScale = d3.scaleLinear()
|
||||||
|
.domain(yExtent)
|
||||||
|
.range([svgHeight - margins.bottom, margins.top]);
|
||||||
|
|
||||||
|
const xScale = d3.scaleLinear()
|
||||||
|
.domain([0, byteFrequency.length - 1])
|
||||||
|
.range([margins.left - binWidth, svgWidth - margins.right]);
|
||||||
|
|
||||||
|
svg.selectAll("rect")
|
||||||
|
.data(byteFrequency)
|
||||||
|
.enter().append("rect")
|
||||||
|
.attr("x", (_, i) => xScale(i) + binWidth)
|
||||||
|
.attr("y", dataPoint => yScale(dataPoint))
|
||||||
|
.attr("width", binWidth)
|
||||||
|
.attr("height", dataPoint => yScale(yExtent[0]) - yScale(dataPoint))
|
||||||
|
.attr("fill", "blue");
|
||||||
|
|
||||||
|
this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "", "Byte", "Byte Frequency");
|
||||||
|
|
||||||
|
return svg._groups[0][0].outerHTML;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a byte frequency histogram
|
||||||
|
*
|
||||||
|
* @param {byteArray} input
|
||||||
|
* @param {number} blockSize
|
||||||
|
* @returns {HTML}
|
||||||
|
*/
|
||||||
|
createEntropyCurve(entropyData) {
|
||||||
|
const margins = { top: 30, right: 20, bottom: 50, left: 30 };
|
||||||
|
|
||||||
|
const svgWidth = 500,
|
||||||
|
svgHeight = 500;
|
||||||
|
|
||||||
|
const document = new nodom.Document();
|
||||||
|
let svg = document.createElement("svg");
|
||||||
|
svg = d3.select(svg)
|
||||||
|
.attr("width", "100%")
|
||||||
|
.attr("height", "100%")
|
||||||
|
.attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`);
|
||||||
|
|
||||||
|
const yScale = d3.scaleLinear()
|
||||||
|
.domain([0, d3.max(entropyData, d => d)])
|
||||||
|
.range([svgHeight - margins.bottom, margins.top]);
|
||||||
|
|
||||||
|
const xScale = d3.scaleLinear()
|
||||||
|
.domain([0, entropyData.length])
|
||||||
|
.range([margins.left, svgWidth - margins.right]);
|
||||||
|
|
||||||
|
const line = d3.line()
|
||||||
|
.x((_, i) => xScale(i))
|
||||||
|
.y(d => yScale(d))
|
||||||
|
.curve(d3.curveMonotoneX);
|
||||||
|
|
||||||
|
if (entropyData.length > 0) {
|
||||||
|
svg.append("path")
|
||||||
|
.datum(entropyData)
|
||||||
|
.attr("d", line);
|
||||||
|
|
||||||
|
svg.selectAll("path").attr("fill", "none").attr("stroke", "steelblue");
|
||||||
|
}
|
||||||
|
|
||||||
|
this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "Scanning Entropy", "Block", "Entropy");
|
||||||
|
|
||||||
|
return svg._groups[0][0].outerHTML;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates an image representation of the entropy
|
||||||
|
*
|
||||||
|
* @param {byteArray} input
|
||||||
|
* @param {number} blockSize
|
||||||
|
* @returns {HTML}
|
||||||
|
*/
|
||||||
|
createEntropyImage(entropyData) {
|
||||||
|
const svgHeight = 100,
|
||||||
|
svgWidth = 100,
|
||||||
|
cellSize = 1,
|
||||||
|
nodes = [];
|
||||||
|
|
||||||
|
for (let i = 0; i < entropyData.length; i++) {
|
||||||
|
nodes.push({
|
||||||
|
x: i % svgWidth,
|
||||||
|
y: Math.floor(i / svgWidth),
|
||||||
|
entropy: entropyData[i]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const document = new nodom.Document();
|
||||||
|
let svg = document.createElement("svg");
|
||||||
|
svg = d3.select(svg)
|
||||||
|
.attr("width", "100%")
|
||||||
|
.attr("height", "100%")
|
||||||
|
.attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`);
|
||||||
|
|
||||||
|
const greyScale = d3.scaleLinear()
|
||||||
|
.domain([0, d3.max(entropyData, d => d)])
|
||||||
|
.range(["#000000", "#FFFFFF"])
|
||||||
|
.interpolate(d3.interpolateRgb);
|
||||||
|
|
||||||
|
svg
|
||||||
|
.selectAll("rect")
|
||||||
|
.data(nodes)
|
||||||
|
.enter().append("rect")
|
||||||
|
.attr("x", d => d.x * cellSize)
|
||||||
|
.attr("y", d => d.y * cellSize)
|
||||||
|
.attr("width", cellSize)
|
||||||
|
.attr("height", cellSize)
|
||||||
|
.style("fill", d => greyScale(d.entropy));
|
||||||
|
|
||||||
|
return svg._groups[0][0].outerHTML;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Displays the entropy as a scale bar for web apps.
|
* Displays the entropy as a scale bar for web apps.
|
||||||
*
|
*
|
||||||
* @param {number} entropy
|
* @param {number} entropy
|
||||||
* @returns {html}
|
* @returns {html}
|
||||||
*/
|
*/
|
||||||
present(entropy) {
|
createShannonEntropyVisualization(entropy) {
|
||||||
return `Shannon entropy: ${entropy}
|
return `Shannon entropy: ${entropy}
|
||||||
<br><canvas id='chart-area'></canvas><br>
|
<br><canvas id='chart-area'></canvas><br>
|
||||||
- 0 represents no randomness (i.e. all the bytes in the data have the same value) whereas 8, the maximum, represents a completely random string.
|
- 0 represents no randomness (i.e. all the bytes in the data have the same value) whereas 8, the maximum, represents a completely random string.
|
||||||
@ -92,6 +375,26 @@ The following results show the entropy of chunks of the input data. Chunks with
|
|||||||
</script>`;
|
</script>`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {byteArray} input
|
||||||
|
* @param {Object[]} args
|
||||||
|
* @returns {html}
|
||||||
|
*/
|
||||||
|
run(input, args) {
|
||||||
|
const visualizationType = args[0];
|
||||||
|
|
||||||
|
if (visualizationType === 'Shannon') {
|
||||||
|
return this.createShannonEntropyVisualization(this.calculateShannonEntropy(input));
|
||||||
|
} else if (visualizationType === "Histogram (Bar)") {
|
||||||
|
return this.createByteFrequencyBarHistogram(this.calculateByteFrequency(input));
|
||||||
|
} else if (visualizationType === "Histogram (Line)") {
|
||||||
|
return this.createByteFrequencyLineHistogram(this.calculateByteFrequency(input));
|
||||||
|
} else if (visualizationType === "Curve") {
|
||||||
|
return this.createEntropyCurve(this.calculateScanningEntropy(input).entropyData);
|
||||||
|
} else if (visualizationType === "Image") {
|
||||||
|
return this.createEntropyImage(this.calculateScanningEntropy(input).entropyData);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export default Entropy;
|
export default Entropy;
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user