From 525cb0689f9481fec15863ec0d6239511638157c Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 2 Apr 2019 17:27:14 +0100 Subject: [PATCH 1/2] Added 'Protobuf Decode', 'VarInt Decode' and 'VarInt Encode' operations --- src/core/config/Categories.json | 3 + src/core/lib/Protobuf.mjs | 285 +++++++++++++++++++++++++ src/core/operations/ProtobufDecode.mjs | 46 ++++ src/core/operations/VarIntDecode.mjs | 46 ++++ src/core/operations/VarIntEncode.mjs | 46 ++++ 5 files changed, 426 insertions(+) create mode 100644 src/core/lib/Protobuf.mjs create mode 100644 src/core/operations/ProtobufDecode.mjs create mode 100644 src/core/operations/VarIntDecode.mjs create mode 100644 src/core/operations/VarIntEncode.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index e9451a28eb..2d194c37c4 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -169,6 +169,9 @@ "Parse URI", "URL Encode", "URL Decode", + "Protobuf Decode", + "VarInt Encode", + "VarInt Decode", "Format MAC addresses", "Change IP format", "Group IP addresses", diff --git a/src/core/lib/Protobuf.mjs b/src/core/lib/Protobuf.mjs new file mode 100644 index 0000000000..408a481bba --- /dev/null +++ b/src/core/lib/Protobuf.mjs @@ -0,0 +1,285 @@ +import Utils from "../Utils"; + +/** + * Protobuf lib. Contains functions to decode protobuf serialised + * data without a schema or .proto file. + * + * Provides utility functions to encode and decode variable length + * integers (varint). + * + * @author GCHQ Contributor [3] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ +class Protobuf { + + /** + * Protobuf constructor + * + * @param {byteArray} data + */ + constructor(data) { + // Check we have a byteArray + if (data instanceof Array) { + this.data = data; + } else { + throw new Error("Protobuf input must be a byteArray"); + } + + // Set up masks + this.TYPE = 0x07; + this.NUMBER = 0x78; + this.MSB = 0x80; + this.VALUE = 0x7f; + + // Declare offset and length + this.offset = 0; + this.LENGTH = data.length; + } + + // Public Functions + + /** + * Encode a varint from a number + * + * @param {number} number + * @returns {byteArray} + */ + static varIntEncode(number) { + const MSB = 0x80, + VALUE = 0x7f, + MSBALL = ~VALUE, + INT = Math.pow(2, 31); + const out = []; + let offset = 0; + + while (number >= INT) { + out[offset++] = (number & 0xff) | MSB; + number /= 128; + } + while (number & MSBALL) { + out[offset++] = (number & 0xff) | MSB; + number >>>= 7; + } + out[offset] = number | 0; + return out; + } + + /** + * Decode a varint from the byteArray + * + * @param {byteArray} input + * @returns {number} + */ + static varIntDecode(input) { + const pb = new Protobuf(input); + return pb._varInt(); + } + + /** + * Parse Protobuf data + * + * @param {byteArray} input + * @returns {Object} + */ + static decode(input) { + const pb = new Protobuf(input); + return pb._parse(); + } + + // Private Class Functions + + /** + * Main private parsing function + * + * @private + * @returns {Object} + */ + _parse() { + let object = {}; + // Continue reading whilst we still have data + while (this.offset < this.LENGTH) { + const field = this._parseField(); + object = this._addField(field, object); + } + // Throw an error if we have gone beyond the end of the data + if (this.offset > this.LENGTH) { + throw new Error("Exhausted Buffer"); + } + return object; + } + + /** + * Add a field read from the protobuf data into the Object. As + * protobuf fields can appear multiple times, if the field already + * exists we need to add the new field into an array of fields + * for that key. + * + * @private + * @param {Object} field + * @param {Object} object + * @returns {Object} + */ + _addField(field, object) { + // Get the field key/values + const key = field.key; + const value = field.value; + object[key] = object.hasOwnProperty(key) ? + object[key] instanceof Array ? + object[key].concat([value]) : + [object[key], value] : + value; + return object; + } + + /** + * Parse a field and return the Object read from the record + * + * @private + * @returns {Object} + */ + _parseField() { + // Get the field headers + const header = this._fieldHeader(); + const type = header.type; + const key = header.key; + switch (type) { + // varint + case 0: + return { "key": key, "value": this._varInt() }; + // fixed 64 + case 1: + return { "key": key, "value": this._uint64() }; + // length delimited + case 2: + return { "key": key, "value": this._lenDelim() }; + // fixed 32 + case 5: + return { "key": key, "value": this._uint32() }; + // unknown type + default: + throw new Error("Unknown type 0x" + type.toString(16)); + } + } + + /** + * Parse the field header and return the type and key + * + * @private + * @returns {Object} + */ + _fieldHeader() { + // Make sure we call type then number to preserve offset + return { "type": this._fieldType(), "key": this._fieldNumber() }; + } + + /** + * Parse the field type from the field header. Type is stored in the + * lower 3 bits of the tag byte. This does not move the offset on as + * we need to read the field number from the tag byte too. + * + * @private + * @returns {number} + */ + _fieldType() { + // Field type stored in lower 3 bits of tag byte + return this.data[this.offset] & this.TYPE; + } + + /** + * Parse the field number (i.e. the key) from the field header. The + * field number is stored in the upper 5 bits of the tag byte - but + * is also varint encoded so the follow on bytes may need to be read + * when field numbers are > 15. + * + * @private + * @returns {number} + */ + _fieldNumber() { + let shift = -3; + let fieldNumber = 0; + do { + fieldNumber += shift < 28 ? + shift === -3 ? + (this.data[this.offset] & this.NUMBER) >> -shift : + (this.data[this.offset] & this.VALUE) << shift : + (this.data[this.offset] & this.VALUE) * Math.pow(2, shift); + shift += 7; + } while ((this.data[this.offset++] & this.MSD) === this.MSB); + return fieldNumber; + } + + // Field Parsing Functions + + /** + * Read off a varint from the data + * + * @private + * @returns {number} + */ + _varInt() { + let value = 0; + let shift = 0; + // Keep reading while upper bit set + do { + value += shift < 28 ? + (this.data[this.offset] & this.VALUE) << shift : + (this.data[this.offset] & this.VALUE) * Math.pow(2, shift); + shift += 7; + } while ((this.data[this.offset++] & this.MSB) === this.MSB); + return value; + } + + /** + * Read off a 64 bit unsigned integer from the data + * + * @private + * @returns {number} + */ + _uint64() { + // Read off a Uint64 + let num = this.data[this.offset++] * 0x1000000 + (this.data[this.offset++] << 16) + (this.data[this.offset++] << 8) + this.data[this.offset++]; + num = num * 0x100000000 + this.data[this.offset++] * 0x1000000 + (this.data[this.offset++] << 16) + (this.data[this.offset++] << 8) + this.data[this.offset++]; + return num; + } + + /** + * Read off a length delimited field from the data + * + * @private + * @returns {Object|string} + */ + _lenDelim() { + // Read off the field length + const length = this._varInt(); + const fieldBytes = this.data.slice(this.offset, this.offset + length); + let field; + try { + // Attempt to parse as a new Protobuf Object + const pbObject = new Protobuf(fieldBytes); + field = pbObject._parse(); + } catch (err) { + // Otherwise treat as bytes + field = Utils.byteArrayToChars(fieldBytes); + } + // Move the offset and return the field + this.offset += length; + return field; + } + + /** + * Read a 32 bit unsigned integer from the data + * + * @private + * @returns {number} + */ + _uint32() { + // Use a dataview to read off the integer + const dataview = new DataView(new Uint8Array(this.data.slice(this.offset, this.offset + 4)).buffer); + const value = dataview.getUint32(0); + this.offset += 4; + return value; + } +} + +export default Protobuf; diff --git a/src/core/operations/ProtobufDecode.mjs b/src/core/operations/ProtobufDecode.mjs new file mode 100644 index 0000000000..ec520d05c0 --- /dev/null +++ b/src/core/operations/ProtobufDecode.mjs @@ -0,0 +1,46 @@ +/** + * @author GCHQ Contributor [3] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Protobuf from "../lib/Protobuf"; + +/** + * Protobuf Decode operation + */ +class ProtobufDecode extends Operation { + + /** + * ProtobufDecode constructor + */ + constructor() { + super(); + + this.name = "Protobuf Decode"; + this.module = "Default"; + this.description = "Decodes any Protobuf encoded data to a JSON representation of the data using the field number as the field key."; + this.infoURL = "https://wikipedia.org/wiki/Protocol_Buffers"; + this.inputType = "byteArray"; + this.outputType = "JSON"; + this.args = []; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {JSON} + */ + run(input, args) { + try { + return Protobuf.decode(input); + } catch (err) { + throw new OperationError(err); + } + } + +} + +export default ProtobufDecode; diff --git a/src/core/operations/VarIntDecode.mjs b/src/core/operations/VarIntDecode.mjs new file mode 100644 index 0000000000..df570be9b7 --- /dev/null +++ b/src/core/operations/VarIntDecode.mjs @@ -0,0 +1,46 @@ +/** + * @author GCHQ Contributor [3] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Protobuf from "../lib/Protobuf"; + +/** + * VarInt Decode operation + */ +class VarIntDecode extends Operation { + + /** + * VarIntDecode constructor + */ + constructor() { + super(); + + this.name = "VarInt Decode"; + this.module = "Default"; + this.description = "Decodes a VarInt encoded integer. VarInt is an efficient way of encoding variable length integers and is commonly used with Protobuf."; + this.infoURL = "https://developers.google.com/protocol-buffers/docs/encoding#varints"; + this.inputType = "byteArray"; + this.outputType = "number"; + this.args = []; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {number} + */ + run(input, args) { + try { + return Protobuf.varIntDecode(input); + } catch (err) { + throw new OperationError(err); + } + } + +} + +export default VarIntDecode; diff --git a/src/core/operations/VarIntEncode.mjs b/src/core/operations/VarIntEncode.mjs new file mode 100644 index 0000000000..69a552f63a --- /dev/null +++ b/src/core/operations/VarIntEncode.mjs @@ -0,0 +1,46 @@ +/** + * @author GCHQ Contributor [3] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Protobuf from "../lib/Protobuf"; + +/** + * VarInt Encode operation + */ +class VarIntEncode extends Operation { + + /** + * VarIntEncode constructor + */ + constructor() { + super(); + + this.name = "VarInt Encode"; + this.module = "Default"; + this.description = "Encodes a Vn integer as a VarInt. VarInt is an efficient way of encoding variable length integers and is commonly used with Protobuf."; + this.infoURL = "https://developers.google.com/protocol-buffers/docs/encoding#varints"; + this.inputType = "number"; + this.outputType = "byteArray"; + this.args = []; + } + + /** + * @param {number} input + * @param {Object[]} args + * @returns {byteArray} + */ + run(input, args) { + try { + return Protobuf.varIntEncode(input); + } catch (err) { + throw new OperationError(err); + } + } + +} + +export default VarIntEncode; From f6977ea26476492034299480935da2989d3fc11b Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 12 Apr 2019 16:13:10 +0100 Subject: [PATCH 2/2] Added test for 'Decode Protobuf' operation --- tests/operations/index.mjs | 1 + tests/operations/tests/Protobuf.mjs | 36 +++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 tests/operations/tests/Protobuf.mjs diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index e9fc62713d..41d78c3537 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -89,6 +89,7 @@ import "./tests/MultipleBombe"; import "./tests/Typex"; import "./tests/BLAKE2b"; import "./tests/BLAKE2s"; +import "./tests/Protobuf"; // Cannot test operations that use the File type yet //import "./tests/SplitColourChannels"; diff --git a/tests/operations/tests/Protobuf.mjs b/tests/operations/tests/Protobuf.mjs new file mode 100644 index 0000000000..957e4d54a3 --- /dev/null +++ b/tests/operations/tests/Protobuf.mjs @@ -0,0 +1,36 @@ +/** + * Protobuf tests. + * + * @author n1474335 [n1474335@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import TestRegister from "../TestRegister"; + +TestRegister.addTests([ + { + name: "Protobuf Decode", + input: "0d1c0000001203596f751a024d65202b2a0a0a066162633132331200", + expectedOutput: JSON.stringify({ + "1": 469762048, + "2": "You", + "3": "Me", + "4": 43, + "5": { + "1": "abc123", + "2": {} + } + }, null, 4), + recipeConfig: [ + { + "op": "From Hex", + "args": ["Auto"] + }, + { + "op": "Protobuf Decode", + "args": [] + } + ] + }, +]);