Skip to content

Commit 451858b

Browse files
authoredNov 11, 2021
feat: hash uniformity for base digests
1 parent f7dbfe1 commit 451858b

File tree

4 files changed

+76
-30
lines changed

4 files changed

+76
-30
lines changed
 

‎lib/getHashDigest.js

+28-19
Original file line numberDiff line numberDiff line change
@@ -11,34 +11,46 @@ const baseEncodeTables = {
1111
64: "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_",
1212
};
1313

14-
function encodeBufferToBase(buffer, base) {
14+
/**
15+
* @param {Uint32Array} uint32Array Treated as a long base-0x100000000 number, little endian
16+
* @param {number} divisor The divisor
17+
* @return {number} Modulo (remainder) of the division
18+
*/
19+
function divmod32(uint32Array, divisor) {
20+
let carry = 0;
21+
for (let i = uint32Array.length - 1; i >= 0; i--) {
22+
const value = carry * 0x100000000 + uint32Array[i];
23+
carry = value % divisor;
24+
uint32Array[i] = Math.floor(value / divisor);
25+
}
26+
return carry;
27+
}
28+
29+
function encodeBufferToBase(buffer, base, length) {
1530
const encodeTable = baseEncodeTables[base];
1631

1732
if (!encodeTable) {
1833
throw new Error("Unknown encoding base" + base);
1934
}
2035

21-
const readLength = buffer.length;
22-
const Big = require("big.js");
36+
// Input bits are only enough to generate this many characters
37+
const limit = Math.ceil((buffer.length * 8) / Math.log2(base));
38+
length = Math.min(length, limit);
2339

24-
Big.RM = Big.DP = 0;
40+
// Most of the crypto digests (if not all) has length a multiple of 4 bytes.
41+
// Fewer numbers in the array means faster math.
42+
const uint32Array = new Uint32Array(Math.ceil(buffer.length / 4));
2543

26-
let b = new Big(0);
27-
28-
for (let i = readLength - 1; i >= 0; i--) {
29-
b = b.times(256).plus(buffer[i]);
30-
}
44+
// Make sure the input buffer data is copied and is not mutated by reference.
45+
// divmod32() would corrupt the BulkUpdateDecorator cache otherwise.
46+
buffer.copy(Buffer.from(uint32Array.buffer));
3147

3248
let output = "";
3349

34-
while (b.gt(0)) {
35-
output = encodeTable[b.mod(base)] + output;
36-
b = b.div(base);
50+
for (let i = 0; i < length; i++) {
51+
output = encodeTable[divmod32(uint32Array, base)] + output;
3752
}
3853

39-
Big.DP = 20;
40-
Big.RM = 1;
41-
4254
return output;
4355
}
4456

@@ -110,10 +122,7 @@ function getHashDigest(buffer, algorithm, digestType, maxLength) {
110122
digestType === "base58" ||
111123
digestType === "base62"
112124
) {
113-
return encodeBufferToBase(hash.digest(), digestType.substr(4)).substr(
114-
0,
115-
maxLength
116-
);
125+
return encodeBufferToBase(hash.digest(), digestType.substr(4), maxLength);
117126
} else {
118127
return hash.digest(digestType || "hex").substr(0, maxLength);
119128
}

‎package.json

+1-3
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,7 @@
33
"version": "3.1.3",
44
"author": "Tobias Koppers @sokra",
55
"description": "utils for webpack loaders",
6-
"dependencies": {
7-
"big.js": "^6.1.1"
8-
},
6+
"dependencies": {},
97
"scripts": {
108
"lint": "prettier --list-different . && eslint .",
119
"pretest": "yarn lint",

‎test/getHashDigest.test.js

+47-3
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@ describe("getHashDigest()", () => {
1111
["abc\\0💩", "xxhash64", "hex", undefined, "86733ec125b93904"],
1212
["abc\\0💩", "xxhash64", "base64", undefined, "hnM+wSW5OQQ="],
1313
["abc\\0♥", "xxhash64", "base64", undefined, "S5o0KX3APSA="],
14-
["abc\\0💩", "xxhash64", "base52", undefined, "cfByjQcJZIU"],
15-
["abc\\0♥", "xxhash64", "base52", undefined, "qdLyAQjLlod"],
14+
["abc\\0💩", "xxhash64", "base52", undefined, "acfByjQcJZIU"],
15+
["abc\\0♥", "xxhash64", "base52", undefined, "aqdLyAQjLlod"],
1616

1717
["test string", "md4", "hex", 4, "2e06"],
1818
["test string", "md4", "base64", undefined, "Lgbt1PFiMmjFpRcw2KCyrw=="],
@@ -34,7 +34,8 @@ describe("getHashDigest()", () => {
3434
],
3535
["test string", "md5", "base52", undefined, "dJnldHSAutqUacjgfBQGLQx"],
3636
["test string", "md5", "base64", undefined, "b421md6Yb6t6IWJbeRZYnA=="],
37-
["test string", "md5", "base26", 6, "bhtsgu"],
37+
["test string", "md5", "base26", undefined, "bhtsgujtzvmjtgtzlqvubqggbvgx"],
38+
["test string", "md5", "base26", 6, "ggbvgx"],
3839
["abc\\0♥", "md5", "hex", undefined, "2e897b64f8050e66aff98d38f7a012c5"],
3940
["abc\\0💩", "md5", "hex", undefined, "63ad5b3d675c5890e0c01ed339ba0187"],
4041
["abc\\0💩", "md5", "base64", undefined, "Y61bPWdcWJDgwB7TOboBhw=="],
@@ -79,3 +80,46 @@ describe("getHashDigest()", () => {
7980
);
8081
});
8182
});
83+
84+
function testDistribution(digestType, length, tableSize, iterations) {
85+
const lowerBound = Math.round(iterations / 2);
86+
const upperBound = Math.round(iterations * 2);
87+
88+
const stats = [];
89+
for (let i = tableSize * iterations; i-- > 0; ) {
90+
const generatedString = loaderUtils.getHashDigest(
91+
`Some input #${i}`,
92+
undefined,
93+
digestType,
94+
length
95+
);
96+
97+
for (let pos = 0; pos < length; pos++) {
98+
const char = generatedString[pos];
99+
stats[pos] = stats[pos] || {};
100+
stats[pos][char] = (stats[pos][char] || 0) + 1;
101+
}
102+
}
103+
104+
for (let pos = 0; pos < length; pos++) {
105+
const chars = Object.keys(stats[pos]).sort();
106+
test(`distinct chars at position ${pos}`, () => {
107+
expect(chars.length).toBe(tableSize);
108+
});
109+
for (const char of chars) {
110+
test(`occurences of char "${char}" at position ${pos} should be around ${iterations}`, () => {
111+
expect(stats[pos][char]).toBeLessThanOrEqual(upperBound);
112+
expect(stats[pos][char]).toBeGreaterThanOrEqual(lowerBound);
113+
});
114+
}
115+
}
116+
}
117+
118+
describe("getHashDigest() char distribution", () => {
119+
describe("should be uniform for base62", () => {
120+
testDistribution("base62", 8, 62, 100);
121+
});
122+
describe("should be uniform for base26", () => {
123+
testDistribution("base26", 8, 26, 100);
124+
});
125+
});

‎yarn.lock

-5
Original file line numberDiff line numberDiff line change
@@ -865,11 +865,6 @@ bcrypt-pbkdf@^1.0.0:
865865
dependencies:
866866
tweetnacl "^0.14.3"
867867

868-
big.js@^6.1.1:
869-
version "6.1.1"
870-
resolved "https://registry.yarnpkg.com/big.js/-/big.js-6.1.1.tgz#63b35b19dc9775c94991ee5db7694880655d5537"
871-
integrity sha512-1vObw81a8ylZO5ePrtMay0n018TcftpTA5HFKDaSuiUDBo8biRBtjIobw60OpwuvrGk+FsxKamqN4cnmj/eXdg==
872-
873868
brace-expansion@^1.1.7:
874869
version "1.1.11"
875870
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"

0 commit comments

Comments
 (0)