From ba07a8060ca52f3710d3053937c7c79fd8da25c1 Mon Sep 17 00:00:00 2001 From: Larry McQueary Date: Tue, 30 Aug 2016 15:25:02 -0600 Subject: [PATCH] Update NUID implementation to match Go (resolves #58) --- src/main/java/io/nats/client/NUID.java | 361 ++++++++++++--------- src/test/java/io/nats/client/NUIDTest.java | 98 +++--- 2 files changed, 251 insertions(+), 208 deletions(-) diff --git a/src/main/java/io/nats/client/NUID.java b/src/main/java/io/nats/client/NUID.java index 3f81acb3a..6c473897c 100644 --- a/src/main/java/io/nats/client/NUID.java +++ b/src/main/java/io/nats/client/NUID.java @@ -1,165 +1,214 @@ package io.nats.client; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; import java.util.Random; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - public class NUID { - final static Logger logger = LoggerFactory.getLogger(NUID.class); - - // NUID needs to be very fast to generate and truly unique, all while being entropy pool friendly. - // We will use 12 bytes of crypto generated data (entropy draining), and 10 bytes of sequential data - // that is started at a pseudo random number and increments with a pseudo-random increment. - // Total is 22 bytes of base 36 ascii text :) - - // Constants - static final char[] digits = { '0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z' }; - static final int base = 36; - static final int preLen = 12; - static final int seqLen = 10; - static final long maxPre = 4738381338321616896L; // base^preLen == 36^12 - static final long maxSeq = 3656158440062976L; // base^seqLen == 36^10 - static final long minInc = 33L; - static final long maxInc = 333L; - static final int totalLen = preLen + seqLen; - static Random srand; - static Random prand; - - // Instance fields - char[] pre; - long seq; - long inc; - - - // Global NUID - public static NUID globalNUID = new NUID(); - private static Object lock = new Object(); - - static NUID getInstance() { - if (globalNUID == null) { - globalNUID = new NUID(); - } - return globalNUID; - } - - public NUID () { - if (srand == null) { - try { - srand = SecureRandom.getInstance("SHA1PRNG"); - } catch (NoSuchAlgorithmException e) { - logger.error("stan: nuid algorithm not found", e); - } - prand = new Random(); - } - seq = nextLong(prand, maxSeq); - inc = minInc + nextLong(prand, maxInc-minInc); - pre = new char[preLen]; - for (int i = 0; i < preLen; i++) { - pre[i] = '0'; - } - randomizePrefix(); - } - - // Generate the next NUID string from the global locked NUID instance. - public static String nextGlobal() { - synchronized(lock) { - return getInstance().next(); - } - } - - // Generate the next NUID string. - public String next() { - // Increment and capture. - seq += inc; - if (seq >= maxSeq) { - randomizePrefix(); - resetSequential(); - } - - // Copy prefix - char[] b = new char[totalLen]; - System.arraycopy(pre, 0, b, 0, preLen); - - // copy in the seq in base36. - int i = b.length; - for (long l = seq; i > preLen; l /= base) { - i--; - b[i] = digits[(int)(l%base)]; - } - return new String(b); - } - - // Resets the sequntial portion of the NUID - void resetSequential() { - seq = nextLong(prand, maxSeq); - inc = minInc + nextLong(prand, maxInc-minInc); - } - - // Generate a new prefix from random. - // This will drain entropy and will be called automatically when we exhaust the sequential - // Will panic if it gets an error from rand.Int() - public void randomizePrefix() { - long n = nextLong(srand, maxPre); - int i = pre.length; - for (long l = n; i>0; l /= base) { - i--; - pre[i] = digits[(int)(l % base)]; - } - } - - static long nextLong(Random rng, long n) { - // error checking and 2^x checking removed for simplicity. - long bits, val; - do { - bits = (rng.nextLong() << 1) >>> 1; - val = bits % n; - } while (bits-val+(n-1) < 0L); - return val; - } - - /** - * @return the pre - */ - char[] getPre() { - return pre; - } - - /** - * @param pre the pre to set - */ - void setPre(char[] pre) { - this.pre = pre; - } - - /** - * @return the seq - */ - long getSeq() { - return seq; - } - - /** - * @param seq the seq to set - */ - void setSeq(long seq) { - this.seq = seq; - } - - /** - * @return the inc - */ - long getInc() { - return inc; - } - - /** - * @param inc the inc to set - */ - void setInc(long inc) { - this.inc = inc; - } + final static Logger logger = LoggerFactory.getLogger(NUID.class); + + /* + * NUID needs to be very fast to generate and truly unique, all while being entropy pool + * friendly. We will use 12 bytes of crypto generated data (entropy draining), and 10 bytes of + * sequential data that is started at a pseudo random number and increments with a pseudo-random + * increment. Total is 22 bytes of base 62 ascii text :) + */ + + // Constants + static final char[] digits = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', + 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', + 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z' }; + static final int base = 62; + static final int preLen = 12; + static final int seqLen = 10; + static final long maxSeq = 839299365868340224L; // base^seqLen == 62^10 + static final long minInc = 33L; + static final long maxInc = 333L; + static final int totalLen = preLen + seqLen; + static SecureRandom srand; + static Random prand; + + // Instance fields + char[] pre; + long seq; + long inc; + + + // Global NUID + public static NUID globalNUID = new NUID(); + private static Object lock = new Object(); + + static NUID getInstance() { + if (globalNUID == null) { + globalNUID = new NUID(); + } + return globalNUID; + } + + public NUID() { + // Generate a cryto random int, 0 <= val < max to seed pseudorandom + long seed = 0L; + if (srand == null) { + try { + srand = SecureRandom.getInstance("SHA1PRNG"); + seed = bytesToLong(srand.generateSeed(8)); // seed with 8 bytes (64 bits) + } catch (NoSuchAlgorithmException e) { + logger.error("nats: nuid algorithm not found", e); + } + + if (seed != 0L) { + prand = new Random(seed); + } else { + prand = new Random(); + } + } + seq = nextLong(prand, maxSeq); + inc = minInc + nextLong(prand, maxInc - minInc); + pre = new char[preLen]; + for (int i = 0; i < preLen; i++) { + pre[i] = '0'; + } + randomizePrefix(); + } + + /** + * Generate the next NUID string from the global locked NUID instance. + * + * @return the next NUID string from the global locked NUID instance. + */ + public static String nextGlobal() { + synchronized (lock) { + return getInstance().next(); + } + } + + /** + * Generate the next NUID string from this instance. + * + * @return the next NUID string from this instance. + */ + public String next() { + // Increment and capture. + seq += inc; + if (seq >= maxSeq) { + randomizePrefix(); + resetSequential(); + } + + // Copy prefix + char[] b = new char[totalLen]; + System.arraycopy(pre, 0, b, 0, preLen); + + // copy in the seq in base36. + int i = b.length; + for (long l = seq; i > preLen; l /= base) { + i--; + b[i] = digits[(int) (l % base)]; + } + return new String(b); + } + + // Resets the sequntial portion of the NUID + void resetSequential() { + seq = nextLong(prand, maxSeq); + inc = minInc + nextLong(prand, maxInc - minInc); + } + + /* + * Generate a new prefix from random. This *can* drain entropy and will be called automatically + * when we exhaust the sequential range. + */ + + void randomizePrefix() { + byte[] cb = new byte[preLen]; + + // Use SecureRandom for prefix only + srand.nextBytes(cb); + + for (int i = 0; i < preLen; i++) { + pre[i] = digits[(cb[i] & 0xFF) % base]; + } + } + + static long nextLong(Random rng, long maxValue) { + // error checking and 2^x checking removed for simplicity. + long bits; + long val; + do { + bits = (rng.nextLong() << 1) >>> 1; + val = bits % maxValue; + } while (bits - val + (maxValue - 1) < 0L); + return val; + } + + byte[] longToBytes(long x) { + ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE); + buffer.putLong(x); + return buffer.array(); + } + + long bytesToLong(byte[] bytes) { + ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE); + buffer.put(bytes); + buffer.flip();// need flip + return buffer.getLong(); + } + + /** + * @return the pre + */ + char[] getPre() { + return pre; + } + + /** + * Sets the prefix. + * + * @param pre the pre to set + */ + void setPre(char[] pre) { + this.pre = pre; + } + + /** + * Return the current sequence value. + * + * @return the seq + */ + long getSeq() { + return seq; + } + + /** + * Set the sequence to the supplied value. + * + * @param seq the seq to set + */ + void setSeq(long seq) { + this.seq = seq; + } + + /** + * Return the current increment. + * + * @return the inc + */ + long getInc() { + return inc; + } + + /** + * Set the increment to the supplied value. + * + * @param inc the inc to set + */ + void setInc(long inc) { + this.inc = inc; + } } diff --git a/src/test/java/io/nats/client/NUIDTest.java b/src/test/java/io/nats/client/NUIDTest.java index d017490ce..feeb0ee5f 100644 --- a/src/test/java/io/nats/client/NUIDTest.java +++ b/src/test/java/io/nats/client/NUIDTest.java @@ -19,10 +19,7 @@ import org.slf4j.LoggerFactory; import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -@Category(UnitTest.class) public class NUIDTest { final Logger logger = LoggerFactory.getLogger(NUIDTest.class); @@ -42,15 +39,25 @@ public void setUp() throws Exception {} public void tearDown() throws Exception {} @Test - public void testGlobalNUID() { + @Category(UnitTest.class) + public void testDigits() { + if (NUID.digits.length != NUID.base) { + fail("digits length does not match base modulo"); + } + } + + @Test + @Category(UnitTest.class) + public void testGlobalNUIDInit() { NUID nuid = NUID.getInstance(); assertNotNull(nuid); assertNotNull("Expected prefix to be initialized", nuid.getPre()); assertEquals(NUID.preLen, nuid.getPre().length); - assertNotEquals("Expected seq to be non-zero", nuid.getSeq()); + assertNotEquals("Expected seq to be non-zero", 0, nuid.getSeq()); } @Test + @Category(UnitTest.class) public void testNUIDRollover() { NUID gnuid = NUID.getInstance(); gnuid.setSeq(NUID.maxSeq); @@ -61,59 +68,16 @@ public void testNUIDRollover() { } @Test - public void testNUIDLen() { + @Category(UnitTest.class) + public void testGUIDLen() { String nuid = new NUID().next(); + System.err.println("NUID: " + nuid); assertEquals(String.format("Expected len of %d, got %d", NUID.totalLen, nuid.length()), NUID.totalLen, nuid.length()); } @Test - public void testNUIDSpeed() { - long count = 10000000; - NUID nuid = new NUID(); - - long start = System.nanoTime(); - for (int i = 0; i < count; i++) { - nuid.next(); - } - long elapsedNsec = System.nanoTime() - start; - logger.info("Average generation time for {} NUIDs was {}ns", count, - (double) elapsedNsec / count); - - } - - @Test - public void testGlobalNUIDSpeed() { - long count = 10000000; - NUID nuid = NUID.getInstance(); - - long start = System.nanoTime(); - for (int i = 0; i < count; i++) { - nuid.next(); - } - long elapsedNsec = System.nanoTime() - start; - logger.info("Average generation time for {} global NUIDs was {}ns", count, - (double) elapsedNsec / count); - - } - - @Test - public void testBasicUniqueness() { - int count = 10000000; - Map nuidMap = new HashMap(count); - - for (int i = 0; i < count; i++) { - // String n = NUID.nextGlobal(); - String nuid = new NUID().next(); - if (nuidMap.get(nuid) != null) { - fail("Duplicate NUID found: " + nuid); - } else { - nuidMap.put(nuid, true); - } - } - } - - @Test + @Category(UnitTest.class) public void testProperPrefix() { char min = (char) 255; char max = (char) 0; @@ -140,6 +104,36 @@ public void testProperPrefix() { } } } + } + + @Test + @Category(BenchmarkTest.class) + public void benchmarkNUIDSpeed() { + long count = 10000000; + NUID nuid = new NUID(); + + long start = System.nanoTime(); + for (int i = 0; i < count; i++) { + nuid.next(); + } + long elapsedNsec = System.nanoTime() - start; + logger.info("Average generation time for {} NUIDs was {}ns", count, + (double) elapsedNsec / count); + + } + + @Test + @Category(BenchmarkTest.class) + public void benchmarkGlobalNUIDSpeed() { + long count = 10000000; + NUID nuid = NUID.getInstance(); + long start = System.nanoTime(); + for (int i = 0; i < count; i++) { + nuid.next(); + } + long elapsedNsec = System.nanoTime() - start; + logger.info("Average generation time for {} global NUIDs was {}ns", count, + (double) elapsedNsec / count); } }