diff --git a/lib/command.js b/lib/command.js index 8d75ff70..c0eb3efa 100644 --- a/lib/command.js +++ b/lib/command.js @@ -5,6 +5,7 @@ var Promise = require('bluebird'); var fbuffer = require('flexbuffer'); var utils = require('./utils'); var commands = require('redis-commands'); +var calculateSlot = require('cluster-key-slot'); /** * Command instance @@ -84,7 +85,7 @@ Command.prototype.getSlot = function () { if (typeof this._slot === 'undefined') { var key = this.getKeys()[0]; if (key) { - this.slot = utils.calcSlot(key); + this.slot = calculateSlot(key); } else { this.slot = null; } diff --git a/lib/pipeline.js b/lib/pipeline.js index 2f359223..487f4fd5 100644 --- a/lib/pipeline.js +++ b/lib/pipeline.js @@ -4,9 +4,9 @@ var Commander = require('./commander'); var Command = require('./command'); var fbuffer = require('flexbuffer'); var Promise = require('bluebird'); -var utils = require('./utils'); var util = require('util'); var commands = require('redis-commands'); +var calculateSlot = require('cluster-key-slot'); function Pipeline(redis) { Commander.call(this); @@ -210,25 +210,33 @@ Pipeline.prototype.exec = function (callback) { if (_.isEmpty(this._queue)) { this.resolve([]); } - var pipelineSlot; - // Check whether scripts exists and get a sampleKey. - var scripts = []; - for (var i = 0; i < this._queue.length; ++i) { - var item = this._queue[i]; - if (this.isCluster) { - var keys = item.getKeys(); - for (var j = 0; j < keys.length; ++j) { - var slot = utils.calcSlot(keys[j]); - if (typeof pipelineSlot === 'undefined') { - pipelineSlot = slot; - } - if (pipelineSlot !== slot) { - this.reject(new Error('All keys in the pipeline should belong to the same slot(expect "' + - keys[j] + '" belongs to slot ' + pipelineSlot + ').')); - return this.promise; - } + var pipelineSlot, i; + if (this.isCluster) { + // List of the first key for each command + var sampleKeys = []; + for (i = 0; i < this._queue.length; i++) { + var keys = this._queue[i].getKeys(); + if (keys.length) { + sampleKeys.push(keys[0]); + } + } + + if (sampleKeys.length) { + pipelineSlot = calculateSlot.generateMulti(sampleKeys); + if (pipelineSlot < 0) { + this.reject(new Error('All keys in the pipeline should belong to the same slot')); + return this.promise; } + } else { + // Send the pipeline to a random node + pipelineSlot = Math.random() * 16384 | 0; } + } + + // Check whether scripts exists + var scripts = []; + for (i = 0; i < this._queue.length; ++i) { + var item = this._queue[i]; if (this.isCluster && item.isCustomCommand) { this.reject(new Error('Sending custom commands in pipeline is not supported in Cluster mode.')); return this.promise; @@ -242,14 +250,12 @@ Pipeline.prototype.exec = function (callback) { } scripts.push(script); } - if (this.isCluster && typeof pipelineSlot === 'undefined') { - pipelineSlot = Math.random() * 16384 | 0; - } var _this = this; if (!scripts.length) { return execPipeline(); } + return this.redis.script('exists', scripts.map(function (item) { return item.sha; })).then(function (results) { diff --git a/lib/utils/crc.js b/lib/utils/crc.js deleted file mode 100644 index d3495506..00000000 --- a/lib/utils/crc.js +++ /dev/null @@ -1,91 +0,0 @@ -'use strict'; - -/* - * Copyright 2001-2010 Georges Menie (www.menie.org) - * Copyright 2010 Salvatore Sanfilippo (adapted to Redis coding style) - * Copyright 2015 Zihua Li (http://zihua.li) (ported to JavaScript) - * All rights reserved. - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the University of California, Berkeley nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* CRC16 implementation according to CCITT standards. - * - * Note by @antirez: this is actually the XMODEM CRC 16 algorithm, using the - * following parameters: - * - * Name : "XMODEM", also known as "ZMODEM", "CRC-16/ACORN" - * Width : 16 bit - * Poly : 1021 (That is actually x^16 + x^12 + x^5 + 1) - * Initialization : 0000 - * Reflect Input byte : False - * Reflect Output CRC : False - * Xor constant to output CRC : 0000 - * Output for "123456789" : 31C3 - */ - -var crc16tab = [ - 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, - 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, - 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, - 0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, - 0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, - 0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d, - 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4, - 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc, - 0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, - 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b, - 0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12, - 0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, - 0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, - 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49, - 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70, - 0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78, - 0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f, - 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067, - 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, - 0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, - 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d, - 0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, - 0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c, - 0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634, - 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab, - 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3, - 0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, - 0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92, - 0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9, - 0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, - 0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, - 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0 -]; - -function crc16(str) { - var buf = new Buffer(str); - var crc = 0; - for(var i = 0, len = buf.length; i < len; i++) { - crc = (crc << 8) ^ crc16tab[((crc >> 8) ^ buf[i]) & 0x00ff]; - } - return crc; -} - -module.exports = crc16; diff --git a/lib/utils/index.js b/lib/utils/index.js index 2bfa8063..0eab9589 100644 --- a/lib/utils/index.js +++ b/lib/utils/index.js @@ -213,24 +213,6 @@ exports.toArg = function (arg) { return String(arg); }; -var crc16 = require('./crc'); -/** - * Calculate slot by key - * - * @param {string} key - * @return {number} - */ -exports.calcSlot = function (key) { - var s = key.indexOf('{'); - if (s !== -1) { - var e = key.indexOf('}', s + 2); - if (e !== -1) { - key = key.slice(s + 1, e); - } - } - return crc16(key) & 16383; -}; - /** * Optimize error stack * diff --git a/package.json b/package.json index 76fad7ce..646fbe38 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "license": "MIT", "dependencies": { "bluebird": "^3.3.4", + "cluster-key-slot": "^1.0.5", "debug": "^2.2.0", "double-ended-queue": "^2.1.0-0", "flexbuffer": "0.0.6", diff --git a/test/functional/cluster.js b/test/functional/cluster.js index d759dd51..1cb353ac 100644 --- a/test/functional/cluster.js +++ b/test/functional/cluster.js @@ -1,6 +1,7 @@ 'use strict'; var utils = require('../../lib/utils'); +var calculateSlot = require('cluster-key-slot'); var Promise = require('bluebird'); describe('cluster', function () { @@ -325,7 +326,7 @@ describe('cluster', function () { if (argv[0] === 'get' && argv[1] === 'foo') { expect(moved).to.eql(false); moved = true; - return new Error('MOVED ' + utils.calcSlot('foo') + ' 127.0.0.1:30001'); + return new Error('MOVED ' + calculateSlot('foo') + ' 127.0.0.1:30001'); } }); @@ -345,7 +346,7 @@ describe('cluster', function () { ]; } if (argv[0] === 'get' && argv[1] === 'foo') { - return new Error('MOVED ' + utils.calcSlot('foo') + ' 127.0.0.1:30002'); + return new Error('MOVED ' + calculateSlot('foo') + ' 127.0.0.1:30002'); } }); var node2 = new MockServer(30002, function (argv) { @@ -397,7 +398,7 @@ describe('cluster', function () { if (argv[0] === 'get' && argv[1] === 'foo') { expect(moved).to.eql(false); moved = true; - return new Error('MOVED ' + utils.calcSlot('foo') + ' 127.0.0.1:30001'); + return new Error('MOVED ' + calculateSlot('foo') + ' 127.0.0.1:30001'); } }); @@ -439,7 +440,7 @@ describe('cluster', function () { disconnect([node1, node2], done); }); } else { - return new Error('ASK ' + utils.calcSlot('foo') + ' 127.0.0.1:30001'); + return new Error('ASK ' + calculateSlot('foo') + ' 127.0.0.1:30001'); } } }); @@ -530,7 +531,7 @@ describe('cluster', function () { ]; } else if (argv[0] === 'get' && argv[1] === 'foo') { redirectTimes += 1; - return new Error('ASK ' + utils.calcSlot('foo') + ' 127.0.0.1:30001'); + return new Error('ASK ' + calculateSlot('foo') + ' 127.0.0.1:30001'); } }; var node1 = new MockServer(30001, argvHandler); @@ -650,7 +651,7 @@ describe('cluster', function () { expect(moved).to.eql(false); moved = true; } - return new Error('MOVED ' + utils.calcSlot('foo') + ' 127.0.0.1:30001'); + return new Error('MOVED ' + calculateSlot('foo') + ' 127.0.0.1:30001'); } }); @@ -693,7 +694,7 @@ describe('cluster', function () { return slotTable; } if (argv[1] === 'foo') { - return new Error('ASK ' + utils.calcSlot('foo') + ' 127.0.0.1:30001'); + return new Error('ASK ' + calculateSlot('foo') + ' 127.0.0.1:30001'); } }); @@ -755,7 +756,7 @@ describe('cluster', function () { return slotTable; } if (argv[0] === 'get' && argv[1] === 'foo') { - return new Error('MOVED ' + utils.calcSlot('foo') + ' 127.0.0.1:30001'); + return new Error('MOVED ' + calculateSlot('foo') + ' 127.0.0.1:30001'); } }); @@ -836,7 +837,7 @@ describe('cluster', function () { } if (argv[0] === 'get' && argv[1] === 'foo') { moved = true; - return new Error('MOVED ' + utils.calcSlot('foo') + ' 127.0.0.1:30001'); + return new Error('MOVED ' + calculateSlot('foo') + ' 127.0.0.1:30001'); } if (argv[0] === 'exec') { return new Error('EXECABORT Transaction discarded because of previous errors.'); @@ -889,7 +890,7 @@ describe('cluster', function () { return slotTable; } if (argv[0] === 'get' && argv[1] === 'foo') { - return new Error('ASK ' + utils.calcSlot('foo') + ' 127.0.0.1:30001'); + return new Error('ASK ' + calculateSlot('foo') + ' 127.0.0.1:30001'); } if (argv[0] === 'exec') { return new Error('EXECABORT Transaction discarded because of previous errors.'); diff --git a/test/unit/command.js b/test/unit/command.js index 56c544dd..c34ca924 100644 --- a/test/unit/command.js +++ b/test/unit/command.js @@ -74,6 +74,22 @@ describe('Command', function () { }); }); + describe('#getSlot()', function () { + it('should return correctly', function () { + expectSlot('123', 5970); + expectSlot('ab{c', 4619); + expectSlot('ab{c}2', 7365); + expectSlot('ab{{c}2', 2150); + expectSlot('ab{qq}{c}2', 5598); + expectSlot('ab}', 11817); + expectSlot('encoding', 3060); + + function expectSlot(key, slot) { + expect(new Command('get', [key]).getSlot()).to.eql(slot); + } + }); + }); + describe('.checkFlag()', function () { it('should return correct result', function () { expect(Command.checkFlag('VALID_IN_SUBSCRIBER_MODE', 'ping')).to.eql(true); diff --git a/test/unit/utils.js b/test/unit/utils.js index 1106cd52..a2664ff3 100644 --- a/test/unit/utils.js +++ b/test/unit/utils.js @@ -1,6 +1,7 @@ 'use strict'; var utils = require('../../lib/utils'); + describe('utils', function () { describe('.bufferEqual', function () { it('should return correctly', function () { @@ -97,18 +98,6 @@ describe('utils', function () { }); }); - describe('.calcSlot', function () { - it('should return correctly', function () { - expect(utils.calcSlot('123')).to.eql(5970); - expect(utils.calcSlot('ab{c')).to.eql(4619); - expect(utils.calcSlot('ab{c}2')).to.eql(7365); - expect(utils.calcSlot('ab{{c}2')).to.eql(2150); - expect(utils.calcSlot('ab{qq}{c}2')).to.eql(5598); - expect(utils.calcSlot('ab}')).to.eql(11817); - expect(utils.calcSlot('encoding')).to.eql(3060); - }); - }); - describe('.toArg', function () { it('should return correctly', function () { expect(utils.toArg(null)).to.eql('');