From 5ea1a8e8cdb7d5ccde2dbc46cc9cd5cd191336b3 Mon Sep 17 00:00:00 2001 From: Dani Tseitlin Date: Thu, 30 Dec 2021 21:38:56 +0200 Subject: [PATCH] Improving the parsing of Search query results (#195) --- modules/module.base.ts | 65 +++++++++------------ modules/redisearch/redisearch.commander.ts | 3 - modules/redisearch/redisearch.helpers.ts | 67 +++++++++++++++++++++- modules/redisearch/redisearch.ts | 7 ++- modules/redisearch/redisearch.types.ts | 21 ++++--- package-lock.json | 6 ++ package.json | 3 +- tests/data/models/sample1.json | 3 + tests/redisearch.ts | 21 ++++++- 9 files changed, 143 insertions(+), 53 deletions(-) create mode 100644 tests/data/models/sample1.json diff --git a/modules/module.base.ts b/modules/module.base.ts index 981039a5..6aa7851e 100644 --- a/modules/module.base.ts +++ b/modules/module.base.ts @@ -110,7 +110,7 @@ export class Module { * @param isSearchQuery If we should try to build search result object from result array (default: false) */ // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types - handleResponse(response: any, isSearchQuery = false): any { + handleResponse(response: any): any { if(this.returnRawResponse === true) { return response } @@ -119,48 +119,13 @@ export class Module { (typeof response === 'string' || typeof response === 'number' || (Array.isArray(response) && response.length % 2 === 1 && response.length > 1 && !this.isOnlyTwoDimensionalArray(response)) || - (Array.isArray(response) && response.length === 0)) && - !isSearchQuery + (Array.isArray(response) && response.length === 0)) ) { return response; } else if(Array.isArray(response) && response.length === 1) { return this.handleResponse(response[0]) } - else if(isSearchQuery) { - //Search queries should be parsed into objects, if possible. - let responseObjects = response; - if(Array.isArray(response) && response.length % 2 === 1) { - // Put index as 0th element - responseObjects = [response[0]]; - // Go through returned keys (doc:1, doc:2, ...) - for(let i = 1; i < response.length; i += 2) { - // propertyArray is the key-value pairs eg: ['name', 'John'] - const propertyArray = response[i + 1]; - responseObjects.push({ - key: response[i] //This is the key, 'eg doc:1' - }); - if(Array.isArray(propertyArray) && propertyArray.length % 2 === 0) { - for(let j = 0; j < propertyArray.length; j += 2) { - // Add keys to last responseObjects item - // propertyArray[j] = key name - // propertyArray[j+1] = value - responseObjects[responseObjects.length - 1][propertyArray[j]] = propertyArray[j + 1]; - } - } - } - } - //Check for a single dimensional array, these should only be keys, if im right - else if(response.every(entry => !Array.isArray(entry))) { - responseObjects = [response[0]]; - for(let i = 1; i < response.length; i++) { - responseObjects.push({ - key: response[i], - }); - } - } - return responseObjects; - } else if(Array.isArray(response) && response.length > 1 && this.isOnlyTwoDimensionalArray(response)) { return this.handleResponse(this.reduceArrayDimension(response)) } @@ -227,6 +192,32 @@ export class Module { } } +/** + * Logging a message + * @param level The level of the log + * @param msg The log message + */ +export function log(level: LogLevel, msg: string): void { + if(level === LogLevel.DEBUG && this.showDebugLogs === true) { + console.debug(msg) + } + else if(level === LogLevel.Error) { + throw new Error(msg) + } + else { + console.log(msg); + } +} + +/** + * Enum representing the log levels + */ +export enum LogLevel { + INFO, + DEBUG, + Error +} + /** * The Redis module class options */ diff --git a/modules/redisearch/redisearch.commander.ts b/modules/redisearch/redisearch.commander.ts index c81cdc57..cc2c17c3 100644 --- a/modules/redisearch/redisearch.commander.ts +++ b/modules/redisearch/redisearch.commander.ts @@ -223,9 +223,6 @@ export class SearchCommander { command: 'FT.SEARCH', args: args } - //const response = await this.sendCommand('FT.SEARCH', args); - //const parseResponse = parameters?.parseSearchQueries ?? true; - //return this.handleResponse(response, parseResponse); } /** diff --git a/modules/redisearch/redisearch.helpers.ts b/modules/redisearch/redisearch.helpers.ts index 40dcbf43..a7afb573 100644 --- a/modules/redisearch/redisearch.helpers.ts +++ b/modules/redisearch/redisearch.helpers.ts @@ -1,6 +1,7 @@ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { FTAggregateResponse, FTAggregateResponseItem, FTSpellCheckResponse } from "./redisearch.types"; +import { log, LogLevel } from "../module.base"; +import { FTAggregateResponse, FTAggregateResponseItem, FTParsedSearchResponse, FTSpellCheckResponse } from "./redisearch.types"; export class RedisearchHelpers { /** @@ -44,4 +45,68 @@ export class RedisearchHelpers { items: items }; } + + /** + * Parsing the response of Search QUERY + * @param response The raw response from Redis + * @returns A parsed or processed response + */ + handleQueryResponse(response: any) { + log(LogLevel.DEBUG, `****** Function handleQueryResponse ******`); + //Search queries should be parsed into objects, if possible. + let responseObjects = response; + //If the response is an array with 1 item, we will return it as the value. + if(Array.isArray(response) && response.length === 1 && !Array.isArray(response[0])) { + log(LogLevel.DEBUG, `The response is ${response[0]}`); + return response[0]; + } + //In case we have an array with a odd number of items, we will parse it as required. + else if(Array.isArray(response) && response.length % 2 === 1) { + // Put index as 0th element + responseObjects = [response[0]]; + // Go through returned keys (doc:1, doc:2, ...) + for(let i = 1; i < response.length; i += 2) { + // propertyArray is the key-value pairs eg: ['name', 'John'] + const propertyArray = response[i + 1]; + responseObjects.push({ + key: response[i] //This is the key, 'eg doc:1' + }); + if(Array.isArray(propertyArray) && propertyArray.length % 2 === 0) { + for(let j = 0; j < propertyArray.length; j += 2) { + // Add keys to last responseObjects item + // propertyArray[j] = key name + // propertyArray[j+1] = value + responseObjects[responseObjects.length - 1][propertyArray[j]] = propertyArray[j + 1]; + } + } + } + } + //Check for a single dimensional array, these should only be keys, if im right + else if(response.every((entry: any) => !Array.isArray(entry))) { + responseObjects = [response[0]]; + for(let i = 1; i < response.length; i++) { + responseObjects.push({ + key: response[i], + }); + } + } + else { + log(LogLevel.DEBUG, 'Parsing response to JSON:') + const responses = response + const resultCounts = responses[0]; + responseObjects = {} + responseObjects.resultsCount = resultCounts; + responseObjects.documentIds = [] + responseObjects.data = [] + for(let i = 1; i < responses.length; i ++) { + if(Array.isArray(responses[i])) { + responseObjects.data = responseObjects.data.concat(responses[i]) + } + else { + responseObjects.documentIds.push(responses[i]) + } + } + } + return responseObjects as FTParsedSearchResponse; + } } \ No newline at end of file diff --git a/modules/redisearch/redisearch.ts b/modules/redisearch/redisearch.ts index 38e91bde..be075ee4 100644 --- a/modules/redisearch/redisearch.ts +++ b/modules/redisearch/redisearch.ts @@ -60,8 +60,11 @@ export class Redisearch extends Module { async search(index: string, query: string, parameters?: FTSearchParameters): Promise { const command = this.searchCommander.search(index, query, parameters); const response = await this.sendCommand(command); - const parseResponse = parameters?.parseSearchQueries ?? true; - return this.handleResponse(response, parseResponse); + if(this.returnRawResponse === true) { + return this.handleResponse(response); + } + + return this.searchHelpers.handleQueryResponse(response); } /** diff --git a/modules/redisearch/redisearch.types.ts b/modules/redisearch/redisearch.types.ts index 777c77e2..ec8d578d 100644 --- a/modules/redisearch/redisearch.types.ts +++ b/modules/redisearch/redisearch.types.ts @@ -328,12 +328,7 @@ export interface FTSearchParameters { * The num argument of the 'LIMIT' parameter */ num: number - }, - /** - * If to parse search results to objects or leave them in their array form - * @default true - */ - parseSearchQueries?: boolean + } } /** @@ -654,7 +649,7 @@ export interface FTSpellCheckResponse { * The response type of the FT search function. * The function results in 0 when no results are found, else as an array. */ -export type FTSearchResponse = number | FTSearchArrayResponse; +export type FTSearchResponse = number | FTSearchArrayResponse | FTParsedSearchResponse; /** * The response type of the FT search function as an array @@ -675,4 +670,16 @@ export type FTAggregateResponse = { export type FTAggregateResponseItem = { name: string, value: number | string | boolean +} + +/** + * The parsed response of the search function + * @param resultsCount The number of results returned + * @param documentIds Pairs of document IDs + * @param data A nested array of attribute/value pairs + */ +export type FTParsedSearchResponse = { + resultsCount: number, + documentIds: string[], + data: string[] } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 9f833ccb..fb7cfae6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -894,6 +894,12 @@ "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==", "dev": true }, + "fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha1-invTcYa23d84E/I4WLV+yq9eQdQ=", + "dev": true + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", diff --git a/package.json b/package.json index 4e2c449a..b45f9ebf 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,8 @@ "npm-package-deployer": "0.2.9", "ts-node": "9.0.0", "typedoc": "0.22.10", - "typescript": "4.1.2" + "typescript": "4.1.2", + "fs": "0.0.1-security" }, "dependencies": { "ioredis": "4.27.4" diff --git a/tests/data/models/sample1.json b/tests/data/models/sample1.json new file mode 100644 index 00000000..ec884a1c --- /dev/null +++ b/tests/data/models/sample1.json @@ -0,0 +1,3 @@ +[ + {"id":500,"title":"KAS","description":"edge case description"} +] \ No newline at end of file diff --git a/tests/redisearch.ts b/tests/redisearch.ts index 28e0002c..ca396650 100644 --- a/tests/redisearch.ts +++ b/tests/redisearch.ts @@ -1,7 +1,8 @@ import { cliArguments } from 'cli-argument-parser' import { expect } from 'chai' import { RedisModules } from '../modules/redis-modules' -import { FTSearchArrayResponse } from '../modules/redisearch/redisearch.types' +import { FTParsedSearchResponse, FTSearchArrayResponse } from '../modules/redisearch/redisearch.types' +import * as fs from 'fs'; let redis: RedisModules const index = 'idx' const query = '@text:name' @@ -20,7 +21,7 @@ describe('RediSearch Module testing', async function () { redis = new RedisModules({ host: cliArguments.host, port: parseInt(cliArguments.port) - }) + }, { showDebugLogs: true }) await redis.connect() }) after(async () => { @@ -460,4 +461,20 @@ describe('RediSearch Module testing', async function () { const response = await redis.search_module_dropindex(`${index}-droptest`) expect(response).to.equal('OK', 'The response of the FT.DROPINDEX command') }) + it('Testing the parse of search function as JSON', async () => { + const json = fs.readFileSync('tests/data/models/sample1.json', { encoding: 'utf-8'}); + const parsedJSON = JSON.parse(json); + await redis.search_module_create('li-index', 'JSON', [{ + name: '$.title', + type: 'TEXT', + }, { + name: '$.description', + type: 'TEXT', + }]); + + await Promise.all(parsedJSON.map(async (p: { id: number; }) => await redis.rejson_module_set(`li:${p.id}`, '$', JSON.stringify(p)))); + const result = await redis.search_module_search('li-index', 'KAS', { limit: { first: 0, num: 20 }, withScores: true }) as FTParsedSearchResponse; + const { resultsCount } = result; + expect(resultsCount).to.equal(1, 'The count of the results'); + }) }) \ No newline at end of file