Improving the parsing of Search query results (#195)

modules/module.base.ts

@@ -110,7 +110,7 @@ export class Module {
      * @param isSearchQuery If we should try to build search result object from result array (default: false)
      */
     // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
-    handleResponse(response: any, isSearchQuery = false): any {
+    handleResponse(response: any): any {
         if(this.returnRawResponse === true) {
             return response
         }
@@ -119,48 +119,13 @@ export class Module {
             (typeof response === 'string' ||
             typeof response === 'number' ||
             (Array.isArray(response) && response.length % 2 === 1 && response.length > 1 && !this.isOnlyTwoDimensionalArray(response)) ||
-            (Array.isArray(response) && response.length === 0)) &&
-            !isSearchQuery
+            (Array.isArray(response) && response.length === 0))
         ) {
             return response;
         }
         else if(Array.isArray(response) && response.length === 1) {
             return this.handleResponse(response[0])
         }
-        else if(isSearchQuery) {
-            //Search queries should be parsed into objects, if possible.
-            let responseObjects = response;
-            if(Array.isArray(response) && response.length % 2 === 1) {
-                // Put index as 0th element
-                responseObjects = [response[0]];
-                // Go through returned keys (doc:1, doc:2, ...)
-                for(let i = 1; i < response.length; i += 2) {
-                    // propertyArray is the key-value pairs eg: ['name', 'John']
-                    const propertyArray = response[i + 1];
-                    responseObjects.push({
-                        key: response[i] //This is the key, 'eg doc:1'
-                    });
-                    if(Array.isArray(propertyArray) && propertyArray.length % 2 === 0) {
-                        for(let j = 0; j < propertyArray.length; j += 2) {
-                            // Add keys to last responseObjects item
-                            // propertyArray[j] = key name
-                            // propertyArray[j+1] = value
-                            responseObjects[responseObjects.length - 1][propertyArray[j]] = propertyArray[j + 1];
-                        }
-                    }
-                }
-            }
-            //Check for a single dimensional array, these should only be keys, if im right
-            else if(response.every(entry => !Array.isArray(entry))) {
-                responseObjects = [response[0]];
-                for(let i = 1; i < response.length; i++) {
-                    responseObjects.push({
-                        key: response[i],
-                    });
-                }
-            }
-            return responseObjects;
-        }
         else if(Array.isArray(response) && response.length > 1 && this.isOnlyTwoDimensionalArray(response)) {
             return this.handleResponse(this.reduceArrayDimension(response))
         }
@@ -227,6 +192,32 @@ export class Module {
     }
 }
 
+/**
+ * Logging a message
+ * @param level The level of the log
+ * @param msg The log message
+ */
+export function log(level: LogLevel, msg: string): void {
+    if(level === LogLevel.DEBUG && this.showDebugLogs === true) {
+        console.debug(msg)
+    }
+    else if(level === LogLevel.Error) {
+        throw new Error(msg)
+    }
+    else {
+        console.log(msg);
+    }
+}
+
+/**
+ * Enum representing the log levels
+ */
+export enum LogLevel {
+    INFO,
+    DEBUG,
+    Error
+}
+
 /**
  * The Redis module class options
  */

modules/redisearch/redisearch.commander.ts

@@ -223,9 +223,6 @@ export class SearchCommander {
             command: 'FT.SEARCH',
             args: args
         }
-        //const response = await this.sendCommand('FT.SEARCH', args);
-        //const parseResponse = parameters?.parseSearchQueries ?? true;
-        //return this.handleResponse(response, parseResponse);
     }
 
     /**

modules/redisearch/redisearch.helpers.ts

@@ -1,6 +1,7 @@
 /* eslint-disable @typescript-eslint/explicit-module-boundary-types */
 /* eslint-disable @typescript-eslint/no-explicit-any */
-import { FTAggregateResponse, FTAggregateResponseItem, FTSpellCheckResponse } from "./redisearch.types";
+import { log, LogLevel } from "../module.base";
+import { FTAggregateResponse, FTAggregateResponseItem, FTParsedSearchResponse, FTSpellCheckResponse } from "./redisearch.types";
 
 export class RedisearchHelpers {
     /**
@@ -44,4 +45,68 @@ export class RedisearchHelpers {
             items: items
         };
     }
+
+    /**
+     * Parsing the response of Search QUERY
+     * @param response The raw response from Redis
+     * @returns A parsed or processed response
+     */
+    handleQueryResponse(response: any) {
+        log(LogLevel.DEBUG, `****** Function handleQueryResponse ******`);
+        //Search queries should be parsed into objects, if possible.
+        let responseObjects = response;
+        //If the response is an array with 1 item, we will return it as the value.
+        if(Array.isArray(response) && response.length === 1 && !Array.isArray(response[0])) {
+            log(LogLevel.DEBUG, `The response is ${response[0]}`);
+            return response[0];
+        }
+        //In case we have an array with a odd number of items, we will parse it as required. 
+        else if(Array.isArray(response) && response.length % 2 === 1) {
+            // Put index as 0th element
+            responseObjects = [response[0]];
+            // Go through returned keys (doc:1, doc:2, ...)
+            for(let i = 1; i < response.length; i += 2) {
+                // propertyArray is the key-value pairs eg: ['name', 'John']
+                const propertyArray = response[i + 1];
+                responseObjects.push({
+                    key: response[i] //This is the key, 'eg doc:1'
+                });
+                if(Array.isArray(propertyArray) && propertyArray.length % 2 === 0) {
+                    for(let j = 0; j < propertyArray.length; j += 2) {
+                        // Add keys to last responseObjects item
+                        // propertyArray[j] = key name
+                        // propertyArray[j+1] = value
+                        responseObjects[responseObjects.length - 1][propertyArray[j]] = propertyArray[j + 1];
+                    }
+                }
+            }
+        }
+        //Check for a single dimensional array, these should only be keys, if im right
+        else if(response.every((entry: any) => !Array.isArray(entry))) {
+            responseObjects = [response[0]];
+            for(let i = 1; i < response.length; i++) {
+                responseObjects.push({
+                    key: response[i],
+                });
+            }
+        }
+        else {
+            log(LogLevel.DEBUG, 'Parsing response to JSON:')
+            const responses = response
+            const resultCounts = responses[0];
+            responseObjects = {}
+            responseObjects.resultsCount = resultCounts;
+            responseObjects.documentIds = []
+            responseObjects.data = []
+            for(let i = 1; i < responses.length; i ++) {
+                if(Array.isArray(responses[i])) {
+                    responseObjects.data = responseObjects.data.concat(responses[i])
+                }
+                else {
+                    responseObjects.documentIds.push(responses[i])
+                }
+            }
+        }
+        return responseObjects as FTParsedSearchResponse;
+    }
 }

modules/redisearch/redisearch.ts

@@ -60,8 +60,11 @@ export class Redisearch extends Module {
     async search(index: string, query: string, parameters?: FTSearchParameters): Promise<FTSearchResponse> {
         const command = this.searchCommander.search(index, query, parameters);
         const response = await this.sendCommand(command);
-        const parseResponse = parameters?.parseSearchQueries ?? true;
-        return this.handleResponse(response, parseResponse);
+        if(this.returnRawResponse === true) {
+            return this.handleResponse(response);
+        }
+
+        return this.searchHelpers.handleQueryResponse(response);
     }
 
     /**

modules/redisearch/redisearch.types.ts

@@ -328,12 +328,7 @@ export interface FTSearchParameters {
         * The num argument of the 'LIMIT' parameter
         */
         num: number
-    },
-    /**
-    * If to parse search results to objects or leave them in their array form
-    * @default true
-    */
-    parseSearchQueries?: boolean
+    }
 }
 
 /**
@@ -654,7 +649,7 @@ export interface FTSpellCheckResponse {
  * The response type of the FT search function.
  * The function results in 0 when no results are found, else as an array.
  */
-export type FTSearchResponse = number | FTSearchArrayResponse;
+export type FTSearchResponse = number | FTSearchArrayResponse | FTParsedSearchResponse;
 
 /**
  * The response type of the FT search function as an array
@@ -675,4 +670,16 @@ export type FTAggregateResponse = {
 export type FTAggregateResponseItem = {
     name: string,
     value: number | string | boolean
+}
+
+/**
+ * The parsed response of the search function
+ * @param resultsCount The number of results returned
+ * @param documentIds Pairs of document IDs
+ * @param data A nested array of attribute/value pairs 
+ */
+export type FTParsedSearchResponse = {
+    resultsCount: number,
+    documentIds: string[],
+    data: string[]
 }

package.json

@@ -68,7 +68,8 @@
     "npm-package-deployer": "0.2.9",
     "ts-node": "9.0.0",
     "typedoc": "0.22.10",
-    "typescript": "4.1.2"
+    "typescript": "4.1.2",
+    "fs": "0.0.1-security"
   },
   "dependencies": {
     "ioredis": "4.27.4"

tests/data/models/sample1.json

@@ -0,0 +1,3 @@
+[
+    {"id":500,"title":"KAS","description":"edge case description"}
+]

tests/redisearch.ts

@@ -1,7 +1,8 @@
 import { cliArguments } from 'cli-argument-parser'
 import { expect } from 'chai'
 import { RedisModules } from '../modules/redis-modules'
-import { FTSearchArrayResponse } from '../modules/redisearch/redisearch.types'
+import { FTParsedSearchResponse, FTSearchArrayResponse } from '../modules/redisearch/redisearch.types'
+import * as fs from 'fs';
 let redis: RedisModules
 const index = 'idx'
 const query = '@text:name'
@@ -20,7 +21,7 @@ describe('RediSearch Module testing', async function () {
         redis = new RedisModules({
             host: cliArguments.host,
             port: parseInt(cliArguments.port)
-        })
+        }, { showDebugLogs: true })
         await redis.connect()
     })
     after(async () => {
@@ -460,4 +461,20 @@ describe('RediSearch Module testing', async function () {
         const response = await redis.search_module_dropindex(`${index}-droptest`)
         expect(response).to.equal('OK', 'The response of the FT.DROPINDEX command')
     })
+    it('Testing the parse of search function as JSON', async () => {
+        const json = fs.readFileSync('tests/data/models/sample1.json', { encoding: 'utf-8'});
+        const parsedJSON = JSON.parse(json);
+        await redis.search_module_create('li-index', 'JSON', [{
+            name: '$.title',
+            type: 'TEXT',
+        }, {
+            name: '$.description',
+            type: 'TEXT',
+        }]);
+
+        await Promise.all(parsedJSON.map(async (p: { id: number; }) => await redis.rejson_module_set(`li:${p.id}`, '$', JSON.stringify(p))));
+        const result = await redis.search_module_search('li-index', 'KAS', { limit: { first: 0, num: 20 }, withScores: true }) as FTParsedSearchResponse;
+        const { resultsCount } = result;
+        expect(resultsCount).to.equal(1, 'The count of the results');
+    })
 })