Add sort parameter to named query config (#3157)

cofiem · web-flow · commit eb484076fb40 · 2025-07-17T16:09:02.000+09:30
* centralise common named query execution into NamedQueryService

Add method for iterating named query result pages.

* add named query sorting parameter

There is no sort order specified by default.
The format in namedQuery.js is the same as one of the sails ORM formats.

* add sort order to namedQuery so it is tested
diff --git a/config/namedQuery.js b/config/namedQuery.js
@@ -13,6 +13,7 @@ module.exports.namedQuery = {
       'metadata.title': null,
       'dateCreated': null
     },
+    sort: [{'lastSaveDate': 'DESC'}],
     queryParams: {
       'title': {
         type: 'string',
diff --git a/core/src/model/storage/NamedQueryModel.ts b/core/src/model/storage/NamedQueryModel.ts
@@ -9,5 +9,5 @@ export class NamedQueryModel {
     collectionName?: string;
     resultObjectMapping?: string;
     brandIdFieldPath?: string;
-
+    sort?: string;
   }
diff --git a/typescript/api/controllers/webservice/ReportController.ts b/typescript/api/controllers/webservice/ReportController.ts
@@ -87,15 +87,8 @@ export module Controllers {
           return this.apiFail(req, res, 400, new APIErrorResponse("Rows must not be greater than 100"));
         }
         let namedQueryConfig = sails.config.namedQuery[queryName];
-
-        let configMongoQuery = namedQueryConfig.mongoQuery;
-        let collectionName = _.get(namedQueryConfig, 'collectionName', '');
-        let resultObjectMapping = _.get(namedQueryConfig, 'resultObjectMapping', {});
-        let brandIdFieldPath = _.get(namedQueryConfig, 'brandIdFieldPath', '');
-        let mongoQuery = _.clone(configMongoQuery);
-        let queryParams = namedQueryConfig.queryParams;
         let paramMap = _.clone(req.query);
-        let response = await NamedQueryService.performNamedQuery(brandIdFieldPath,resultObjectMapping,collectionName,mongoQuery,queryParams,paramMap,brand,start,rows)
+        let response = await NamedQueryService.performNamedQueryFromConfig(namedQueryConfig, paramMap, brand, start, rows);
         sails.log.verbose(`NamedQueryService response: ${JSON.stringify(response)}`);
         return this.apiRespond(req, res, response, 200)
       } catch (error) {
diff --git a/typescript/api/services/NamedQueryService.ts b/typescript/api/services/NamedQueryService.ts
@@ -46,8 +46,10 @@ export module Services {
 
     protected _exportedMethods: any = [
       "bootstrap",
+      "getNamedQueryConfig",
       "performNamedQuery",
-      "getNamedQueryConfig"
+      "performNamedQueryFromConfig",
+      "performNamedQueryFromConfigResults",
     ];
 
      public async bootstrap (defBrand) {
@@ -83,7 +85,8 @@ export module Services {
         queryParams: JSON.stringify(config.queryParams),
         collectionName: config.collectionName,
         resultObjectMapping: JSON.stringify(config.resultObjectMapping),
-        brandIdFieldPath: config.brandIdFieldPath
+        brandIdFieldPath: config.brandIdFieldPath,
+        sort: (config.sort !== undefined && Array.isArray(config.sort) && config.sort.length > 0) ? JSON.stringify(config.sort) : "",
       }));
     }
 
@@ -95,47 +98,47 @@ export module Services {
       return new NamedQueryConfig(nQDBEntry)
     }
 
-    async performNamedQuery(brandIdFieldPath, resultObjectMapping, collectionName, mongoQuery, queryParams, paramMap, brand, start, rows, user=undefined):Promise<ListAPIResponse<Object>> {
-      
+    async performNamedQuery(brandIdFieldPath, resultObjectMapping, collectionName, mongoQuery, queryParams, paramMap, brand, start, rows, user = undefined, sort: NamedQuerySortConfig | undefined = undefined): Promise<ListAPIResponse<Object>> {
+      const criteriaMeta = {enableExperimentalDeepTargets: true};
       this.setParamsInQuery(mongoQuery, queryParams, paramMap);
 
       let that = this;
-      
+
+      // Add branding
       if(brandIdFieldPath != '') {
         mongoQuery[brandIdFieldPath] = brand.id;
       }
-      sails.log.debug("Mongo query to be executed");
-      sails.log.debug(mongoQuery);
-      
+      sails.log.debug("Mongo query to be executed", mongoQuery);
+
+      // Get the total count of matching records
       let totalItems = 0;
       if(collectionName == 'user') {
-        totalItems = await User.count(mongoQuery).meta({
-          enableExperimentalDeepTargets: true
-        });
+        totalItems = await User.count(mongoQuery).meta(criteriaMeta);
       } else {
-        totalItems = await Record.count(mongoQuery).meta({
-          enableExperimentalDeepTargets: true
-        });
+        totalItems = await Record.count(mongoQuery).meta(criteriaMeta);
+      }
+
+      // Build query criteria
+      const criteria = {
+        where: mongoQuery,
+        skip: start,
+        limit: rows
+      };
+
+      // Add sorting
+      if (sort !== undefined && Array.isArray(sort) && (sort?.length ?? 0) > 0) {
+        // e.g. [{ name:  'ASC'}, { age: 'DESC' }]
+        criteria['sort'] = sort;
       }
 
+      // Run query
+      sails.log.debug("Mongo query criteria", criteria);
       let results = [];
       if (totalItems > 0) {
         if(collectionName == 'user') {
-          results = await User.find({
-            where: mongoQuery,
-            skip: start,
-            limit: rows
-          }).meta({
-            enableExperimentalDeepTargets: true
-          });
+          results = await User.find(criteria).meta(criteriaMeta);
         } else {
-          results = await Record.find({
-            where: mongoQuery,
-            skip: start,
-            limit: rows
-          }).meta({
-            enableExperimentalDeepTargets: true
-          });
+          results = await Record.find(criteria).meta(criteriaMeta);
         }
       }
       
@@ -299,6 +302,69 @@ export module Services {
       return _.get(variables, templateOrPath);
     }
 
+    public async performNamedQueryFromConfig(config: NamedQueryConfig, paramMap, brand, start, rows, user?) {
+      sails.log.debug("performNamedQueryFromConfig with parameters", {
+        config: config,
+        paramMap: paramMap,
+        brand: brand,
+        start: start,
+        rows: rows,
+        user: user
+      });
+      const collectionName = _.get(config, 'collectionName', '') ?? '';
+      const resultObjectMapping = _.get(config, 'resultObjectMapping', {}) ?? {};
+      const brandIdFieldPath = _.get(config, 'brandIdFieldPath', '') ?? '';
+      const mongoQuery = _.clone(_.get(config, 'mongoQuery', {}) ?? {});
+      const queryParams = _.get(config, 'queryParams', {}) ?? {};
+      const sort = _.get(config, 'sort', []) ?? [];
+      return await this.performNamedQuery(brandIdFieldPath, resultObjectMapping, collectionName, mongoQuery, queryParams, paramMap, brand, start, rows, user, sort);
+    }
+
+    public async performNamedQueryFromConfigResults(config: NamedQueryConfig, paramMap: Record<string, string>, brand, queryName: string, start: number = 0,rows: number = 30, maxRecords: number = 100, user = undefined) {
+      const records = [];
+      let requestCount = 0;
+      sails.log.debug(`All named query results: start query with name '${queryName}' brand ${JSON.stringify(brand)} start ${start} rows ${rows} paramMap ${JSON.stringify(paramMap)}`);
+
+      while (true) {
+        // Keep going while there are more results.
+
+        const response = await this.performNamedQueryFromConfig(config, paramMap, brand, start, rows, user);
+        requestCount += 1;
+
+        if (!response) {
+          // stop if there is no response
+          sails.log.warn(`All named query results: invalid query response for '${queryName}'`);
+          break;
+        }
+
+        // add the new records to the collected records
+        sails.log.debug(`All named query results: add results for '${queryName}': start ${start} rows ${rows} new results ${response.records.length} summary ${JSON.stringify(response.summary)}`);
+        for (const record of response.records) {
+          records.push(record);
+        }
+
+        const currentRetrievedCount = start + rows;
+        if (response.summary.numFound <= currentRetrievedCount) {
+          // stop if the total count is less than or equal to the number of records retrieved so far
+          sails.log.debug(`All named query results: reached end of results for '${queryName}': start ${start} rows ${rows} total results ${records.length}`);
+          break;
+        }
+
+        // update the start point
+        start = currentRetrievedCount;
+
+        // Check the number of records and fail if it is more than maxRecords.
+        if (records.length > maxRecords){
+          sails.log.warn(`All named query results: returning early before finished with ${records.length} results for '${queryName}' from ${requestCount} requests because the number of records is more than max records ${maxRecords}`);
+        }
+
+        // continue the while loop
+      }
+
+      sails.log.log(`All named query results: returning ${records.length} results for '${queryName}' from ${requestCount} requests`);
+      return records;
+    }
+
   }
 }
 module.exports = new Services.NamedQueryService().exports();
@@ -319,6 +385,8 @@ enum NamedQueryFormatOptions {
   ISODate = 'ISODate'
 }
 
+type NamedQuerySortConfig = Record<string, "ASC" | "DESC">[];
+
 class QueryParameterDefinition {
   required:boolean
   type:DataType
@@ -342,6 +410,7 @@ export class NamedQueryConfig {
   collectionName: string;
   resultObjectMapping: any;
   brandIdFieldPath: string;
+  sort: NamedQuerySortConfig | undefined;
 
   constructor(values:any) {
       this.name = values.name;
@@ -355,6 +424,7 @@ export class NamedQueryConfig {
       this.collectionName = values.collectionName;
       this.resultObjectMapping = JSON.parse(values.resultObjectMapping);
       this.brandIdFieldPath = values.brandIdFieldPath;
+      this.sort = (values.sort !== undefined && values.sort.length > 0) ? JSON.parse(values.sort) : [];
   }
 }
 
diff --git a/typescript/api/services/ReportsService.ts b/typescript/api/services/ReportsService.ts
@@ -160,16 +160,8 @@ export module Services {
       if (report.reportSource == ReportSource.database) {
 
         let namedQueryConfig = await NamedQueryService.getNamedQueryConfig(brand, report.databaseQuery.queryName)
-
-        let configMongoQuery = namedQueryConfig.mongoQuery;
-        let collectionName = _.get(namedQueryConfig, 'collectionName', '');
-        let resultObjectMapping = _.get(namedQueryConfig, 'resultObjectMapping', {});
-        let brandIdFieldPath = _.get(namedQueryConfig, 'brandIdFieldPath', '');
-        let mongoQuery = _.clone(configMongoQuery);
-        let queryParams = namedQueryConfig.queryParams;
         let paramMap = this.buildNamedQueryParamMap(req, report)
-
-        let dbResult = await NamedQueryService.performNamedQuery(brandIdFieldPath, resultObjectMapping, collectionName, mongoQuery, queryParams, paramMap, brand, start, rows);
+        let dbResult = await NamedQueryService.performNamedQueryFromConfig(namedQueryConfig, paramMap, brand, start, rows);
         return this.getTranslateDatabaseResultToReportResult(dbResult, report);
       } else {
         let url = this.buildSolrParams(brand, req, report, start, rows, 'json');
@@ -282,17 +274,9 @@ export module Services {
       let result: ReportResult = null
       if (report.reportSource == ReportSource.database) {
 
-        let namedQueryConfig = await NamedQueryService.getNamedQueryConfig(brand, report.databaseQuery.queryName)
-
-        let configMongoQuery = namedQueryConfig.mongoQuery;
-        let collectionName = _.get(namedQueryConfig, 'collectionName', '');
-        let resultObjectMapping = _.get(namedQueryConfig, 'resultObjectMapping', {});
-        let brandIdFieldPath = _.get(namedQueryConfig, 'brandIdFieldPath', '');
-        let mongoQuery = _.clone(configMongoQuery);
-        let queryParams = namedQueryConfig.queryParams;
-        let paramMap = this.buildNamedQueryParamMap(req, report)
-
-        let dbResult = await NamedQueryService.performNamedQuery(brandIdFieldPath, resultObjectMapping, collectionName, mongoQuery, queryParams, paramMap, brand, start, rows);
+        let namedQueryConfig = await NamedQueryService.getNamedQueryConfig(brand, report.databaseQuery.queryName);
+        let paramMap = this.buildNamedQueryParamMap(req, report);
+        let dbResult = await NamedQueryService.performNamedQueryFromConfig(namedQueryConfig, paramMap, brand, start, rows);
         result = this.getTranslateDatabaseResultToReportResult(dbResult, report);
       } else {
         var url = this.buildSolrParams(brand, req, report, start, rows, 'json');
diff --git a/typescript/api/services/VocabService.ts b/typescript/api/services/VocabService.ts
@@ -152,16 +152,8 @@ export module Services {
       if (queryConfig.querySource == 'database') {
 
         let namedQueryConfig = await NamedQueryService.getNamedQueryConfig(brand, queryConfig.databaseQuery.queryName);
-
-        let configMongoQuery = namedQueryConfig.mongoQuery;
-        let collectionName = _.get(namedQueryConfig, 'collectionName', '');
-        let resultObjectMapping = _.get(namedQueryConfig, 'resultObjectMapping', {});
-        let brandIdFieldPath = _.get(namedQueryConfig, 'brandIdFieldPath', '');
-        let mongoQuery = _.clone(configMongoQuery);
-        let queryParams = namedQueryConfig.queryParams;
         let paramMap = this.buildNamedQueryParamMap(queryConfig, searchString, user);
-
-        let dbResults = await NamedQueryService.performNamedQuery(brandIdFieldPath, resultObjectMapping, collectionName, mongoQuery, queryParams, paramMap, brand, start, rows);
+        let dbResults = await NamedQueryService.performNamedQueryFromConfig(namedQueryConfig, paramMap, brand, start, rows);
         if(queryConfig.resultObjectMapping) {
           return this.getResultObjectMappings(dbResults,queryConfig);
         } else {

Original file line number	Diff line number	Diff line change
`@@ -9,5 +9,5 @@ export class NamedQueryModel {`
`9`	`9`	`collectionName?: string;`
`10`	`10`	`resultObjectMapping?: string;`
`11`	`11`	`brandIdFieldPath?: string;`
`12`		`-`
	`12`	`+ sort?: string;`
`13`	`13`	`}`