From d5544e994a41ef3c408b2ef6786ef96fa95c9583 Mon Sep 17 00:00:00 2001 From: Philipp Date: Thu, 19 Aug 2021 16:42:18 +0200 Subject: [PATCH] feat: query filters --- README.md | 50 +++++++--- package-lock.json | 9 +- package.json | 4 + src/decorator.spec.ts | 7 ++ src/decorator.ts | 17 +++- src/paginate.spec.ts | 218 ++++++++++++++++++++++++++++++++++++++++-- src/paginate.ts | 142 ++++++++++++++++++++++++--- 7 files changed, 408 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 5167d96..79157ae 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Pagination and filtering helper method for TypeORM repositories or query builder - Pagination conforms to [JSON:API](https://jsonapi.org/) - Sort by multiple columns - Search across columns -- Filter using operators *(in progress)* +- Filter using operators (`$eq`, `$not`, `$null`, `$in`, `$gt`, `$gte`, `$lt`, `$lte`) ## Installation @@ -30,7 +30,7 @@ The following code exposes a route that can be utilized like so: #### Endpoint ```url -http://localhost:3000/cats?limit=5&page=2&sortBy=color:DESC&search=i +http://localhost:3000/cats?limit=5&page=2&sortBy=color:DESC&search=i&filter.age=$gte:3 ``` #### Result @@ -41,27 +41,32 @@ http://localhost:3000/cats?limit=5&page=2&sortBy=color:DESC&search=i { "id": 4, "name": "George", - "color": "white" + "color": "white", + "age": 3 }, { "id": 5, "name": "Leche", - "color": "white" + "color": "white", + "age": 6 }, { "id": 2, "name": "Garfield", - "color": "ginger" + "color": "ginger", + "age": 4 }, { "id": 1, "name": "Milo", - "color": "brown" + "color": "brown", + "age": 5 }, { "id": 3, "name": "Kitty", - "color": "black" + "color": "black", + "age": 3 } ], "meta": { @@ -70,14 +75,17 @@ http://localhost:3000/cats?limit=5&page=2&sortBy=color:DESC&search=i "currentPage": 2, "totalPages": 3, "sortBy": [["color", "DESC"]], - "search": "i" + "search": "i", + "filter": { + "age": "$gte:3" + } }, "links": { - "first": "http://localhost:3000/cats?limit=5&page=1&sortBy=color:DESC&search=i", - "previous": "http://localhost:3000/cats?limit=5&page=1&sortBy=color:DESC&search=i", - "current": "http://localhost:3000/cats?limit=5&page=2&sortBy=color:DESC&search=i", - "next": "http://localhost:3000/cats?limit=5&page=3&sortBy=color:DESC&search=i", - "last": "http://localhost:3000/cats?limit=5&page=3&sortBy=color:DESC&search=i" + "first": "http://localhost:3000/cats?limit=5&page=1&sortBy=color:DESC&search=i&filter.age=$gte:3", + "previous": "http://localhost:3000/cats?limit=5&page=1&sortBy=color:DESC&search=i&filter.age=$gte:3", + "current": "http://localhost:3000/cats?limit=5&page=2&sortBy=color:DESC&search=i&filter.age=$gte:3", + "next": "http://localhost:3000/cats?limit=5&page=3&sortBy=color:DESC&search=i&filter.age=$gte:3", + "last": "http://localhost:3000/cats?limit=5&page=3&sortBy=color:DESC&search=i&filter.age=$gte:3" } } ``` @@ -88,7 +96,7 @@ http://localhost:3000/cats?limit=5&page=2&sortBy=color:DESC&search=i import { Controller, Injectable, Get } from '@nestjs/common' import { InjectRepository } from '@nestjs/typeorm' import { Repository, Entity, PrimaryGeneratedColumn, Column } from 'typeorm' -import { Paginate, PaginateQuery, paginate, Paginated } from 'nestjs-paginate' +import { FilterOperator, Paginate, PaginateQuery, paginate, Paginated } from 'nestjs-paginate' @Entity() export class CatEntity { @@ -100,6 +108,9 @@ export class CatEntity { @Column('text') color: string + + @Column('int') + age: number } @Injectable() @@ -114,6 +125,9 @@ export class CatsService { sortableColumns: ['id', 'name', 'color'], searchableColumns: ['name', 'color'], defaultSortBy: [['id', 'DESC']], + filterableColumns: { + age: [FilterOperator.GTE, FilterOperator.LTE], + }, }) } } @@ -177,5 +191,13 @@ const paginateConfig: PaginateConfig { * https://typeorm.io/#/find-optionsfind-options.md */ where: { color: 'ginger' } + + /** + * Required: false + * Type: FilterOperator - Based on TypeORM find operators + * Default: None + * Find more at https://typeorm.io/#/find-options/advanced-options + */ + filterableColumns: { age: [FilterOperator.EQ, FilterOperator.IN] } } ``` diff --git a/package-lock.json b/package-lock.json index 23a8d90..47923d3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1207,6 +1207,12 @@ "integrity": "sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==", "dev": true }, + "@types/lodash": { + "version": "4.14.172", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.172.tgz", + "integrity": "sha512-/BHF5HAx3em7/KkzVKm3LrsD6HZAXuXO1AJZQ3cRRBZj4oHZDviWPYu0aEplAqDFNHZPW6d3G7KN+ONcCCC7pw==", + "dev": true + }, "@types/mime": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", @@ -4575,8 +4581,7 @@ "lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "dev": true + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "lodash.clonedeep": { "version": "4.5.0", diff --git a/package.json b/package.json index 43a3e3c..b0eaa44 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "@nestjs/common": "^8.0.6", "@types/express": "^4.17.13", "@types/jest": "^27.0.1", + "@types/lodash": "^4.14.172", "@types/node": "^16.6.2", "@typescript-eslint/eslint-plugin": "^4.29.2", "@typescript-eslint/parser": "^4.29.2", @@ -82,5 +83,8 @@ "branches": [ "master" ] + }, + "dependencies": { + "lodash": "^4.17.21" } } diff --git a/src/decorator.spec.ts b/src/decorator.spec.ts index d87e8d0..327eedd 100644 --- a/src/decorator.spec.ts +++ b/src/decorator.spec.ts @@ -43,6 +43,7 @@ describe('Decorator', () => { limit: undefined, sortBy: undefined, search: undefined, + filter: undefined, path: 'http://localhost/items', }) }) @@ -53,6 +54,8 @@ describe('Decorator', () => { limit: '20', sortBy: ['id:ASC', 'createdAt:DESC'], search: 'white', + 'filter.name': '$not:$eq:Kitty', + 'filter.createdAt': ['$gte:2020-01-01', '$lte:2020-12-31'], }) const result: PaginateQuery = decoratorfactory(null, context) @@ -66,6 +69,10 @@ describe('Decorator', () => { ], search: 'white', path: 'http://localhost/items', + filter: { + name: '$not:$eq:Kitty', + createdAt: ['$gte:2020-01-01', '$lte:2020-12-31'], + }, }) }) }) diff --git a/src/decorator.ts b/src/decorator.ts index 9b1ad99..22f9c31 100644 --- a/src/decorator.ts +++ b/src/decorator.ts @@ -1,11 +1,13 @@ import { createParamDecorator, ExecutionContext } from '@nestjs/common' import { Request } from 'express' +import { pickBy, Dictionary, isString, mapKeys } from 'lodash' export interface PaginateQuery { page?: number limit?: number sortBy?: [string, string][] search?: string + filter?: { [column: string]: string | string[] } path: string } @@ -17,8 +19,8 @@ export const Paginate = createParamDecorator((_data: unknown, ctx: ExecutionCont const sortBy: [string, string][] = [] if (query.sortBy) { const params = !Array.isArray(query.sortBy) ? [query.sortBy] : query.sortBy - for (const param of params as string[]) { - if (typeof param === 'string') { + for (const param of params) { + if (isString(param)) { const items = param.split(':') if (items.length === 2) { sortBy.push(items as [string, string]) @@ -27,11 +29,22 @@ export const Paginate = createParamDecorator((_data: unknown, ctx: ExecutionCont } } + const filter = mapKeys( + pickBy( + query, + (param, name) => + name.includes('filter.') && + (isString(param) || (Array.isArray(param) && (param as any[]).every((p) => isString(p)))) + ) as Dictionary, + (_param, name) => name.replace('filter.', '') + ) + return { page: query.page ? parseInt(query.page.toString(), 10) : undefined, limit: query.limit ? parseInt(query.limit.toString(), 10) : undefined, sortBy: sortBy.length ? sortBy : undefined, search: query.search ? query.search.toString() : undefined, + filter: Object.keys(filter).length ? filter : undefined, path, } }) diff --git a/src/paginate.spec.ts b/src/paginate.spec.ts index e9ce1bf..8882fb4 100644 --- a/src/paginate.spec.ts +++ b/src/paginate.spec.ts @@ -1,5 +1,5 @@ -import { createConnection, Repository, Column } from 'typeorm' -import { Paginated, paginate, PaginateConfig } from './paginate' +import { createConnection, Repository, Column, In } from 'typeorm' +import { Paginated, paginate, PaginateConfig, FilterOperator } from './paginate' import { PaginateQuery } from './decorator' import { Entity, PrimaryGeneratedColumn, CreateDateColumn } from 'typeorm' import { HttpException } from '@nestjs/common' @@ -15,6 +15,9 @@ export class CatEntity { @Column() color: string + @Column({ nullable: true }) + age: number | null + @CreateDateColumn() createdAt: string } @@ -33,18 +36,18 @@ describe('paginate', () => { }) repo = connection.getRepository(CatEntity) cats = await repo.save([ - repo.create({ name: 'Milo', color: 'brown' }), - repo.create({ name: 'Garfield', color: 'ginger' }), - repo.create({ name: 'Shadow', color: 'black' }), - repo.create({ name: 'George', color: 'white' }), - repo.create({ name: 'Leche', color: 'white' }), + repo.create({ name: 'Milo', color: 'brown', age: 6 }), + repo.create({ name: 'Garfield', color: 'ginger', age: null }), + repo.create({ name: 'Shadow', color: 'black', age: 4 }), + repo.create({ name: 'George', color: 'white', age: 3 }), + repo.create({ name: 'Leche', color: 'white', age: null }), ]) }) it('should return an instance of Paginated', async () => { const config: PaginateConfig = { sortableColumns: ['id'], - defaultSortBy: [['createdAt', 'DESC']], + defaultSortBy: [['id', 'ASC']], defaultLimit: 1, } const query: PaginateQuery = { @@ -162,6 +165,205 @@ describe('paginate', () => { expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&search=i') }) + it('should return result based on where config and filter', async () => { + const config: PaginateConfig = { + sortableColumns: ['id'], + where: { + color: 'white', + }, + filterableColumns: { + name: [FilterOperator.NOT], + }, + } + const query: PaginateQuery = { + path: '', + filter: { + name: '$not:Leche', + }, + } + + const result = await paginate(query, repo, config) + + expect(result.meta.filter).toStrictEqual({ + name: '$not:Leche', + }) + expect(result.data).toStrictEqual([cats[3]]) + expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&filter.name=$not:Leche') + }) + + it('should return result based on multiple filter', async () => { + const config: PaginateConfig = { + sortableColumns: ['id'], + filterableColumns: { + name: [FilterOperator.NOT], + color: [FilterOperator.EQ], + }, + } + const query: PaginateQuery = { + path: '', + filter: { + name: '$not:Leche', + color: 'white', + }, + } + + const result = await paginate(query, repo, config) + + expect(result.meta.filter).toStrictEqual({ + name: '$not:Leche', + color: 'white', + }) + expect(result.data).toStrictEqual([cats[3]]) + expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&filter.name=$not:Leche&filter.color=white') + }) + + it('should return result based on filter and search term', async () => { + const config: PaginateConfig = { + sortableColumns: ['id'], + searchableColumns: ['name', 'color'], + filterableColumns: { + id: [FilterOperator.NOT, FilterOperator.IN], + }, + } + const query: PaginateQuery = { + path: '', + search: 'white', + filter: { + id: '$not:$in:1,2,5', + }, + } + + const result = await paginate(query, repo, config) + + expect(result.meta.search).toStrictEqual('white') + expect(result.meta.filter).toStrictEqual({ id: '$not:$in:1,2,5' }) + expect(result.data).toStrictEqual([cats[3]]) + expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&search=white&filter.id=$not:$in:1,2,5') + }) + + it('should return result based on filter and where config', async () => { + const config: PaginateConfig = { + sortableColumns: ['id'], + where: { + color: In(['black', 'white']), + }, + filterableColumns: { + id: [FilterOperator.NOT, FilterOperator.IN], + }, + } + const query: PaginateQuery = { + path: '', + filter: { + id: '$not:$in:1,2,5', + }, + } + + const result = await paginate(query, repo, config) + + expect(result.data).toStrictEqual([cats[2], cats[3]]) + expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&filter.id=$not:$in:1,2,5') + }) + + it('should return result based on range filter', async () => { + const config: PaginateConfig = { + sortableColumns: ['id'], + filterableColumns: { + age: [FilterOperator.GTE], + }, + } + const query: PaginateQuery = { + path: '', + filter: { + age: '$gte:4', + }, + } + + const result = await paginate(query, repo, config) + + expect(result.data).toStrictEqual([cats[0], cats[2]]) + expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&filter.age=$gte:4') + }) + + it('should return result based on is null query', async () => { + const config: PaginateConfig = { + sortableColumns: ['id'], + filterableColumns: { + age: [FilterOperator.NULL], + }, + } + const query: PaginateQuery = { + path: '', + filter: { + age: '$null', + }, + } + + const result = await paginate(query, repo, config) + + expect(result.data).toStrictEqual([cats[1], cats[4]]) + expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&filter.age=$null') + }) + + it('should return result based on not null query', async () => { + const config: PaginateConfig = { + sortableColumns: ['id'], + filterableColumns: { + age: [FilterOperator.NOT, FilterOperator.NULL], + }, + } + const query: PaginateQuery = { + path: '', + filter: { + age: '$not:$null', + }, + } + + const result = await paginate(query, repo, config) + + expect(result.data).toStrictEqual([cats[0], cats[2], cats[3]]) + expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&filter.age=$not:$null') + }) + + it('should ignore filterable column which is not configured', async () => { + const config: PaginateConfig = { + sortableColumns: ['id'], + filterableColumns: { + name: [FilterOperator.NOT, FilterOperator.NULL], + }, + } + const query: PaginateQuery = { + path: '', + filter: { + age: '$not:$null', + }, + } + + const result = await paginate(query, repo, config) + + expect(result.data).toStrictEqual(cats) + expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&filter.age=$not:$null') + }) + + it('should ignore filter operator which is not configured', async () => { + const config: PaginateConfig = { + sortableColumns: ['id'], + filterableColumns: { + age: [FilterOperator.NOT], + }, + } + const query: PaginateQuery = { + path: '', + filter: { + age: '$not:$null', + }, + } + + const result = await paginate(query, repo, config) + + expect(result.data).toStrictEqual(cats) + expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&filter.age=$not:$null') + }) + it('should throw an error when no sortableColumns', async () => { const config: PaginateConfig = { sortableColumns: [], diff --git a/src/paginate.ts b/src/paginate.ts index eb08e9f..7448e60 100644 --- a/src/paginate.ts +++ b/src/paginate.ts @@ -1,6 +1,23 @@ -import { Repository, FindConditions, SelectQueryBuilder, ObjectLiteral, ILike } from 'typeorm' +import { + Repository, + FindConditions, + SelectQueryBuilder, + ObjectLiteral, + FindOperator, + Equal, + MoreThan, + MoreThanOrEqual, + In, + IsNull, + LessThan, + LessThanOrEqual, + Not, + ILike, +} from 'typeorm' import { PaginateQuery } from './decorator' import { ServiceUnavailableException } from '@nestjs/common' +import { values, mapKeys } from 'lodash' +import { stringify } from 'querystring' type Column = Extract type Order = [Column, 'ASC' | 'DESC'] @@ -15,6 +32,7 @@ export class Paginated { totalPages: number sortBy: SortBy search: string + filter?: { [column: string]: string | string[] } } links: { first?: string @@ -32,7 +50,18 @@ export interface PaginateConfig { defaultSortBy?: SortBy defaultLimit?: number where?: FindConditions - queryBuilder?: SelectQueryBuilder + filterableColumns?: { [key in Column]?: FilterOperator[] } +} + +export enum FilterOperator { + EQ = '$eq', + GT = '$gt', + GTE = '$gte', + IN = '$in', + NULL = '$null', + LT = '$lt', + LTE = '$lte', + NOT = '$not', } export async function paginate( @@ -43,7 +72,6 @@ export async function paginate( let page = query.page || 1 const limit = Math.min(query.limit || config.defaultLimit || 20, config.maxLimit || 100) const sortBy = [] as SortBy - const search = query.search const path = query.path function isEntityKey(sortableColumns: Column[], column: string): column is Column { @@ -87,21 +115,108 @@ export async function paginate( } } - const where: ObjectLiteral[] = [] - if (search && config.searchableColumns) { - for (const column of config.searchableColumns) { - where.push({ [column]: ILike(`%${search}%`), ...config.where }) - } + let hasWhereClause = false + + if (config.where) { + queryBuilder = queryBuilder.where(config.where) + hasWhereClause = true } - ;[items, totalItems] = await queryBuilder.where(where.length ? where : config.where || {}).getManyAndCount() + if (query.search && config.searchableColumns) { + const search: ObjectLiteral[] = [] + for (const column of config.searchableColumns) { + search.push({ [column]: ILike(`%${query.search}%`) }) + } + queryBuilder = queryBuilder[hasWhereClause ? 'andWhere' : 'where'](search) + hasWhereClause = true + } + + if (query.filter) { + const filter = {} + function getOperatorFn(op: FilterOperator): (...args: any[]) => FindOperator { + switch (op) { + case FilterOperator.EQ: + return Equal + case FilterOperator.GT: + return MoreThan + case FilterOperator.GTE: + return MoreThanOrEqual + case FilterOperator.IN: + return In + case FilterOperator.NULL: + return IsNull + case FilterOperator.LT: + return LessThan + case FilterOperator.LTE: + return LessThanOrEqual + case FilterOperator.NOT: + return Not + } + } + function isOperator(value: any): value is FilterOperator { + return values(FilterOperator).includes(value) + } + for (const column of Object.keys(query.filter)) { + if (!(column in config.filterableColumns)) { + continue + } + const allowedOperators = config.filterableColumns[column as Column] + const input = query.filter[column] + const statements = !Array.isArray(input) ? [input] : input + for (const raw of statements) { + const tokens = raw.split(':') + if (tokens.length === 0 || tokens.length > 3) { + continue + } else if (tokens.length === 2) { + if (tokens[1] !== FilterOperator.NULL) { + tokens.unshift(null) + } + } else if (tokens.length === 1) { + if (tokens[0] === FilterOperator.NULL) { + tokens.unshift(null) + } else { + tokens.unshift(null, FilterOperator.EQ) + } + } + const [op2, op1, value] = tokens + + if (!isOperator(op1) || !allowedOperators.includes(op1)) { + continue + } + if (isOperator(op2) && !allowedOperators.includes(op2)) { + continue + } + if (isOperator(op1)) { + const args = op1 === FilterOperator.IN ? value.split(',') : value + filter[column] = getOperatorFn(op1)(args) + } + if (isOperator(op2)) { + filter[column] = getOperatorFn(op2)(filter[column]) + } + } + } + + queryBuilder = queryBuilder[hasWhereClause ? 'andWhere' : 'where'](filter) + } + + ;[items, totalItems] = await queryBuilder.getManyAndCount() let totalPages = totalItems / limit if (totalItems % limit) totalPages = Math.ceil(totalPages) - const options = `&limit=${limit}${sortBy.map((order) => `&sortBy=${order.join(':')}`).join('')}${ - search ? `&search=${search}` : '' - }` + const sortByQuery = sortBy.map((order) => `&sortBy=${order.join(':')}`).join('') + const searchQuery = query.search ? `&search=${query.search}` : '' + const filterQuery = query.filter + ? '&' + + stringify( + mapKeys(query.filter, (_param, name) => 'filter.' + name), + '&', + '=', + { encodeURIComponent: (str) => str } + ) + : '' + + const options = `&limit=${limit}${sortByQuery}${searchQuery}${filterQuery}` const buildLink = (p: number): string => path + '?page=' + p + options @@ -113,7 +228,8 @@ export async function paginate( currentPage: page, totalPages: totalPages, sortBy, - search, + search: query.search, + filter: query.filter, }, links: { first: page == 1 ? undefined : buildLink(1),