feat: query filters

README.md

@@ -13,7 +13,7 @@ Pagination and filtering helper method for TypeORM repositories or query builder
 - Pagination conforms to [JSON:API](https://jsonapi.org/)
 - Sort by multiple columns
 - Search across columns
-- Filter using operators *(in progress)*
+- Filter using operators (`$eq`, `$not`, `$null`, `$in`, `$gt`, `$gte`, `$lt`, `$lte`)
 
 ## Installation
 
@@ -30,7 +30,7 @@ The following code exposes a route that can be utilized like so:
 #### Endpoint
 
 ```url
-http://localhost:3000/cats?limit=5&page=2&sortBy=color:DESC&search=i
+http://localhost:3000/cats?limit=5&page=2&sortBy=color:DESC&search=i&filter.age=$gte:3
 ```
 
 #### Result
@@ -41,27 +41,32 @@ http://localhost:3000/cats?limit=5&page=2&sortBy=color:DESC&search=i
     {
       "id": 4,
       "name": "George",
-      "color": "white"
+      "color": "white",
+      "age": 3
     },
     {
       "id": 5,
       "name": "Leche",
-      "color": "white"
+      "color": "white",
+      "age": 6
     },
     {
       "id": 2,
       "name": "Garfield",
-      "color": "ginger"
+      "color": "ginger",
+      "age": 4
     },
     {
       "id": 1,
       "name": "Milo",
-      "color": "brown"
+      "color": "brown",
+      "age": 5
     },
     {
       "id": 3,
       "name": "Kitty",
-      "color": "black"
+      "color": "black",
+      "age": 3
     }
   ],
   "meta": {
@@ -70,14 +75,17 @@ http://localhost:3000/cats?limit=5&page=2&sortBy=color:DESC&search=i
     "currentPage": 2,
     "totalPages": 3,
     "sortBy": [["color", "DESC"]],
-    "search": "i"
+    "search": "i",
+    "filter": {
+      "age": "$gte:3"
+    }
   },
   "links": {
-    "first": "http://localhost:3000/cats?limit=5&page=1&sortBy=color:DESC&search=i",
+    "first": "http://localhost:3000/cats?limit=5&page=1&sortBy=color:DESC&search=i&filter.age=$gte:3",
-    "previous": "http://localhost:3000/cats?limit=5&page=1&sortBy=color:DESC&search=i",
+    "previous": "http://localhost:3000/cats?limit=5&page=1&sortBy=color:DESC&search=i&filter.age=$gte:3",
-    "current": "http://localhost:3000/cats?limit=5&page=2&sortBy=color:DESC&search=i",
+    "current": "http://localhost:3000/cats?limit=5&page=2&sortBy=color:DESC&search=i&filter.age=$gte:3",
-    "next": "http://localhost:3000/cats?limit=5&page=3&sortBy=color:DESC&search=i",
+    "next": "http://localhost:3000/cats?limit=5&page=3&sortBy=color:DESC&search=i&filter.age=$gte:3",
-    "last": "http://localhost:3000/cats?limit=5&page=3&sortBy=color:DESC&search=i"
+    "last": "http://localhost:3000/cats?limit=5&page=3&sortBy=color:DESC&search=i&filter.age=$gte:3"
   }
 }
 ```
@@ -88,7 +96,7 @@ http://localhost:3000/cats?limit=5&page=2&sortBy=color:DESC&search=i
 import { Controller, Injectable, Get } from '@nestjs/common'
 import { InjectRepository } from '@nestjs/typeorm'
 import { Repository, Entity, PrimaryGeneratedColumn, Column } from 'typeorm'
-import { Paginate, PaginateQuery, paginate, Paginated } from 'nestjs-paginate'
+import { FilterOperator, Paginate, PaginateQuery, paginate, Paginated } from 'nestjs-paginate'
 
 @Entity()
 export class CatEntity {
@@ -100,6 +108,9 @@ export class CatEntity {
 
   @Column('text')
   color: string
+
+  @Column('int')
+  age: number
 }
 
 @Injectable()
@@ -114,6 +125,9 @@ export class CatsService {
       sortableColumns: ['id', 'name', 'color'],
       searchableColumns: ['name', 'color'],
       defaultSortBy: [['id', 'DESC']],
+      filterableColumns: {
+        age: [FilterOperator.GTE, FilterOperator.LTE],
+      },
     })
   }
 }
@@ -177,5 +191,13 @@ const paginateConfig: PaginateConfig<CatEntity> {
    * https://typeorm.io/#/find-optionsfind-options.md
    */
   where: { color: 'ginger' }
+
+  /**
+   * Required: false
+   * Type: FilterOperator - Based on TypeORM find operators
+   * Default: None
+   * Find more at https://typeorm.io/#/find-options/advanced-options
+   */
+  filterableColumns: { age: [FilterOperator.EQ, FilterOperator.IN] }
 }
 ```

package-lock.json

@@ -1207,6 +1207,12 @@
       "integrity": "sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==",
       "dev": true
     },
+    "@types/lodash": {
+      "version": "4.14.172",
+      "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.172.tgz",
+      "integrity": "sha512-/BHF5HAx3em7/KkzVKm3LrsD6HZAXuXO1AJZQ3cRRBZj4oHZDviWPYu0aEplAqDFNHZPW6d3G7KN+ONcCCC7pw==",
+      "dev": true
+    },
     "@types/mime": {
       "version": "1.3.2",
       "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz",
@@ -4575,8 +4581,7 @@
     "lodash": {
       "version": "4.17.21",
       "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
-      "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==",
+      "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
-      "dev": true
     },
     "lodash.clonedeep": {
       "version": "4.5.0",

package.json

@@ -32,6 +32,7 @@
     "@nestjs/common": "^8.0.6",
     "@types/express": "^4.17.13",
     "@types/jest": "^27.0.1",
+    "@types/lodash": "^4.14.172",
     "@types/node": "^16.6.2",
     "@typescript-eslint/eslint-plugin": "^4.29.2",
     "@typescript-eslint/parser": "^4.29.2",
@@ -82,5 +83,8 @@
     "branches": [
       "master"
     ]
+  },
+  "dependencies": {
+    "lodash": "^4.17.21"
   }
 }

src/decorator.spec.ts

@@ -43,6 +43,7 @@ describe('Decorator', () => {
             limit: undefined,
             sortBy: undefined,
             search: undefined,
+            filter: undefined,
             path: 'http://localhost/items',
         })
     })
@@ -53,6 +54,8 @@ describe('Decorator', () => {
             limit: '20',
             sortBy: ['id:ASC', 'createdAt:DESC'],
             search: 'white',
+            'filter.name': '$not:$eq:Kitty',
+            'filter.createdAt': ['$gte:2020-01-01', '$lte:2020-12-31'],
         })
 
         const result: PaginateQuery = decoratorfactory(null, context)
@@ -66,6 +69,10 @@ describe('Decorator', () => {
             ],
             search: 'white',
             path: 'http://localhost/items',
+            filter: {
+                name: '$not:$eq:Kitty',
+                createdAt: ['$gte:2020-01-01', '$lte:2020-12-31'],
+            },
         })
     })
 })

src/decorator.ts

@@ -1,11 +1,13 @@
 import { createParamDecorator, ExecutionContext } from '@nestjs/common'
 import { Request } from 'express'
+import { pickBy, Dictionary, isString, mapKeys } from 'lodash'
 
 export interface PaginateQuery {
     page?: number
     limit?: number
     sortBy?: [string, string][]
     search?: string
+    filter?: { [column: string]: string | string[] }
     path: string
 }
 
@@ -17,8 +19,8 @@ export const Paginate = createParamDecorator((_data: unknown, ctx: ExecutionCont
     const sortBy: [string, string][] = []
     if (query.sortBy) {
         const params = !Array.isArray(query.sortBy) ? [query.sortBy] : query.sortBy
-        for (const param of params as string[]) {
+        for (const param of params) {
-            if (typeof param === 'string') {
+            if (isString(param)) {
                 const items = param.split(':')
                 if (items.length === 2) {
                     sortBy.push(items as [string, string])
@@ -27,11 +29,22 @@ export const Paginate = createParamDecorator((_data: unknown, ctx: ExecutionCont
         }
     }
 
+    const filter = mapKeys(
+        pickBy(
+            query,
+            (param, name) =>
+                name.includes('filter.') &&
+                (isString(param) || (Array.isArray(param) && (param as any[]).every((p) => isString(p))))
+        ) as Dictionary<string | string[]>,
+        (_param, name) => name.replace('filter.', '')
+    )
+
     return {
         page: query.page ? parseInt(query.page.toString(), 10) : undefined,
         limit: query.limit ? parseInt(query.limit.toString(), 10) : undefined,
         sortBy: sortBy.length ? sortBy : undefined,
         search: query.search ? query.search.toString() : undefined,
+        filter: Object.keys(filter).length ? filter : undefined,
         path,
     }
 })

src/paginate.spec.ts

@@ -1,5 +1,5 @@
-import { createConnection, Repository, Column } from 'typeorm'
+import { createConnection, Repository, Column, In } from 'typeorm'
-import { Paginated, paginate, PaginateConfig } from './paginate'
+import { Paginated, paginate, PaginateConfig, FilterOperator } from './paginate'
 import { PaginateQuery } from './decorator'
 import { Entity, PrimaryGeneratedColumn, CreateDateColumn } from 'typeorm'
 import { HttpException } from '@nestjs/common'
@@ -15,6 +15,9 @@ export class CatEntity {
     @Column()
     color: string
 
+    @Column({ nullable: true })
+    age: number | null
+
     @CreateDateColumn()
     createdAt: string
 }
@@ -33,18 +36,18 @@ describe('paginate', () => {
         })
         repo = connection.getRepository(CatEntity)
         cats = await repo.save([
-            repo.create({ name: 'Milo', color: 'brown' }),
+            repo.create({ name: 'Milo', color: 'brown', age: 6 }),
-            repo.create({ name: 'Garfield', color: 'ginger' }),
+            repo.create({ name: 'Garfield', color: 'ginger', age: null }),
-            repo.create({ name: 'Shadow', color: 'black' }),
+            repo.create({ name: 'Shadow', color: 'black', age: 4 }),
-            repo.create({ name: 'George', color: 'white' }),
+            repo.create({ name: 'George', color: 'white', age: 3 }),
-            repo.create({ name: 'Leche', color: 'white' }),
+            repo.create({ name: 'Leche', color: 'white', age: null }),
         ])
     })
 
     it('should return an instance of Paginated', async () => {
         const config: PaginateConfig<CatEntity> = {
             sortableColumns: ['id'],
-            defaultSortBy: [['createdAt', 'DESC']],
+            defaultSortBy: [['id', 'ASC']],
             defaultLimit: 1,
         }
         const query: PaginateQuery = {
@@ -162,6 +165,205 @@ describe('paginate', () => {
         expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&search=i')
     })
 
+    it('should return result based on where config and filter', async () => {
+        const config: PaginateConfig<CatEntity> = {
+            sortableColumns: ['id'],
+            where: {
+                color: 'white',
+            },
+            filterableColumns: {
+                name: [FilterOperator.NOT],
+            },
+        }
+        const query: PaginateQuery = {
+            path: '',
+            filter: {
+                name: '$not:Leche',
+            },
+        }
+
+        const result = await paginate<CatEntity>(query, repo, config)
+
+        expect(result.meta.filter).toStrictEqual({
+            name: '$not:Leche',
+        })
+        expect(result.data).toStrictEqual([cats[3]])
+        expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&filter.name=$not:Leche')
+    })
+
+    it('should return result based on multiple filter', async () => {
+        const config: PaginateConfig<CatEntity> = {
+            sortableColumns: ['id'],
+            filterableColumns: {
+                name: [FilterOperator.NOT],
+                color: [FilterOperator.EQ],
+            },
+        }
+        const query: PaginateQuery = {
+            path: '',
+            filter: {
+                name: '$not:Leche',
+                color: 'white',
+            },
+        }
+
+        const result = await paginate<CatEntity>(query, repo, config)
+
+        expect(result.meta.filter).toStrictEqual({
+            name: '$not:Leche',
+            color: 'white',
+        })
+        expect(result.data).toStrictEqual([cats[3]])
+        expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&filter.name=$not:Leche&filter.color=white')
+    })
+
+    it('should return result based on filter and search term', async () => {
+        const config: PaginateConfig<CatEntity> = {
+            sortableColumns: ['id'],
+            searchableColumns: ['name', 'color'],
+            filterableColumns: {
+                id: [FilterOperator.NOT, FilterOperator.IN],
+            },
+        }
+        const query: PaginateQuery = {
+            path: '',
+            search: 'white',
+            filter: {
+                id: '$not:$in:1,2,5',
+            },
+        }
+
+        const result = await paginate<CatEntity>(query, repo, config)
+
+        expect(result.meta.search).toStrictEqual('white')
+        expect(result.meta.filter).toStrictEqual({ id: '$not:$in:1,2,5' })
+        expect(result.data).toStrictEqual([cats[3]])
+        expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&search=white&filter.id=$not:$in:1,2,5')
+    })
+
+    it('should return result based on filter and where config', async () => {
+        const config: PaginateConfig<CatEntity> = {
+            sortableColumns: ['id'],
+            where: {
+                color: In(['black', 'white']),
+            },
+            filterableColumns: {
+                id: [FilterOperator.NOT, FilterOperator.IN],
+            },
+        }
+        const query: PaginateQuery = {
+            path: '',
+            filter: {
+                id: '$not:$in:1,2,5',
+            },
+        }
+
+        const result = await paginate<CatEntity>(query, repo, config)
+
+        expect(result.data).toStrictEqual([cats[2], cats[3]])
+        expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&filter.id=$not:$in:1,2,5')
+    })
+
+    it('should return result based on range filter', async () => {
+        const config: PaginateConfig<CatEntity> = {
+            sortableColumns: ['id'],
+            filterableColumns: {
+                age: [FilterOperator.GTE],
+            },
+        }
+        const query: PaginateQuery = {
+            path: '',
+            filter: {
+                age: '$gte:4',
+            },
+        }
+
+        const result = await paginate<CatEntity>(query, repo, config)
+
+        expect(result.data).toStrictEqual([cats[0], cats[2]])
+        expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&filter.age=$gte:4')
+    })
+
+    it('should return result based on is null query', async () => {
+        const config: PaginateConfig<CatEntity> = {
+            sortableColumns: ['id'],
+            filterableColumns: {
+                age: [FilterOperator.NULL],
+            },
+        }
+        const query: PaginateQuery = {
+            path: '',
+            filter: {
+                age: '$null',
+            },
+        }
+
+        const result = await paginate<CatEntity>(query, repo, config)
+
+        expect(result.data).toStrictEqual([cats[1], cats[4]])
+        expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&filter.age=$null')
+    })
+
+    it('should return result based on not null query', async () => {
+        const config: PaginateConfig<CatEntity> = {
+            sortableColumns: ['id'],
+            filterableColumns: {
+                age: [FilterOperator.NOT, FilterOperator.NULL],
+            },
+        }
+        const query: PaginateQuery = {
+            path: '',
+            filter: {
+                age: '$not:$null',
+            },
+        }
+
+        const result = await paginate<CatEntity>(query, repo, config)
+
+        expect(result.data).toStrictEqual([cats[0], cats[2], cats[3]])
+        expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&filter.age=$not:$null')
+    })
+
+    it('should ignore filterable column which is not configured', async () => {
+        const config: PaginateConfig<CatEntity> = {
+            sortableColumns: ['id'],
+            filterableColumns: {
+                name: [FilterOperator.NOT, FilterOperator.NULL],
+            },
+        }
+        const query: PaginateQuery = {
+            path: '',
+            filter: {
+                age: '$not:$null',
+            },
+        }
+
+        const result = await paginate<CatEntity>(query, repo, config)
+
+        expect(result.data).toStrictEqual(cats)
+        expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&filter.age=$not:$null')
+    })
+
+    it('should ignore filter operator which is not configured', async () => {
+        const config: PaginateConfig<CatEntity> = {
+            sortableColumns: ['id'],
+            filterableColumns: {
+                age: [FilterOperator.NOT],
+            },
+        }
+        const query: PaginateQuery = {
+            path: '',
+            filter: {
+                age: '$not:$null',
+            },
+        }
+
+        const result = await paginate<CatEntity>(query, repo, config)
+
+        expect(result.data).toStrictEqual(cats)
+        expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&filter.age=$not:$null')
+    })
+
     it('should throw an error when no sortableColumns', async () => {
         const config: PaginateConfig<CatEntity> = {
             sortableColumns: [],

src/paginate.ts

@@ -1,6 +1,23 @@
-import { Repository, FindConditions, SelectQueryBuilder, ObjectLiteral, ILike } from 'typeorm'
+import {
+    Repository,
+    FindConditions,
+    SelectQueryBuilder,
+    ObjectLiteral,
+    FindOperator,
+    Equal,
+    MoreThan,
+    MoreThanOrEqual,
+    In,
+    IsNull,
+    LessThan,
+    LessThanOrEqual,
+    Not,
+    ILike,
+} from 'typeorm'
 import { PaginateQuery } from './decorator'
 import { ServiceUnavailableException } from '@nestjs/common'
+import { values, mapKeys } from 'lodash'
+import { stringify } from 'querystring'
 
 type Column<T> = Extract<keyof T, string>
 type Order<T> = [Column<T>, 'ASC' | 'DESC']
@@ -15,6 +32,7 @@ export class Paginated<T> {
         totalPages: number
         sortBy: SortBy<T>
         search: string
+        filter?: { [column: string]: string | string[] }
     }
     links: {
         first?: string
@@ -32,7 +50,18 @@ export interface PaginateConfig<T> {
     defaultSortBy?: SortBy<T>
     defaultLimit?: number
     where?: FindConditions<T>
-    queryBuilder?: SelectQueryBuilder<T>
+    filterableColumns?: { [key in Column<T>]?: FilterOperator[] }
+}
+
+export enum FilterOperator {
+    EQ = '$eq',
+    GT = '$gt',
+    GTE = '$gte',
+    IN = '$in',
+    NULL = '$null',
+    LT = '$lt',
+    LTE = '$lte',
+    NOT = '$not',
 }
 
 export async function paginate<T>(
@@ -43,7 +72,6 @@ export async function paginate<T>(
     let page = query.page || 1
     const limit = Math.min(query.limit || config.defaultLimit || 20, config.maxLimit || 100)
     const sortBy = [] as SortBy<T>
-    const search = query.search
     const path = query.path
 
     function isEntityKey(sortableColumns: Column<T>[], column: string): column is Column<T> {
@@ -87,21 +115,108 @@ export async function paginate<T>(
         }
     }
 
-    const where: ObjectLiteral[] = []
+    let hasWhereClause = false
-    if (search && config.searchableColumns) {
+
+    if (config.where) {
+        queryBuilder = queryBuilder.where(config.where)
+        hasWhereClause = true
+    }
+
+    if (query.search && config.searchableColumns) {
+        const search: ObjectLiteral[] = []
         for (const column of config.searchableColumns) {
-            where.push({ [column]: ILike(`%${search}%`), ...config.where })
+            search.push({ [column]: ILike(`%${query.search}%`) })
+        }
+        queryBuilder = queryBuilder[hasWhereClause ? 'andWhere' : 'where'](search)
+        hasWhereClause = true
+    }
+
+    if (query.filter) {
+        const filter = {}
+        function getOperatorFn(op: FilterOperator): (...args: any[]) => FindOperator<T> {
+            switch (op) {
+                case FilterOperator.EQ:
+                    return Equal
+                case FilterOperator.GT:
+                    return MoreThan
+                case FilterOperator.GTE:
+                    return MoreThanOrEqual
+                case FilterOperator.IN:
+                    return In
+                case FilterOperator.NULL:
+                    return IsNull
+                case FilterOperator.LT:
+                    return LessThan
+                case FilterOperator.LTE:
+                    return LessThanOrEqual
+                case FilterOperator.NOT:
+                    return Not
+            }
+        }
+        function isOperator(value: any): value is FilterOperator {
+            return values(FilterOperator).includes(value)
+        }
+        for (const column of Object.keys(query.filter)) {
+            if (!(column in config.filterableColumns)) {
+                continue
+            }
+            const allowedOperators = config.filterableColumns[column as Column<T>]
+            const input = query.filter[column]
+            const statements = !Array.isArray(input) ? [input] : input
+            for (const raw of statements) {
+                const tokens = raw.split(':')
+                if (tokens.length === 0 || tokens.length > 3) {
+                    continue
+                } else if (tokens.length === 2) {
+                    if (tokens[1] !== FilterOperator.NULL) {
+                        tokens.unshift(null)
+                    }
+                } else if (tokens.length === 1) {
+                    if (tokens[0] === FilterOperator.NULL) {
+                        tokens.unshift(null)
+                    } else {
+                        tokens.unshift(null, FilterOperator.EQ)
+                    }
+                }
+                const [op2, op1, value] = tokens
+
+                if (!isOperator(op1) || !allowedOperators.includes(op1)) {
+                    continue
+                }
+                if (isOperator(op2) && !allowedOperators.includes(op2)) {
+                    continue
+                }
+                if (isOperator(op1)) {
+                    const args = op1 === FilterOperator.IN ? value.split(',') : value
+                    filter[column] = getOperatorFn(op1)(args)
+                }
+                if (isOperator(op2)) {
+                    filter[column] = getOperatorFn(op2)(filter[column])
+                }
             }
         }
 
-    ;[items, totalItems] = await queryBuilder.where(where.length ? where : config.where || {}).getManyAndCount()
+        queryBuilder = queryBuilder[hasWhereClause ? 'andWhere' : 'where'](filter)
+    }
+
+    ;[items, totalItems] = await queryBuilder.getManyAndCount()
 
     let totalPages = totalItems / limit
     if (totalItems % limit) totalPages = Math.ceil(totalPages)
 
-    const options = `&limit=${limit}${sortBy.map((order) => `&sortBy=${order.join(':')}`).join('')}${
+    const sortByQuery = sortBy.map((order) => `&sortBy=${order.join(':')}`).join('')
-        search ? `&search=${search}` : ''
+    const searchQuery = query.search ? `&search=${query.search}` : ''
-    }`
+    const filterQuery = query.filter
+        ? '&' +
+          stringify(
+              mapKeys(query.filter, (_param, name) => 'filter.' + name),
+              '&',
+              '=',
+              { encodeURIComponent: (str) => str }
+          )
+        : ''
+
+    const options = `&limit=${limit}${sortByQuery}${searchQuery}${filterQuery}`
 
     const buildLink = (p: number): string => path + '?page=' + p + options
 
@@ -113,7 +228,8 @@ export async function paginate<T>(
             currentPage: page,
             totalPages: totalPages,
             sortBy,
-            search,
+            search: query.search,
+            filter: query.filter,
         },
         links: {
             first: page == 1 ? undefined : buildLink(1),