fix: sorting mariadb/mysql (#945)

.github/workflows/main.yml

@@ -28,6 +28,18 @@ jobs:
                     --health-timeout 5s
                     --health-retries 5
 
+            mariadb:
+                image: mariadb:latest
+                env:
+                    MYSQL_ROOT_PASSWORD: pass
+                    MYSQL_DATABASE: test
+                ports:
+                    - 3306:3306
+                options: --health-cmd "mariadb-admin ping"
+                    --health-interval 10s
+                    --health-timeout 5s
+                    --health-retries 5
+
         steps:
             - uses: actions/checkout@v4
             - name: Use Node.js ${{ matrix.node-version }}
@@ -41,7 +53,7 @@ jobs:
 
             # TODO: run postgres and sqlite in parallel
             - run: DB=postgres npm run test
-
+            - run: DB=mariadb npm run test
             - run: npm run test:cov
               if: github.event_name == 'push' && matrix.node-version == '20.x'
             - run: 'bash <(curl -s https://codecov.io/bash)'

.vscode/launch.json

@@ -0,0 +1,24 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "Test Debug",
+            "runtimeExecutable": "npm",
+            "runtimeArgs": [
+                "--preserve-symlinks",
+                "run",
+                "test",
+                "--",
+                "--inspect-brk",
+            ],
+            "console": "integratedTerminal",
+            "restart": true,
+            "sourceMaps": true,
+            "cwd": "${workspaceRoot}",
+            "autoAttachChildProcesses": true,
+            "envFile": "${workspaceFolder}/.env"
+        },
+    ]
+}

.vscode/settings.json

@@ -1,6 +1,6 @@
 {
   "editor.formatOnSave": true,
   "editor.codeActionsOnSave": {
-    "source.fixAll.eslint": true
+    "source.fixAll.eslint": "explicit"
   }
 }

docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.5'
-
 services:
   postgres:
     container_name: postgres_nestjs_paginate
@@ -9,4 +7,13 @@ services:
       POSTGRES_PASSWORD: pass
       POSTGRES_DB: test
     ports:
-      - "5432:5432"
+      - "${DB_PORT:-5432}:5432"
+
+  mariadb:
+    container_name: mariadb_nestjs_paginate
+    image: mariadb:latest
+    environment:
+      MYSQL_ROOT_PASSWORD: pass
+      MYSQL_DATABASE: test
+    ports:
+      - "${MARIA_DB_PORT:-3306}:3306"

package-lock.json

@@ -27,6 +27,7 @@
         "eslint-plugin-prettier": "^5.1.3",
         "fastify": "^4.26.2",
         "jest": "^29.7.0",
+        "mysql": "^2.18.1",
         "pg": "^8.12.0",
         "prettier": "^3.0.3",
         "reflect-metadata": "^0.1.14",
@@ -2539,6 +2540,15 @@
         "node": ">=0.6"
       }
     },
+    "node_modules/bignumber.js": {
+      "version": "9.0.0",
+      "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz",
+      "integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A==",
+      "dev": true,
+      "engines": {
+        "node": "*"
+      }
+    },
     "node_modules/bindings": {
       "version": "1.5.0",
       "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
@@ -6027,6 +6037,51 @@
         "mkdirp": "bin/cmd.js"
       }
     },
+    "node_modules/mysql": {
+      "version": "2.18.1",
+      "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz",
+      "integrity": "sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig==",
+      "dev": true,
+      "dependencies": {
+        "bignumber.js": "9.0.0",
+        "readable-stream": "2.3.7",
+        "safe-buffer": "5.1.2",
+        "sqlstring": "2.3.1"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/mysql/node_modules/readable-stream": {
+      "version": "2.3.7",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
+      "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
+      "dev": true,
+      "dependencies": {
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
+      }
+    },
+    "node_modules/mysql/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+      "dev": true
+    },
+    "node_modules/mysql/node_modules/string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+      "dev": true,
+      "dependencies": {
+        "safe-buffer": "~5.1.0"
+      }
+    },
     "node_modules/mz": {
       "version": "2.7.0",
       "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
@@ -7550,6 +7605,15 @@
         }
       }
     },
+    "node_modules/sqlstring": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz",
+      "integrity": "sha512-ooAzh/7dxIG5+uDik1z/Rd1vli0+38izZhGzSa34FwR7IbelPWCCKSNIl8jlL/F7ERvy8CB2jNeM1E9i9mXMAQ==",
+      "dev": true,
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
     "node_modules/ssri": {
       "version": "8.0.1",
       "resolved": "https://registry.npmjs.org/ssri/-/ssri-8.0.1.tgz",
@@ -10466,6 +10530,12 @@
       "integrity": "sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg==",
       "dev": true
     },
+    "bignumber.js": {
+      "version": "9.0.0",
+      "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz",
+      "integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A==",
+      "dev": true
+    },
     "bindings": {
       "version": "1.5.0",
       "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
@@ -13078,6 +13148,50 @@
         }
       }
     },
+    "mysql": {
+      "version": "2.18.1",
+      "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz",
+      "integrity": "sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig==",
+      "dev": true,
+      "requires": {
+        "bignumber.js": "9.0.0",
+        "readable-stream": "2.3.7",
+        "safe-buffer": "5.1.2",
+        "sqlstring": "2.3.1"
+      },
+      "dependencies": {
+        "readable-stream": {
+          "version": "2.3.7",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
+          "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
+          "dev": true,
+          "requires": {
+            "core-util-is": "~1.0.0",
+            "inherits": "~2.0.3",
+            "isarray": "~1.0.0",
+            "process-nextick-args": "~2.0.0",
+            "safe-buffer": "~5.1.1",
+            "string_decoder": "~1.1.1",
+            "util-deprecate": "~1.0.1"
+          }
+        },
+        "safe-buffer": {
+          "version": "5.1.2",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+          "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+          "dev": true
+        },
+        "string_decoder": {
+          "version": "1.1.1",
+          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+          "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+          "dev": true,
+          "requires": {
+            "safe-buffer": "~5.1.0"
+          }
+        }
+      }
+    },
     "mz": {
       "version": "2.7.0",
       "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
@@ -14199,6 +14313,12 @@
         "tar": "^6.1.11"
       }
     },
+    "sqlstring": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz",
+      "integrity": "sha512-ooAzh/7dxIG5+uDik1z/Rd1vli0+38izZhGzSa34FwR7IbelPWCCKSNIl8jlL/F7ERvy8CB2jNeM1E9i9mXMAQ==",
+      "dev": true
+    },
     "ssri": {
       "version": "8.0.1",
       "resolved": "https://registry.npmjs.org/ssri/-/ssri-8.0.1.tgz",

package.json

@@ -55,7 +55,8 @@
     "ts-jest": "^29.1.5",
     "ts-node": "^10.9.2",
     "typeorm": "^0.3.17",
-    "typescript": "^5.4.5"
+    "typescript": "^5.4.5",
+    "mysql": "^2.18.1"
   },
   "dependencies": {
     "lodash": "^4.17.21"

src/__tests__/cat-home.entity.ts

@@ -1,6 +1,6 @@
 import { Column, CreateDateColumn, Entity, OneToMany, OneToOne, PrimaryGeneratedColumn, VirtualColumn } from 'typeorm'
-import { CatEntity } from './cat.entity'
 import { CatHomePillowEntity } from './cat-home-pillow.entity'
+import { CatEntity } from './cat.entity'
 
 @Entity()
 export class CatHomeEntity {
@@ -20,7 +20,10 @@ export class CatHomeEntity {
     createdAt: string
 
     @VirtualColumn({
-        query: (alias) => `SELECT CAST(COUNT(*) AS INT)  FROM "cat" WHERE "cat"."homeId" = ${alias}.id`,
+        query: (alias) => {
+            const tck = process.env.DB === 'mariadb' ? '`' : '"'
+            return `SELECT CAST(COUNT(*) AS INT) FROM ${tck}cat${tck} WHERE ${tck}cat${tck}.${tck}homeId${tck} = ${alias}.id`
+        },
     })
     countCat: number
 }

src/paginate.spec.ts

@@ -1,25 +1,26 @@
-import { DataSource, In, Like, Repository, TypeORMError } from 'typeorm'
-import { NO_PAGINATION, paginate, PaginateConfig, Paginated } from './paginate'
-import { PaginateQuery } from './decorator'
 import { HttpException } from '@nestjs/common'
-import { CatEntity, CutenessLevel } from './__tests__/cat.entity'
-import { CatToyEntity } from './__tests__/cat-toy.entity'
-import { CatHomeEntity } from './__tests__/cat-home.entity'
-import { CatHomePillowEntity } from './__tests__/cat-home-pillow.entity'
 import { clone } from 'lodash'
+import * as process from 'process'
+import { DataSource, In, Like, Repository, TypeORMError } from 'typeorm'
+import { BaseDataSourceOptions } from 'typeorm/data-source/BaseDataSourceOptions'
+import { CatHairEntity } from './__tests__/cat-hair.entity'
+import { CatHomePillowEntity } from './__tests__/cat-home-pillow.entity'
+import { CatHomeEntity } from './__tests__/cat-home.entity'
+import { CatToyEntity } from './__tests__/cat-toy.entity'
+import { CatEntity, CutenessLevel } from './__tests__/cat.entity'
+import { ToyShopAddressEntity } from './__tests__/toy-shop-address.entity'
+import { ToyShopEntity } from './__tests__/toy-shop.entity'
+import { PaginateQuery } from './decorator'
 import {
     FilterComparator,
     FilterOperator,
     FilterSuffix,
+    OperatorSymbolToFunction,
     isOperator,
     isSuffix,
-    OperatorSymbolToFunction,
     parseFilterToken,
 } from './filter'
-import { ToyShopEntity } from './__tests__/toy-shop.entity'
+import { NO_PAGINATION, PaginateConfig, Paginated, paginate } from './paginate'
-import { ToyShopAddressEntity } from './__tests__/toy-shop-address.entity'
-import * as process from 'process'
-import { CatHairEntity } from './__tests__/cat-hair.entity'
 
 const isoStringToDate = (isoString) => new Date(isoString)
 
@@ -42,20 +43,8 @@ describe('paginate', () => {
     let catHairs: CatHairEntity[] = []
 
     beforeAll(async () => {
-        dataSource = new DataSource({
+        const dbOptions: Omit<Partial<BaseDataSourceOptions>, 'poolSize'> = {
-            ...(process.env.DB === 'postgres'
+            dropSchema: true,
-                ? {
-                      type: 'postgres',
-                      host: process.env.DB_HOST || 'localhost',
-                      port: +process.env.DB_PORT || 5432,
-                      username: process.env.DB_USERNAME || 'root',
-                      password: process.env.DB_PASSWORD || 'pass',
-                      database: process.env.DB_DATABASE || 'test',
-                  }
-                : {
-                      type: 'sqlite',
-                      database: ':memory:',
-                  }),
             synchronize: true,
             logging: ['error'],
             entities: [
@@ -67,7 +56,41 @@ describe('paginate', () => {
                 ToyShopEntity,
                 process.env.DB === 'postgres' ? CatHairEntity : undefined,
             ],
+        }
+
+        switch (process.env.DB) {
+            case 'postgres':
+                dataSource = new DataSource({
+                    ...dbOptions,
+                    type: 'postgres',
+                    host: process.env.DB_HOST || 'localhost',
+                    port: +process.env.POSTGRESS_DB_PORT || 5432,
+                    username: process.env.DB_USERNAME || 'root',
+                    password: process.env.DB_PASSWORD || 'pass',
+                    database: process.env.DB_DATABASE || 'test',
                 })
+                break
+            case 'mariadb':
+                dataSource = new DataSource({
+                    ...dbOptions,
+                    type: 'mariadb',
+                    host: process.env.DB_HOST || 'localhost',
+                    port: +process.env.MARIA_DB_PORT || 3306,
+                    username: process.env.DB_USERNAME || 'root',
+                    password: process.env.DB_PASSWORD || 'pass',
+                    database: process.env.DB_DATABASE || 'test',
+                })
+                break
+            case 'sqlite':
+                dataSource = new DataSource({
+                    ...dbOptions,
+                    type: 'sqlite',
+                    database: ':memory:',
+                })
+                break
+            default:
+                throw new Error('Invalid DB')
+        }
         await dataSource.initialize()
         catRepo = dataSource.getRepository(CatEntity)
         catToyRepo = dataSource.getRepository(CatToyEntity)
@@ -2936,33 +2959,6 @@ describe('paginate', () => {
                 expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&filter.home.countCat=$gt:0')
             })
 
-            it('should return result sorted by a virtual column', async () => {
-                const config: PaginateConfig<CatEntity> = {
-                    sortableColumns: ['home.countCat'],
-                    relations: ['home'],
-                }
-                const query: PaginateQuery = {
-                    path: '',
-                    sortBy: [['home.countCat', 'ASC']],
-                }
-
-                const result = await paginate<CatEntity>(query, catRepo, config)
-                const expectedResult = [2, 3, 4, 0, 1].map((i) => {
-                    const ret = clone(cats[i])
-                    if (i == 0 || i == 1) {
-                        ret.home = clone(catHomes[i])
-                        ret.home.countCat = cats.filter((cat) => cat.id === ret.home.cat.id).length
-                        delete ret.home.cat
-                    } else {
-                        ret.home = null
-                    }
-                    return ret
-                })
-
-                expect(result.data).toStrictEqual(expectedResult)
-                expect(result.links.current).toBe('?page=1&limit=20&sortBy=home.countCat:ASC')
-            })
-
             it('should return result sorted and filter by a virtual column in main entity', async () => {
                 const config: PaginateConfig<CatHomeEntity> = {
                     sortableColumns: ['countCat'],

src/paginate.ts

@@ -1,3 +1,6 @@
+import { Logger, ServiceUnavailableException } from '@nestjs/common'
+import { mapKeys } from 'lodash'
+import { stringify } from 'querystring'
 import {
     Brackets,
     FindOperator,
@@ -9,28 +12,25 @@ import {
     Repository,
     SelectQueryBuilder,
 } from 'typeorm'
-import { PaginateQuery } from './decorator'
-import { Logger, ServiceUnavailableException } from '@nestjs/common'
-import { mapKeys } from 'lodash'
-import { stringify } from 'querystring'
 import { WherePredicateOperator } from 'typeorm/query-builder/WhereClause'
+import { OrmUtils } from 'typeorm/util/OrmUtils'
+import { PaginateQuery } from './decorator'
+import { FilterOperator, FilterSuffix, addFilter } from './filter'
 import {
+    Column,
+    Order,
+    RelationColumn,
+    SortBy,
     checkIsEmbedded,
     checkIsRelation,
-    Column,
     extractVirtualProperty,
     fixColumnAlias,
     getPropertiesByColumnName,
     getQueryUrlComponents,
     includesAllPrimaryKeyColumns,
     isEntityKey,
-    Order,
     positiveNumberOrDefault,
-    RelationColumn,
-    SortBy,
 } from './helper'
-import { addFilter, FilterOperator, FilterSuffix } from './filter'
-import { OrmUtils } from 'typeorm/util/OrmUtils'
 
 const logger: Logger = new Logger('nestjs-paginate')
 
@@ -240,10 +240,18 @@ export async function paginate<T extends ObjectLiteral>(
         createQueryBuilderRelations(queryBuilder.alias, relations)
     }
 
-    let nullSort: 'NULLS LAST' | 'NULLS FIRST' | undefined = undefined
+    const dbType = (repo instanceof Repository ? repo.manager : repo).connection.options.type
+    const isMariaDbOrMySql = (dbType: string) => dbType === 'mariadb' || dbType === 'mysql'
+    const isMMDb = isMariaDbOrMySql(dbType)
+
+    let nullSort: string | undefined
     if (config.nullSort) {
+        if (isMMDb) {
+            nullSort = config.nullSort === 'last' ? 'IS NULL' : 'IS NOT NULL'
+        } else {
             nullSort = config.nullSort === 'last' ? 'NULLS LAST' : 'NULLS FIRST'
         }
+    }
 
     if (config.sortableColumns.length < 1) {
         const message = "Missing required 'sortableColumns' config."
@@ -269,10 +277,21 @@ export async function paginate<T extends ObjectLiteral>(
         const isRelation = checkIsRelation(queryBuilder, columnProperties.propertyPath)
         const isEmbeded = checkIsEmbedded(queryBuilder, columnProperties.propertyPath)
         let alias = fixColumnAlias(columnProperties, queryBuilder.alias, isRelation, isVirtualProperty, isEmbeded)
+
+        if (isMMDb) {
+            if (isVirtualProperty) {
+                alias = `\`${alias}\``
+            }
+            if (nullSort) {
+                queryBuilder.addOrderBy(`${alias} ${nullSort}`)
+            }
+            queryBuilder.addOrderBy(alias, order[1])
+        } else {
             if (isVirtualProperty) {
                 alias = `"${alias}"`
             }
-        queryBuilder.addOrderBy(alias, order[1], nullSort)
+            queryBuilder.addOrderBy(alias, order[1], nullSort as 'NULLS FIRST' | 'NULLS LAST' | undefined)
+        }
     }
 
     // When we partial select the columns (main or relation) we must add the primary key column otherwise