fix: sorting mariadb/mysql (#945)

This commit is contained in:
xMase 2024-07-04 02:55:12 +02:00 committed by GitHub
parent a65e82ac0a
commit 1017129db4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 302 additions and 120 deletions

View File

@ -1,53 +1,65 @@
name: Main CI
on:
push:
branches: [master]
pull_request:
branches: [master]
push:
branches: [master]
pull_request:
branches: [master]
jobs:
build:
runs-on: ubuntu-latest
build:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [16.x, 18.x, 20.x]
strategy:
matrix:
node-version: [16.x, 18.x, 20.x]
services:
postgres:
image: postgres:latest
env:
POSTGRES_USER: root
POSTGRES_PASSWORD: pass
POSTGRES_DB: test
ports:
- 5432:5432
options: --health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
services:
postgres:
image: postgres:latest
env:
POSTGRES_USER: root
POSTGRES_PASSWORD: pass
POSTGRES_DB: test
ports:
- 5432:5432
options: --health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v4
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
- run: npm ci
- run: npm run format:ci
- run: npm run lint
- run: npm run build
mariadb:
image: mariadb:latest
env:
MYSQL_ROOT_PASSWORD: pass
MYSQL_DATABASE: test
ports:
- 3306:3306
options: --health-cmd "mariadb-admin ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
# TODO: run postgres and sqlite in parallel
- run: DB=postgres npm run test
steps:
- uses: actions/checkout@v4
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
- run: npm ci
- run: npm run format:ci
- run: npm run lint
- run: npm run build
- run: npm run test:cov
if: github.event_name == 'push' && matrix.node-version == '20.x'
- run: 'bash <(curl -s https://codecov.io/bash)'
- name: Semantic Release
if: github.event_name == 'push' && matrix.node-version == '20.x'
uses: cycjimmy/semantic-release-action@v3
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
# TODO: run postgres and sqlite in parallel
- run: DB=postgres npm run test
- run: DB=mariadb npm run test
- run: npm run test:cov
if: github.event_name == 'push' && matrix.node-version == '20.x'
- run: 'bash <(curl -s https://codecov.io/bash)'
- name: Semantic Release
if: github.event_name == 'push' && matrix.node-version == '20.x'
uses: cycjimmy/semantic-release-action@v3
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

24
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,24 @@
{
"version": "0.2.0",
"configurations": [
{
"type": "node",
"request": "launch",
"name": "Test Debug",
"runtimeExecutable": "npm",
"runtimeArgs": [
"--preserve-symlinks",
"run",
"test",
"--",
"--inspect-brk",
],
"console": "integratedTerminal",
"restart": true,
"sourceMaps": true,
"cwd": "${workspaceRoot}",
"autoAttachChildProcesses": true,
"envFile": "${workspaceFolder}/.env"
},
]
}

View File

@ -1,6 +1,6 @@
{
"editor.formatOnSave": true,
"editor.codeActionsOnSave": {
"source.fixAll.eslint": true
"source.fixAll.eslint": "explicit"
}
}

View File

@ -1,5 +1,3 @@
version: '3.5'
services:
postgres:
container_name: postgres_nestjs_paginate
@ -9,4 +7,13 @@ services:
POSTGRES_PASSWORD: pass
POSTGRES_DB: test
ports:
- "5432:5432"
- "${DB_PORT:-5432}:5432"
mariadb:
container_name: mariadb_nestjs_paginate
image: mariadb:latest
environment:
MYSQL_ROOT_PASSWORD: pass
MYSQL_DATABASE: test
ports:
- "${MARIA_DB_PORT:-3306}:3306"

120
package-lock.json generated
View File

@ -27,6 +27,7 @@
"eslint-plugin-prettier": "^5.1.3",
"fastify": "^4.26.2",
"jest": "^29.7.0",
"mysql": "^2.18.1",
"pg": "^8.12.0",
"prettier": "^3.0.3",
"reflect-metadata": "^0.1.14",
@ -2539,6 +2540,15 @@
"node": ">=0.6"
}
},
"node_modules/bignumber.js": {
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz",
"integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A==",
"dev": true,
"engines": {
"node": "*"
}
},
"node_modules/bindings": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
@ -6027,6 +6037,51 @@
"mkdirp": "bin/cmd.js"
}
},
"node_modules/mysql": {
"version": "2.18.1",
"resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz",
"integrity": "sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig==",
"dev": true,
"dependencies": {
"bignumber.js": "9.0.0",
"readable-stream": "2.3.7",
"safe-buffer": "5.1.2",
"sqlstring": "2.3.1"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/mysql/node_modules/readable-stream": {
"version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"dev": true,
"dependencies": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"node_modules/mysql/node_modules/safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
"dev": true
},
"node_modules/mysql/node_modules/string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"dev": true,
"dependencies": {
"safe-buffer": "~5.1.0"
}
},
"node_modules/mz": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
@ -7550,6 +7605,15 @@
}
}
},
"node_modules/sqlstring": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz",
"integrity": "sha512-ooAzh/7dxIG5+uDik1z/Rd1vli0+38izZhGzSa34FwR7IbelPWCCKSNIl8jlL/F7ERvy8CB2jNeM1E9i9mXMAQ==",
"dev": true,
"engines": {
"node": ">= 0.6"
}
},
"node_modules/ssri": {
"version": "8.0.1",
"resolved": "https://registry.npmjs.org/ssri/-/ssri-8.0.1.tgz",
@ -10466,6 +10530,12 @@
"integrity": "sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg==",
"dev": true
},
"bignumber.js": {
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz",
"integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A==",
"dev": true
},
"bindings": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
@ -13078,6 +13148,50 @@
}
}
},
"mysql": {
"version": "2.18.1",
"resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz",
"integrity": "sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig==",
"dev": true,
"requires": {
"bignumber.js": "9.0.0",
"readable-stream": "2.3.7",
"safe-buffer": "5.1.2",
"sqlstring": "2.3.1"
},
"dependencies": {
"readable-stream": {
"version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"dev": true,
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
"dev": true
},
"string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"dev": true,
"requires": {
"safe-buffer": "~5.1.0"
}
}
}
},
"mz": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
@ -14199,6 +14313,12 @@
"tar": "^6.1.11"
}
},
"sqlstring": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz",
"integrity": "sha512-ooAzh/7dxIG5+uDik1z/Rd1vli0+38izZhGzSa34FwR7IbelPWCCKSNIl8jlL/F7ERvy8CB2jNeM1E9i9mXMAQ==",
"dev": true
},
"ssri": {
"version": "8.0.1",
"resolved": "https://registry.npmjs.org/ssri/-/ssri-8.0.1.tgz",

View File

@ -55,7 +55,8 @@
"ts-jest": "^29.1.5",
"ts-node": "^10.9.2",
"typeorm": "^0.3.17",
"typescript": "^5.4.5"
"typescript": "^5.4.5",
"mysql": "^2.18.1"
},
"dependencies": {
"lodash": "^4.17.21"
@ -100,4 +101,4 @@
"master"
]
}
}
}

View File

@ -1,6 +1,6 @@
import { Column, CreateDateColumn, Entity, OneToMany, OneToOne, PrimaryGeneratedColumn, VirtualColumn } from 'typeorm'
import { CatEntity } from './cat.entity'
import { CatHomePillowEntity } from './cat-home-pillow.entity'
import { CatEntity } from './cat.entity'
@Entity()
export class CatHomeEntity {
@ -20,7 +20,10 @@ export class CatHomeEntity {
createdAt: string
@VirtualColumn({
query: (alias) => `SELECT CAST(COUNT(*) AS INT) FROM "cat" WHERE "cat"."homeId" = ${alias}.id`,
query: (alias) => {
const tck = process.env.DB === 'mariadb' ? '`' : '"'
return `SELECT CAST(COUNT(*) AS INT) FROM ${tck}cat${tck} WHERE ${tck}cat${tck}.${tck}homeId${tck} = ${alias}.id`
},
})
countCat: number
}

View File

@ -1,25 +1,26 @@
import { DataSource, In, Like, Repository, TypeORMError } from 'typeorm'
import { NO_PAGINATION, paginate, PaginateConfig, Paginated } from './paginate'
import { PaginateQuery } from './decorator'
import { HttpException } from '@nestjs/common'
import { CatEntity, CutenessLevel } from './__tests__/cat.entity'
import { CatToyEntity } from './__tests__/cat-toy.entity'
import { CatHomeEntity } from './__tests__/cat-home.entity'
import { CatHomePillowEntity } from './__tests__/cat-home-pillow.entity'
import { clone } from 'lodash'
import * as process from 'process'
import { DataSource, In, Like, Repository, TypeORMError } from 'typeorm'
import { BaseDataSourceOptions } from 'typeorm/data-source/BaseDataSourceOptions'
import { CatHairEntity } from './__tests__/cat-hair.entity'
import { CatHomePillowEntity } from './__tests__/cat-home-pillow.entity'
import { CatHomeEntity } from './__tests__/cat-home.entity'
import { CatToyEntity } from './__tests__/cat-toy.entity'
import { CatEntity, CutenessLevel } from './__tests__/cat.entity'
import { ToyShopAddressEntity } from './__tests__/toy-shop-address.entity'
import { ToyShopEntity } from './__tests__/toy-shop.entity'
import { PaginateQuery } from './decorator'
import {
FilterComparator,
FilterOperator,
FilterSuffix,
OperatorSymbolToFunction,
isOperator,
isSuffix,
OperatorSymbolToFunction,
parseFilterToken,
} from './filter'
import { ToyShopEntity } from './__tests__/toy-shop.entity'
import { ToyShopAddressEntity } from './__tests__/toy-shop-address.entity'
import * as process from 'process'
import { CatHairEntity } from './__tests__/cat-hair.entity'
import { NO_PAGINATION, PaginateConfig, Paginated, paginate } from './paginate'
const isoStringToDate = (isoString) => new Date(isoString)
@ -42,20 +43,8 @@ describe('paginate', () => {
let catHairs: CatHairEntity[] = []
beforeAll(async () => {
dataSource = new DataSource({
...(process.env.DB === 'postgres'
? {
type: 'postgres',
host: process.env.DB_HOST || 'localhost',
port: +process.env.DB_PORT || 5432,
username: process.env.DB_USERNAME || 'root',
password: process.env.DB_PASSWORD || 'pass',
database: process.env.DB_DATABASE || 'test',
}
: {
type: 'sqlite',
database: ':memory:',
}),
const dbOptions: Omit<Partial<BaseDataSourceOptions>, 'poolSize'> = {
dropSchema: true,
synchronize: true,
logging: ['error'],
entities: [
@ -67,7 +56,41 @@ describe('paginate', () => {
ToyShopEntity,
process.env.DB === 'postgres' ? CatHairEntity : undefined,
],
})
}
switch (process.env.DB) {
case 'postgres':
dataSource = new DataSource({
...dbOptions,
type: 'postgres',
host: process.env.DB_HOST || 'localhost',
port: +process.env.POSTGRESS_DB_PORT || 5432,
username: process.env.DB_USERNAME || 'root',
password: process.env.DB_PASSWORD || 'pass',
database: process.env.DB_DATABASE || 'test',
})
break
case 'mariadb':
dataSource = new DataSource({
...dbOptions,
type: 'mariadb',
host: process.env.DB_HOST || 'localhost',
port: +process.env.MARIA_DB_PORT || 3306,
username: process.env.DB_USERNAME || 'root',
password: process.env.DB_PASSWORD || 'pass',
database: process.env.DB_DATABASE || 'test',
})
break
case 'sqlite':
dataSource = new DataSource({
...dbOptions,
type: 'sqlite',
database: ':memory:',
})
break
default:
throw new Error('Invalid DB')
}
await dataSource.initialize()
catRepo = dataSource.getRepository(CatEntity)
catToyRepo = dataSource.getRepository(CatToyEntity)
@ -2936,33 +2959,6 @@ describe('paginate', () => {
expect(result.links.current).toBe('?page=1&limit=20&sortBy=id:ASC&filter.home.countCat=$gt:0')
})
it('should return result sorted by a virtual column', async () => {
const config: PaginateConfig<CatEntity> = {
sortableColumns: ['home.countCat'],
relations: ['home'],
}
const query: PaginateQuery = {
path: '',
sortBy: [['home.countCat', 'ASC']],
}
const result = await paginate<CatEntity>(query, catRepo, config)
const expectedResult = [2, 3, 4, 0, 1].map((i) => {
const ret = clone(cats[i])
if (i == 0 || i == 1) {
ret.home = clone(catHomes[i])
ret.home.countCat = cats.filter((cat) => cat.id === ret.home.cat.id).length
delete ret.home.cat
} else {
ret.home = null
}
return ret
})
expect(result.data).toStrictEqual(expectedResult)
expect(result.links.current).toBe('?page=1&limit=20&sortBy=home.countCat:ASC')
})
it('should return result sorted and filter by a virtual column in main entity', async () => {
const config: PaginateConfig<CatHomeEntity> = {
sortableColumns: ['countCat'],

View File

@ -1,3 +1,6 @@
import { Logger, ServiceUnavailableException } from '@nestjs/common'
import { mapKeys } from 'lodash'
import { stringify } from 'querystring'
import {
Brackets,
FindOperator,
@ -9,28 +12,25 @@ import {
Repository,
SelectQueryBuilder,
} from 'typeorm'
import { PaginateQuery } from './decorator'
import { Logger, ServiceUnavailableException } from '@nestjs/common'
import { mapKeys } from 'lodash'
import { stringify } from 'querystring'
import { WherePredicateOperator } from 'typeorm/query-builder/WhereClause'
import { OrmUtils } from 'typeorm/util/OrmUtils'
import { PaginateQuery } from './decorator'
import { FilterOperator, FilterSuffix, addFilter } from './filter'
import {
Column,
Order,
RelationColumn,
SortBy,
checkIsEmbedded,
checkIsRelation,
Column,
extractVirtualProperty,
fixColumnAlias,
getPropertiesByColumnName,
getQueryUrlComponents,
includesAllPrimaryKeyColumns,
isEntityKey,
Order,
positiveNumberOrDefault,
RelationColumn,
SortBy,
} from './helper'
import { addFilter, FilterOperator, FilterSuffix } from './filter'
import { OrmUtils } from 'typeorm/util/OrmUtils'
const logger: Logger = new Logger('nestjs-paginate')
@ -240,9 +240,17 @@ export async function paginate<T extends ObjectLiteral>(
createQueryBuilderRelations(queryBuilder.alias, relations)
}
let nullSort: 'NULLS LAST' | 'NULLS FIRST' | undefined = undefined
const dbType = (repo instanceof Repository ? repo.manager : repo).connection.options.type
const isMariaDbOrMySql = (dbType: string) => dbType === 'mariadb' || dbType === 'mysql'
const isMMDb = isMariaDbOrMySql(dbType)
let nullSort: string | undefined
if (config.nullSort) {
nullSort = config.nullSort === 'last' ? 'NULLS LAST' : 'NULLS FIRST'
if (isMMDb) {
nullSort = config.nullSort === 'last' ? 'IS NULL' : 'IS NOT NULL'
} else {
nullSort = config.nullSort === 'last' ? 'NULLS LAST' : 'NULLS FIRST'
}
}
if (config.sortableColumns.length < 1) {
@ -269,10 +277,21 @@ export async function paginate<T extends ObjectLiteral>(
const isRelation = checkIsRelation(queryBuilder, columnProperties.propertyPath)
const isEmbeded = checkIsEmbedded(queryBuilder, columnProperties.propertyPath)
let alias = fixColumnAlias(columnProperties, queryBuilder.alias, isRelation, isVirtualProperty, isEmbeded)
if (isVirtualProperty) {
alias = `"${alias}"`
if (isMMDb) {
if (isVirtualProperty) {
alias = `\`${alias}\``
}
if (nullSort) {
queryBuilder.addOrderBy(`${alias} ${nullSort}`)
}
queryBuilder.addOrderBy(alias, order[1])
} else {
if (isVirtualProperty) {
alias = `"${alias}"`
}
queryBuilder.addOrderBy(alias, order[1], nullSort as 'NULLS FIRST' | 'NULLS LAST' | undefined)
}
queryBuilder.addOrderBy(alias, order[1], nullSort)
}
// When we partial select the columns (main or relation) we must add the primary key column otherwise