perf: avoid unnecessary count on getManyAndCount (#11524)

* perf: avoid unnecessary count on getManyAndCount

Skip count query when it can be deduced from the
number of returned rows. This will avoid one round
trip and could be very helpful on pagination when the
limit is not reached.

src/query-builder/SelectQueryBuilder.ts

@@ -1875,11 +1875,17 @@ export class SelectQueryBuilder<Entity extends ObjectLiteral>
                 queryRunner,
             )
             this.expressionMap.queryEntity = false
-            const cacheId = this.expressionMap.cacheId
-            // Creates a new cacheId for the count query, or it will retreive the above query results
-            // and count will return 0.
-            this.expressionMap.cacheId = cacheId ? `${cacheId}-count` : cacheId
-            const count = await this.executeCountQuery(queryRunner)
+
+            let count: number | undefined = this.lazyCount(entitiesAndRaw)
+            if (count === undefined) {
+                const cacheId = this.expressionMap.cacheId
+                // Creates a new cacheId for the count query, or it will retrieve the above query results
+                // and count will return 0.
+                if (cacheId) {
+                    this.expressionMap.cacheId = `${cacheId}-count`
+                }
+                count = await this.executeCountQuery(queryRunner)
+            }
             const results: [Entity[], number] = [entitiesAndRaw.entities, count]
 
             // close transaction if we started it
@@ -1903,6 +1909,55 @@ export class SelectQueryBuilder<Entity extends ObjectLiteral>
         }
     }
 
+    private lazyCount(entitiesAndRaw: {
+        entities: Entity[]
+        raw: any[]
+    }): number | undefined {
+        const hasLimit =
+            this.expressionMap.limit !== undefined &&
+            this.expressionMap.limit !== null
+        if (this.expressionMap.joinAttributes.length > 0 && hasLimit) {
+            return undefined
+        }
+
+        const hasTake =
+            this.expressionMap.take !== undefined &&
+            this.expressionMap.take !== null
+
+        // limit overrides take when no join is defined
+        const maxResults = hasLimit
+            ? this.expressionMap.limit
+            : hasTake
+            ? this.expressionMap.take
+            : undefined
+
+        if (
+            maxResults !== undefined &&
+            entitiesAndRaw.entities.length === maxResults
+        ) {
+            // stop here when the result set contains the max number of rows; we need to execute a full count
+            return undefined
+        }
+
+        const hasSkip =
+            this.expressionMap.skip !== undefined &&
+            this.expressionMap.skip !== null &&
+            this.expressionMap.skip > 0
+        const hasOffset =
+            this.expressionMap.offset !== undefined &&
+            this.expressionMap.offset !== null &&
+            this.expressionMap.offset > 0
+
+        // offset overrides skip when no join is defined
+        const previousResults: number = hasOffset
+            ? this.expressionMap.offset!
+            : hasSkip
+            ? this.expressionMap.skip!
+            : 0
+
+        return entitiesAndRaw.entities.length + previousResults
+    }
+
     /**
      * Executes built SQL query and returns raw data stream.
      */

test/other-issues/lazy-count/entity/Comment.ts

@@ -0,0 +1,23 @@
+import {
+    Column,
+    Entity,
+    ManyToOne,
+    PrimaryGeneratedColumn,
+} from "../../../../src"
+import { Post } from "./Post"
+
+@Entity()
+export class Comment {
+    @PrimaryGeneratedColumn("increment")
+    id: number
+
+    @Column()
+    title: string
+
+    @ManyToOne(() => Post, (post) => post.comments)
+    post: Post
+
+    constructor(title?: string) {
+        if (title) this.title = title
+    }
+}

test/other-issues/lazy-count/entity/Post.ts

@@ -0,0 +1,21 @@
+import {
+    Column,
+    Entity,
+    OneToMany,
+    PrimaryGeneratedColumn,
+} from "../../../../src"
+import { Comment } from "./Comment"
+
+@Entity()
+export class Post {
+    @PrimaryGeneratedColumn("increment")
+    id!: number
+
+    @Column()
+    content!: string
+
+    @OneToMany(() => Comment, (comment) => comment.post, {
+        cascade: ["insert"],
+    })
+    comments!: Comment[]
+}

test/other-issues/lazy-count/lazy-count.ts

@@ -0,0 +1,281 @@
+import "reflect-metadata"
+import {
+    closeTestingConnections,
+    createTestingConnections,
+    reloadTestingDatabases,
+} from "../../utils/test-utils"
+import { DataSource } from "../../../src/data-source/DataSource"
+import { Post } from "./entity/Post"
+import { expect } from "chai"
+import { AfterQuerySubscriber } from "./subscribers/AfterQuerySubscriber"
+import { Comment } from "./entity/Comment"
+
+describe("other issues > lazy count", () => {
+    let connections: DataSource[]
+    before(
+        async () =>
+            (connections = await createTestingConnections({
+                entities: [__dirname + "/entity/*{.js,.ts}"],
+                subscribers: [AfterQuerySubscriber],
+                dropSchema: true,
+                schemaCreate: true,
+            })),
+    )
+    beforeEach(() => reloadTestingDatabases(connections))
+    after(() => closeTestingConnections(connections))
+
+    it("skip count query when fewer entities are returned than the limit", () =>
+        Promise.all(
+            connections.map(async function (connection) {
+                await savePostEntities(connection, 5)
+
+                const afterQuery = connection
+                    .subscribers[0] as AfterQuerySubscriber
+                afterQuery.clear()
+
+                const [entities, count] = await connection.manager
+                    .createQueryBuilder(Post, "post")
+                    .limit(10)
+                    .orderBy("post.id")
+                    .getManyAndCount()
+
+                expect(count).to.be.equal(5)
+                expect(entities.length).to.be.equal(5)
+
+                expect(
+                    afterQuery
+                        .getCalledQueries()
+                        .filter((query) => query.match(/(count|cnt)/i)),
+                ).to.be.empty
+            }),
+        ))
+
+    it("skip count query when fewer entities are returned than the take", () =>
+        Promise.all(
+            connections.map(async function (connection) {
+                await savePostEntities(connection, 5)
+
+                const afterQuery = connection
+                    .subscribers[0] as AfterQuerySubscriber
+                afterQuery.clear()
+
+                const [entities, count] = await connection.manager
+                    .createQueryBuilder(Post, "post")
+                    .take(10)
+                    .orderBy("post.id")
+                    .getManyAndCount()
+
+                expect(count).to.be.equal(5)
+                expect(entities.length).to.be.equal(5)
+
+                expect(
+                    afterQuery
+                        .getCalledQueries()
+                        .filter((query) => query.match(/(count|cnt)/i)),
+                ).to.be.empty
+            }),
+        ))
+
+    it("skip count query when an offset is defined", () =>
+        Promise.all(
+            connections.map(async function (connection) {
+                await savePostEntities(connection, 5)
+
+                const afterQuery = connection
+                    .subscribers[0] as AfterQuerySubscriber
+                afterQuery.clear()
+
+                const [entities, count] = await connection.manager
+                    .createQueryBuilder(Post, "post")
+                    .limit(10)
+                    .offset(3)
+                    .orderBy("post.id")
+                    .getManyAndCount()
+
+                expect(count).to.be.equal(5)
+                expect(entities.length).to.be.equal(2)
+
+                expect(
+                    afterQuery
+                        .getCalledQueries()
+                        .filter((query) => query.match(/(count|cnt)/i)),
+                ).to.be.empty
+            }),
+        ))
+
+    it("skip count query when skip is defined", () =>
+        Promise.all(
+            connections.map(async function (connection) {
+                await savePostEntities(connection, 5)
+
+                const afterQuery = connection
+                    .subscribers[0] as AfterQuerySubscriber
+                afterQuery.clear()
+
+                const [entities, count] = await connection.manager
+                    .createQueryBuilder(Post, "post")
+                    .take(10)
+                    .skip(3)
+                    .orderBy("post.id")
+                    .getManyAndCount()
+
+                expect(count).to.be.equal(5)
+                expect(entities.length).to.be.equal(2)
+
+                expect(
+                    afterQuery
+                        .getCalledQueries()
+                        .filter((query) => query.match(/(count|cnt)/i)),
+                ).to.be.empty
+            }),
+        ))
+
+    it("run count query when returned entities reach the take", () =>
+        Promise.all(
+            connections.map(async function (connection) {
+                await savePostEntities(connection, 2)
+
+                const afterQuery = connection
+                    .subscribers[0] as AfterQuerySubscriber
+                afterQuery.clear()
+
+                const [entities, count] = await connection.manager
+                    .createQueryBuilder(Post, "post")
+                    .take(2)
+                    .orderBy("post.id")
+                    .getManyAndCount()
+
+                expect(count).to.be.equal(2)
+                expect(entities.length).to.be.equal(2)
+
+                expect(
+                    afterQuery
+                        .getCalledQueries()
+                        .filter((query) => query.match(/(count|cnt)/i)),
+                ).not.to.be.empty
+            }),
+        ))
+
+    it("skip count query when joining a relation with a take", () =>
+        Promise.all(
+            connections.map(async function (connection) {
+                await savePostEntities(connection, 5)
+
+                const afterQuery = connection
+                    .subscribers[0] as AfterQuerySubscriber
+                afterQuery.clear()
+
+                const [entities, count] = await connection.manager
+                    .createQueryBuilder(Post, "post")
+                    .innerJoin("post.comments", "comments")
+                    .take(20)
+                    .orderBy("post.id")
+                    .getManyAndCount()
+
+                expect(count).to.be.equal(5)
+                expect(entities.length).to.be.equal(5)
+
+                expect(
+                    afterQuery
+                        .getCalledQueries()
+                        .filter((query) => query.match(/(count|cnt)/i)),
+                ).to.be.empty
+            }),
+        ))
+
+    it("run count query when joining a relation with a limit", () =>
+        Promise.all(
+            connections.map(async function (connection) {
+                await savePostEntities(connection, 5)
+
+                const afterQuery = connection
+                    .subscribers[0] as AfterQuerySubscriber
+                afterQuery.clear()
+
+                const [entities, count] = await connection.manager
+                    .createQueryBuilder(Post, "post")
+                    .innerJoin("post.comments", "comments")
+                    .limit(20)
+                    .orderBy("post.id")
+                    .getManyAndCount()
+
+                expect(count).to.be.equal(5)
+                expect(entities.length).to.be.equal(5)
+
+                expect(
+                    afterQuery
+                        .getCalledQueries()
+                        .filter((query) => query.match(/(count|cnt)/i)),
+                ).not.to.be.empty
+            }),
+        ))
+
+    it("skip count query when joining a relation with a take and a skip", () =>
+        Promise.all(
+            connections.map(async function (connection) {
+                await savePostEntities(connection, 5)
+
+                const afterQuery = connection
+                    .subscribers[0] as AfterQuerySubscriber
+                afterQuery.clear()
+
+                const [entities, count] = await connection.manager
+                    .createQueryBuilder(Post, "post")
+                    .innerJoin("post.comments", "comments")
+                    .take(3)
+                    .skip(3)
+                    .orderBy("post.id")
+                    .getManyAndCount()
+
+                expect(count).to.be.equal(5)
+                expect(entities.length).to.be.equal(2)
+
+                expect(
+                    afterQuery
+                        .getCalledQueries()
+                        .filter((query) => query.match(/(count|cnt)/i)),
+                ).to.be.empty
+            }),
+        ))
+
+    it("run count query when joining a relation with a limit and an offset", () =>
+        Promise.all(
+            connections.map(async function (connection) {
+                await savePostEntities(connection, 5)
+
+                const afterQuery = connection
+                    .subscribers[0] as AfterQuerySubscriber
+                afterQuery.clear()
+
+                const [entities, count] = await connection.manager
+                    .createQueryBuilder(Post, "post")
+                    .innerJoin("post.comments", "comments")
+                    .limit(3)
+                    .offset(3)
+                    .orderBy("post.id")
+                    .getManyAndCount()
+
+                expect(count).to.be.equal(5)
+                expect(entities.length).to.be.equal(2)
+
+                expect(
+                    afterQuery
+                        .getCalledQueries()
+                        .filter((query) => query.match(/(count|cnt)/i)),
+                ).not.to.be.empty
+            }),
+        ))
+
+    async function savePostEntities(connection: DataSource, count: number) {
+        for (let i = 1; i <= count; i++) {
+            const post = new Post()
+            post.content = "Hello Post #" + i
+            post.comments = [
+                new Comment(`comment 1 for post ${i}`),
+                new Comment(`comment 2 for post ${i}`),
+            ]
+
+            await connection.manager.save(post)
+        }
+    }
+})

test/other-issues/lazy-count/subscribers/AfterQuerySubscriber.ts

@@ -0,0 +1,22 @@
+import {
+    AfterQueryEvent,
+    EntitySubscriberInterface,
+    EventSubscriber,
+} from "../../../../src"
+
+@EventSubscriber()
+export class AfterQuerySubscriber implements EntitySubscriberInterface {
+    private calledQueries: any[] = []
+
+    afterQuery(event: AfterQueryEvent<any>): void {
+        this.calledQueries.push(event.query)
+    }
+
+    getCalledQueries(): any[] {
+        return this.calledQueries
+    }
+
+    clear(): void {
+        this.calledQueries = []
+    }
+}