From 65e75b4d113fb1820db85bee8eb90f9ebefb70a0 Mon Sep 17 00:00:00 2001 From: Umed Khudoiberdiev Date: Fri, 17 Nov 2017 10:45:01 +0500 Subject: [PATCH] fixed performance issues --- package.json | 2 +- src/driver/mysql/MysqlDriver.ts | 23 ++++---- src/driver/oracle/OracleDriver.ts | 23 ++++---- src/driver/postgres/PostgresDriver.ts | 35 +++++++------ .../sqlite-abstract/AbstractSqliteDriver.ts | 35 +++++++------ src/driver/sqlserver/SqlServerDriver.ts | 33 ++++++------ src/driver/websql/WebsqlDriver.ts | 23 ++++---- src/persistence/SubjectExecutor.ts | 1 + src/query-builder/InsertQueryBuilder.ts | 8 +++ .../bulk-save-case1.ts} | 10 ++-- .../entity/Post.ts | 0 .../bulk-save-case2/bulk-save-case2.ts | 52 +++++++++++++++++++ .../bulk-save-case2/entity/Document.ts | 31 +++++++++++ 13 files changed, 186 insertions(+), 90 deletions(-) rename test/benchmark/{bulk-save/bulk-save.ts => bulk-save-case1/bulk-save-case1.ts} (89%) rename test/benchmark/{bulk-save => bulk-save-case1}/entity/Post.ts (100%) create mode 100644 test/benchmark/bulk-save-case2/bulk-save-case2.ts create mode 100644 test/benchmark/bulk-save-case2/entity/Document.ts diff --git a/package.json b/package.json index b9f057688..7b5e5cdbd 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "typeorm", "private": true, - "version": "0.2.0-alpha.1", + "version": "0.2.0-alpha.2", "description": "Data-Mapper ORM for TypeScript, ES7, ES6, ES5. Supports MySQL, PostgreSQL, MariaDB, SQLite, MS SQL Server, Oracle, WebSQL, MongoDB databases.", "license": "MIT", "readmeFilename": "README.md", diff --git a/src/driver/mysql/MysqlDriver.ts b/src/driver/mysql/MysqlDriver.ts index b13b13496..8da812b6d 100644 --- a/src/driver/mysql/MysqlDriver.ts +++ b/src/driver/mysql/MysqlDriver.ts @@ -260,18 +260,19 @@ export class MysqlDriver implements Driver { return [sql, []]; const escapedParameters: any[] = []; - const keys = Object.keys(parameters).map(parameter => "(:" + parameter + "\\b)").join("|"); - sql = sql.replace(new RegExp(keys, "g"), (key: string) => { - let value = parameters[key.substr(1)]; - if (value instanceof Function) { - return value(); + Object.keys(parameters).forEach(key => { + sql = sql.replace(new RegExp("(:" + key + "\\b)", "g"), (key: string): string => { + let value = parameters[key.substr(1)]; + if (value instanceof Function) { + return value(); - } else { - if (value instanceof ArrayParameter) value = value.value; - escapedParameters.push(parameters[key.substr(1)]); - return "?"; - } - }); // todo: make replace only in value statements, otherwise problems + } else { + if (value instanceof ArrayParameter) value = value.value; + escapedParameters.push(parameters[key.substr(1)]); + return "?"; + } + }); + }); return [sql, escapedParameters]; } diff --git a/src/driver/oracle/OracleDriver.ts b/src/driver/oracle/OracleDriver.ts index b5943173b..2f7c43c5a 100644 --- a/src/driver/oracle/OracleDriver.ts +++ b/src/driver/oracle/OracleDriver.ts @@ -239,18 +239,19 @@ export class OracleDriver implements Driver { if (!parameters || !Object.keys(parameters).length) return [sql, []]; const escapedParameters: any[] = []; - const keys = Object.keys(parameters).map(parameter => "(:" + parameter + "\\b)").join("|"); - sql = sql.replace(new RegExp(keys, "g"), (key: string) => { - let value = parameters[key.substr(1)]; - if (value instanceof Function) { - return value(); + Object.keys(parameters).forEach(key => { + sql = sql.replace(new RegExp("(:" + key + "\\b)", "g"), (key: string): string => { + let value = parameters[key.substr(1)]; + if (value instanceof Function) { + return value(); - } else { - if (value instanceof ArrayParameter) value = value.value; - escapedParameters.push(value); - return key; - } - }); // todo: make replace only in value statements, otherwise problems + } else { + if (value instanceof ArrayParameter) value = value.value; + escapedParameters.push(value); + return key; + } + }); + }); return [sql, escapedParameters]; } diff --git a/src/driver/postgres/PostgresDriver.ts b/src/driver/postgres/PostgresDriver.ts index 8056da917..4314f4cb1 100644 --- a/src/driver/postgres/PostgresDriver.ts +++ b/src/driver/postgres/PostgresDriver.ts @@ -353,24 +353,25 @@ export class PostgresDriver implements Driver { return [sql, []]; const builtParameters: any[] = []; - const keys = Object.keys(parameters).map(parameter => "(:" + parameter + "\\b)").join("|"); - sql = sql.replace(new RegExp(keys, "g"), (key: string): string => { - let value = parameters[key.substr(1)]; - if (value instanceof Array) { - return value.map((v: any) => { - builtParameters.push(v); + Object.keys(parameters).forEach(key => { + sql = sql.replace(new RegExp("(:" + key + "\\b)", "g"), (key: string): string => { + let value = parameters[key.substr(1)]; + if (value instanceof Array) { + return value.map((v: any) => { + builtParameters.push(v); + return "$" + builtParameters.length; + }).join(", "); + + } else if (value instanceof Function) { + return value(); + + } else { + if (value instanceof ArrayParameter) value = value.value; + builtParameters.push(value); return "$" + builtParameters.length; - }).join(", "); - - } else if (value instanceof Function) { - return value(); - - } else { - if (value instanceof ArrayParameter) value = value.value; - builtParameters.push(value); - return "$" + builtParameters.length; - } - }); // todo: make replace only in value statements, otherwise problems + } + }); // todo: make replace only in value statements, otherwise problems + }); return [sql, builtParameters]; } diff --git a/src/driver/sqlite-abstract/AbstractSqliteDriver.ts b/src/driver/sqlite-abstract/AbstractSqliteDriver.ts index 825a3af79..78ae31cbb 100644 --- a/src/driver/sqlite-abstract/AbstractSqliteDriver.ts +++ b/src/driver/sqlite-abstract/AbstractSqliteDriver.ts @@ -271,24 +271,25 @@ export class AbstractSqliteDriver implements Driver { return [sql, []]; const builtParameters: any[] = []; - const keys = Object.keys(parameters).map(parameter => "(:" + parameter + "\\b)").join("|"); - sql = sql.replace(new RegExp(keys, "g"), (key: string): string => { - let value = parameters[key.substr(1)]; - if (value instanceof Array) { - return value.map((v: any) => { - builtParameters.push(v); + Object.keys(parameters).forEach(key => { + sql = sql.replace(new RegExp("(:" + key + "\\b)", "g"), (key: string): string => { + let value = parameters[key.substr(1)]; + if (value instanceof Array) { + return value.map((v: any) => { + builtParameters.push(v); + return "$" + builtParameters.length; + }).join(", "); + + } else if (value instanceof Function) { + return value(); + + } else { + if (value instanceof ArrayParameter) value = value.value; + builtParameters.push(value); return "$" + builtParameters.length; - }).join(", "); - - } else if (value instanceof Function) { - return value(); - - } else { - if (value instanceof ArrayParameter) value = value.value; - builtParameters.push(value); - return "$" + builtParameters.length; - } - }); // todo: make replace only in value statements, otherwise problems + } + }); + }); return [sql, builtParameters]; } diff --git a/src/driver/sqlserver/SqlServerDriver.ts b/src/driver/sqlserver/SqlServerDriver.ts index e29354464..0af79b356 100644 --- a/src/driver/sqlserver/SqlServerDriver.ts +++ b/src/driver/sqlserver/SqlServerDriver.ts @@ -245,23 +245,24 @@ export class SqlServerDriver implements Driver { if (!parameters || !Object.keys(parameters).length) return [sql, []]; const escapedParameters: any[] = []; - const keys = Object.keys(parameters).map(parameter => "(:" + parameter + "\\b)").join("|"); - sql = sql.replace(new RegExp(keys, "g"), (key: string) => { - let value = parameters[key.substr(1)]; - if (value instanceof Array) { - return value.map((v: any) => { - escapedParameters.push(v); - return "@" + (escapedParameters.length - 1); - }).join(", "); - } else if (value instanceof Function) { - return value(); + Object.keys(parameters).forEach(key => { + sql = sql.replace(new RegExp("(:" + key + "\\b)", "g"), (key: string): string => { + let value = parameters[key.substr(1)]; + if (value instanceof Array) { + return value.map((v: any) => { + escapedParameters.push(v); + return "@" + (escapedParameters.length - 1); + }).join(", "); + } else if (value instanceof Function) { + return value(); - } else { - if (value instanceof ArrayParameter) value = value.value; - escapedParameters.push(value); - return "@" + (escapedParameters.length - 1); - } - }); // todo: make replace only in value statements, otherwise problems + } else { + if (value instanceof ArrayParameter) value = value.value; + escapedParameters.push(value); + return "@" + (escapedParameters.length - 1); + } + }); + }); return [sql, escapedParameters]; } diff --git a/src/driver/websql/WebsqlDriver.ts b/src/driver/websql/WebsqlDriver.ts index 8dafecfae..3acb779d2 100644 --- a/src/driver/websql/WebsqlDriver.ts +++ b/src/driver/websql/WebsqlDriver.ts @@ -106,18 +106,19 @@ export class WebsqlDriver extends AbstractSqliteDriver { if (!parameters || !Object.keys(parameters).length) return [sql, []]; const escapedParameters: any[] = []; - const keys = Object.keys(parameters).map(parameter => "(:" + parameter + "\\b)").join("|"); - sql = sql.replace(new RegExp(keys, "g"), (key: string) => { - let value = parameters[key.substr(1)]; - if (value instanceof Function) { - return value(); + Object.keys(parameters).forEach(key => { + sql = sql.replace(new RegExp("(:" + key + "\\b)", "g"), (key: string): string => { + let value = parameters[key.substr(1)]; + if (value instanceof Function) { + return value(); - } else { - if (value instanceof ArrayParameter) value = value.value; - escapedParameters.push(value); - return "?"; - } - }); // todo: make replace only in value statements, otherwise problems + } else { + if (value instanceof ArrayParameter) value = value.value; + escapedParameters.push(value); + return "?"; + } + }); + }); return [sql, escapedParameters]; } diff --git a/src/persistence/SubjectExecutor.ts b/src/persistence/SubjectExecutor.ts index eedb0fde0..74ad40ade 100644 --- a/src/persistence/SubjectExecutor.ts +++ b/src/persistence/SubjectExecutor.ts @@ -163,6 +163,7 @@ export class SubjectExecutor { // then we run insertion in the sequential order which is important since we have an ordered subjects await PromiseUtils.runInSequence(Object.keys(groupedInsertSubjects), async groupName => { + const subjects = groupedInsertSubjects[groupName]; const insertMaps = subjects.map(subject => { if (this.queryRunner.connection.driver instanceof MongoDriver) { diff --git a/src/query-builder/InsertQueryBuilder.ts b/src/query-builder/InsertQueryBuilder.ts index f845e9620..c3ad3b2ce 100644 --- a/src/query-builder/InsertQueryBuilder.ts +++ b/src/query-builder/InsertQueryBuilder.ts @@ -29,6 +29,7 @@ export class InsertQueryBuilder extends QueryBuilder { */ getQuery(): string { let sql = this.createInsertExpression(); + console.log("wtf"); return sql.trim(); } @@ -37,6 +38,7 @@ export class InsertQueryBuilder extends QueryBuilder { */ async execute(): Promise { const queryRunner = this.obtainQueryRunner(); + console.log("hello execution"); let transactionStartedByUs: boolean = false; try { @@ -47,7 +49,9 @@ export class InsertQueryBuilder extends QueryBuilder { transactionStartedByUs = true; } + console.log("getting value sets"); const valueSets: ObjectLiteral[] = this.getValueSets(); + console.log("got value set"); // call before insertion methods in listeners and subscribers if (this.expressionMap.callListeners === true && this.expressionMap.mainAlias!.hasMetadata) { @@ -63,8 +67,10 @@ export class InsertQueryBuilder extends QueryBuilder { } // execute query + console.log("get query and parameters"); const [sql, parameters] = this.getQueryAndParameters(); const insertResult = new InsertResult(); + console.log("query execution"); insertResult.raw = await queryRunner.query(sql, parameters); // load returning results and set them to the entity if entity updation is enabled @@ -201,11 +207,13 @@ export class InsertQueryBuilder extends QueryBuilder { * Creates INSERT express used to perform insert query. */ protected createInsertExpression() { + console.log("getting query"); const tableName = this.getTableName(this.getMainTableName()); const returningExpression = this.createReturningExpression(); const columnsExpression = this.createColumnNamesExpression(); const valuesExpression = this.createValuesExpression(); + console.log("finishing query"); // generate INSERT query let query = `INSERT INTO ${tableName}`; diff --git a/test/benchmark/bulk-save/bulk-save.ts b/test/benchmark/bulk-save-case1/bulk-save-case1.ts similarity index 89% rename from test/benchmark/bulk-save/bulk-save.ts rename to test/benchmark/bulk-save-case1/bulk-save-case1.ts index 1421506c3..646d1f21c 100644 --- a/test/benchmark/bulk-save/bulk-save.ts +++ b/test/benchmark/bulk-save-case1/bulk-save-case1.ts @@ -3,16 +3,13 @@ import {Connection} from "../../../src/connection/Connection"; import {closeTestingConnections, createTestingConnections, reloadTestingDatabases} from "../../utils/test-utils"; import {Post} from "./entity/Post"; -describe.skip("benchmark > bulk-save", () => { +describe.skip("benchmark > bulk-save > case1", () => { let connections: Connection[]; - before(async () => connections = await createTestingConnections({ - entities: [__dirname + "/entity/*{.js,.ts}"], - })); + before(async () => connections = await createTestingConnections({ __dirname })); beforeEach(() => reloadTestingDatabases(connections)); after(() => closeTestingConnections(connections)); - it("testing bulk save of 1000 objects", () => Promise.all(connections.map(async connection => { const posts: Post[] = []; @@ -27,7 +24,8 @@ describe.skip("benchmark > bulk-save", () => { posts.push(post); } - await connection.manager.save(posts); + // await connection.manager.save(posts); + await connection.manager.insert(Post, posts); }))); diff --git a/test/benchmark/bulk-save/entity/Post.ts b/test/benchmark/bulk-save-case1/entity/Post.ts similarity index 100% rename from test/benchmark/bulk-save/entity/Post.ts rename to test/benchmark/bulk-save-case1/entity/Post.ts diff --git a/test/benchmark/bulk-save-case2/bulk-save-case2.ts b/test/benchmark/bulk-save-case2/bulk-save-case2.ts new file mode 100644 index 000000000..1befdc724 --- /dev/null +++ b/test/benchmark/bulk-save-case2/bulk-save-case2.ts @@ -0,0 +1,52 @@ +import "reflect-metadata"; +import {Connection} from "../../../src/connection/Connection"; +import {closeTestingConnections, createTestingConnections, reloadTestingDatabases} from "../../utils/test-utils"; +import {Document} from "./entity/Document"; + +describe.skip("benchmark > bulk-save > case2", () => { + + let connections: Connection[]; + before(async () => connections = await createTestingConnections({ __dirname, enabledDrivers: ["postgres"] })); + beforeEach(() => reloadTestingDatabases(connections)); + after(() => closeTestingConnections(connections)); + + it("testing bulk save of 1000 objects", () => Promise.all(connections.map(async connection => { + + const documents: Document[] = []; + for (let i = 0; i < 1000; i++) { + const document = new Document(); + + document.id = i.toString(); + document.docId = "label/" + i; + document.context = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent vel faucibus nunc. Etiam volutpat vel urna in scelerisque. Cras a erat ipsum. "; + document.label = "label/" + i; + document.distributions = [ + { + weight: "0.9", + id: i, + docId: i + }, + { + weight: "0.23123", + id: i, + docId: i + }, + { + weight: "0.12312", + id: i, + docId: i + } + ]; + document.date = new Date(); + + documents.push(document); + // await connection.manager.save(document); + // await connection.manager.insert(Document, document); + } + + await connection.manager.save(documents); + // await connection.manager.insert(Document, documents); + + }))); + +}); \ No newline at end of file diff --git a/test/benchmark/bulk-save-case2/entity/Document.ts b/test/benchmark/bulk-save-case2/entity/Document.ts new file mode 100644 index 000000000..eba2df8c0 --- /dev/null +++ b/test/benchmark/bulk-save-case2/entity/Document.ts @@ -0,0 +1,31 @@ +import {Entity} from "../../../../src/decorator/entity/Entity"; +import {Column} from "../../../../src/decorator/columns/Column"; +import {PrimaryColumn} from "../../../../src/decorator/columns/PrimaryColumn"; + +@Entity() +export class Document { + + @PrimaryColumn("text") + id: string; + + @Column("text") + docId: string; + + @Column("text") + label: string; + + @Column("text") + context: string; + + @Column({type: "jsonb"}) + distributions: Distribution[]; + + @Column({type: "timestamp with time zone"}) + date: Date; +} + +export interface Distribution { + weight: string; + id: number; + docId: number; +} \ No newline at end of file