archive-gh-me/typeorm
1
0
Fork 0
mirror of https://github.com/typeorm/typeorm.git synced 2025-12-08 21:26:23 +00:00
Code Issues Projects Releases Wiki Activity

fixed performance issues

Browse Source
This commit is contained in:
Umed Khudoiberdiev 2017-11-17 19:10:17 +05:00
parent 57b0646ebc
commit 1e71e853d9
17 changed files with 219 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@ -1,7 +1,7 @@
{
"name": "typeorm",
"private": true,
"version": "0.2.0-alpha.1",
"version": "0.2.0-alpha.5",
"description": "Data-Mapper ORM for TypeScript, ES7, ES6, ES5. Supports MySQL, PostgreSQL, MariaDB, SQLite, MS SQL Server, Oracle, WebSQL, MongoDB databases.",
"license": "MIT",
"readmeFilename": "README.md",

7
src/driver/Driver.ts
View File

@ -88,7 +88,7 @@ export interface Driver {
* Replaces parameters in the given sql with special escaping character
* and an array of parameter names to be passed to a query.
*/
escapeQueryWithParameters(sql: string, parameters: ObjectLiteral): [string, any[]];
escapeQueryWithParameters(sql: string, parameters: ObjectLiteral, nativeParameters: ObjectLiteral): [string, any[]];
/**
* Escapes a table name, column name or an alias.
@ -159,4 +159,9 @@ export interface Driver {
*/
isUUIDGenerationSupported(): boolean;
/**
* Creates an escaped parameter.
*/
createParameter(parameterName: string, index: number): string;
}

9
src/driver/mongodb/MongoDriver.ts
View File

@ -208,7 +208,7 @@ export class MongoDriver implements Driver {
* Replaces parameters in the given sql with special escaping character
* and an array of parameter names to be passed to a query.
*/
escapeQueryWithParameters(sql: string, parameters: ObjectLiteral): [string, any[]] {
escapeQueryWithParameters(sql: string, parameters: ObjectLiteral, nativeParameters: ObjectLiteral): [string, any[]] {
throw new Error(`This operation is not supported by Mongodb driver.`);
}
@ -311,6 +311,13 @@ export class MongoDriver implements Driver {
return false;
}
/**
* Creates an escaped parameter.
*/
createParameter(parameterName: string, index: number): string {
return "";
}
// -------------------------------------------------------------------------
// Protected Methods
// -------------------------------------------------------------------------

13
src/driver/mysql/MysqlDriver.ts
View File

@ -255,11 +255,11 @@ export class MysqlDriver implements Driver {
* Replaces parameters in the given sql with special escaping character
* and an array of parameter names to be passed to a query.
*/
escapeQueryWithParameters(sql: string, parameters: ObjectLiteral): [string, any[]] {
escapeQueryWithParameters(sql: string, parameters: ObjectLiteral, nativeParameters: ObjectLiteral): [string, any[]] {
const escapedParameters: any[] = Object.keys(nativeParameters).map(key => nativeParameters[key]);
if (!parameters || !Object.keys(parameters).length)
return [sql, []];
return [sql, escapedParameters];
const escapedParameters: any[] = [];
const keys = Object.keys(parameters).map(parameter => "(:" + parameter + "\\b)").join("|");
sql = sql.replace(new RegExp(keys, "g"), (key: string) => {
let value = parameters[key.substr(1)];
@ -513,6 +513,13 @@ export class MysqlDriver implements Driver {
return false;
}
/**
* Creates an escaped parameter.
*/
createParameter(parameterName: string, index: number): string {
return "?";
}
// -------------------------------------------------------------------------
// Protected Methods
// -------------------------------------------------------------------------

14
src/driver/oracle/OracleDriver.ts
View File

@ -235,10 +235,11 @@ export class OracleDriver implements Driver {
* Replaces parameters in the given sql with special escaping character
* and an array of parameter names to be passed to a query.
*/
escapeQueryWithParameters(sql: string, parameters: ObjectLiteral): [string, any[]] {
escapeQueryWithParameters(sql: string, parameters: ObjectLiteral, nativeParameters: ObjectLiteral): [string, any[]] {
const escapedParameters: any[] = Object.keys(nativeParameters).map(key => nativeParameters[key]);
if (!parameters || !Object.keys(parameters).length)
return [sql, []];
const escapedParameters: any[] = [];
return [sql, escapedParameters];
const keys = Object.keys(parameters).map(parameter => "(:" + parameter + "\\b)").join("|");
sql = sql.replace(new RegExp(keys, "g"), (key: string) => {
let value = parameters[key.substr(1)];
@ -470,6 +471,13 @@ export class OracleDriver implements Driver {
return false;
}
/**
* Creates an escaped parameter.
*/
createParameter(parameterName: string, index: number): string {
return ":" + parameterName;
}
// -------------------------------------------------------------------------
// Protected Methods
// -------------------------------------------------------------------------

13
src/driver/postgres/PostgresDriver.ts
View File

@ -348,11 +348,11 @@ export class PostgresDriver implements Driver {
* Replaces parameters in the given sql with special escaping character
* and an array of parameter names to be passed to a query.
*/
escapeQueryWithParameters(sql: string, parameters: ObjectLiteral): [string, any[]] {
escapeQueryWithParameters(sql: string, parameters: ObjectLiteral, nativeParameters: ObjectLiteral): [string, any[]] {
const builtParameters: any[] = Object.keys(nativeParameters).map(key => nativeParameters[key]);
if (!parameters || !Object.keys(parameters).length)
return [sql, []];
return [sql, builtParameters];
const builtParameters: any[] = [];
const keys = Object.keys(parameters).map(parameter => "(:" + parameter + "\\b)").join("|");
sql = sql.replace(new RegExp(keys, "g"), (key: string): string => {
let value = parameters[key.substr(1)];
@ -598,6 +598,13 @@ export class PostgresDriver implements Driver {
return true;
}
/**
* Creates an escaped parameter.
*/
createParameter(parameterName: string, index: number): string {
return "$" + (index + 1);
}
// -------------------------------------------------------------------------
// Public Methods
// -------------------------------------------------------------------------

13
src/driver/sqlite-abstract/AbstractSqliteDriver.ts
View File

@ -266,11 +266,11 @@ export class AbstractSqliteDriver implements Driver {
* Replaces parameters in the given sql with special escaping character
* and an array of parameter names to be passed to a query.
*/
escapeQueryWithParameters(sql: string, parameters: ObjectLiteral): [string, any[]] {
escapeQueryWithParameters(sql: string, parameters: ObjectLiteral, nativeParameters: ObjectLiteral): [string, any[]] {
const builtParameters: any[] = Object.keys(nativeParameters).map(key => nativeParameters[key]);
if (!parameters || !Object.keys(parameters).length)
return [sql, []];
return [sql, builtParameters];
const builtParameters: any[] = [];
const keys = Object.keys(parameters).map(parameter => "(:" + parameter + "\\b)").join("|");
sql = sql.replace(new RegExp(keys, "g"), (key: string): string => {
let value = parameters[key.substr(1)];
@ -444,6 +444,13 @@ export class AbstractSqliteDriver implements Driver {
return false;
}
/**
* Creates an escaped parameter.
*/
createParameter(parameterName: string, index: number): string {
return "$" + (index + 1);
}
// -------------------------------------------------------------------------
// Protected Methods
// -------------------------------------------------------------------------

14
src/driver/sqlserver/SqlServerDriver.ts
View File

@ -241,10 +241,11 @@ export class SqlServerDriver implements Driver {
* Replaces parameters in the given sql with special escaping character
* and an array of parameter names to be passed to a query.
*/
escapeQueryWithParameters(sql: string, parameters: ObjectLiteral): [string, any[]] {
escapeQueryWithParameters(sql: string, parameters: ObjectLiteral, nativeParameters: ObjectLiteral): [string, any[]] {
const escapedParameters: any[] = Object.keys(nativeParameters).map(key => nativeParameters[key]);
if (!parameters || !Object.keys(parameters).length)
return [sql, []];
const escapedParameters: any[] = [];
return [sql, escapedParameters];
const keys = Object.keys(parameters).map(parameter => "(:" + parameter + "\\b)").join("|");
sql = sql.replace(new RegExp(keys, "g"), (key: string) => {
let value = parameters[key.substr(1)];
@ -498,6 +499,13 @@ export class SqlServerDriver implements Driver {
return true;
}
/**
* Creates an escaped parameter.
*/
createParameter(parameterName: string, index: number): string {
return "@" + index;
}
// -------------------------------------------------------------------------
// Public Methods
// -------------------------------------------------------------------------

7
src/driver/websql/WebsqlDriver.ts
View File

@ -102,10 +102,11 @@ export class WebsqlDriver extends AbstractSqliteDriver {
* Replaces parameters in the given sql with special escaping character
* and an array of parameter names to be passed to a query.
*/
escapeQueryWithParameters(sql: string, parameters: ObjectLiteral): [string, any[]] {
escapeQueryWithParameters(sql: string, parameters: ObjectLiteral, nativeParameters: ObjectLiteral): [string, any[]] {
const escapedParameters: any[] = Object.keys(nativeParameters).map(key => nativeParameters[key]);
if (!parameters || !Object.keys(parameters).length)
return [sql, []];
const escapedParameters: any[] = [];
return [sql, escapedParameters];
const keys = Object.keys(parameters).map(parameter => "(:" + parameter + "\\b)").join("|");
sql = sql.replace(new RegExp(keys, "g"), (key: string) => {
let value = parameters[key.substr(1)];

66
src/query-builder/InsertQueryBuilder.ts
View File

@ -1,5 +1,4 @@
import {QueryBuilder} from "./QueryBuilder";
import {ArrayParameter} from "./ArrayParameter";
import {ObjectLiteral} from "../common/ObjectLiteral";
import {ObjectType} from "../common/ObjectType";
import {QueryPartialEntity} from "./QueryPartialEntity";
@ -292,9 +291,13 @@ export class InsertQueryBuilder<Entity> extends QueryBuilder<Entity> {
// if column metadatas are given then apply all necessary operations with values
if (columns.length > 0) {
return valueSets.map((valueSet, valueSetIndex) => {
const columnValues = columns.map(column => {
let expression = "";
let parametersCount = Object.keys(this.expressionMap.nativeParameters).length;
valueSets.forEach((valueSet, valueSetIndex) => {
columns.forEach((column, columnIndex) => {
if (columnIndex === 0) {
expression += "(";
}
const paramName = "_inserted_" + valueSetIndex + "_" + column.databaseName;
// extract real value from the entity
@ -312,7 +315,8 @@ export class InsertQueryBuilder<Entity> extends QueryBuilder<Entity> {
// newly inserted entities always have a version equal to 1 (first version)
if (column.isVersion) {
return "1";
expression += "1";
// return "1";
// for create and update dates we insert current date
// no, we don't do it because this constant is already in "default" value of the column
@ -322,44 +326,56 @@ export class InsertQueryBuilder<Entity> extends QueryBuilder<Entity> {
// if column is generated uuid and database does not support its generation and custom generated value was not provided by a user - we generate a new uuid value for insertion
} else if (column.isGenerated && column.generationStrategy === "uuid" && !this.connection.driver.isUUIDGenerationSupported() && value === undefined) {
const paramName = "_uuid_" + column.databaseName + valueSetIndex;
this.setParameter(paramName, RandomGenerator.uuid4());
return ":" + paramName;
value = RandomGenerator.uuid4();
this.expressionMap.nativeParameters[paramName] = value;
expression += this.connection.driver.createParameter(paramName, parametersCount);
parametersCount++;
// if value for this column was not provided then insert default value
} else if (value === undefined) {
if (this.connection.driver instanceof AbstractSqliteDriver) { // unfortunately sqlite does not support DEFAULT expression in INSERT queries
if (column.default !== undefined) { // try to use default defined in the column
return this.connection.driver.normalizeDefault(column);
expression += this.connection.driver.normalizeDefault(column);
} else {
expression += "NULL"; // otherwise simply use NULL and pray if column is nullable
}
return "NULL"; // otherwise simply use NULL and pray if column is nullable
} else {
return "DEFAULT";
expression += "DEFAULT";
}
// support for SQL expressions in queries
} else if (value instanceof Function) {
return value();
expression += value();
// just any other regular value
} else {
if (this.connection.driver instanceof SqlServerDriver) {
this.setParameter(paramName, this.connection.driver.parametrizeValue(column, value));
} else {
if (this.connection.driver instanceof SqlServerDriver)
value = this.connection.driver.parametrizeValue(column, value);
// we need to store array values in a special class to make sure parameter replacement will work correctly
if (value instanceof Array)
value = new ArrayParameter(value);
// we need to store array values in a special class to make sure parameter replacement will work correctly
// if (value instanceof Array)
// value = new ArrayParameter(value);
this.setParameter(paramName, value);
}
return ":" + paramName;
this.expressionMap.nativeParameters[paramName] = value;
expression += this.connection.driver.createParameter(paramName, parametersCount);
parametersCount++;
}
}).join(", ").trim();
return columnValues ? "(" + columnValues + ")" : "";
}).join(", ");
if (columnIndex === columns.length - 1) {
if (valueSetIndex === valueSets.length - 1) {
expression += ")";
} else {
expression += "), ";
}
} else {
expression += ", ";
}
});
});
return expression;
} else { // for tables without metadata
@ -384,8 +400,8 @@ export class InsertQueryBuilder<Entity> extends QueryBuilder<Entity> {
// just any other regular value
} else {
this.setParameter(paramName, value);
return ":" + paramName;
this.expressionMap.nativeParameters[paramName] = value;
return this.connection.driver.createParameter(paramName, Object.keys(this.expressionMap.nativeParameters).length - 1);
}
}).join(", ").trim();

5
src/query-builder/QueryBuilder.ts
View File

@ -357,7 +357,10 @@ export abstract class QueryBuilder<Entity> {
* Gets query to be executed with all parameters used in it.
*/
getQueryAndParameters(): [string, any[]] {
return this.connection.driver.escapeQueryWithParameters(this.getQuery(), this.getParameters());
// this execution order is important because getQuery method generates this.expressionMap.nativeParameters values
const query = this.getQuery();
const parameters = this.getParameters();
return this.connection.driver.escapeQueryWithParameters(query, parameters, this.expressionMap.nativeParameters);
}
/**

12
src/query-builder/QueryExpressionMap.ts
View File

@ -233,6 +233,12 @@ export class QueryExpressionMap {
*/
useTransaction: boolean = false;
/**
* Extra parameters.
* Used in InsertQueryBuilder to avoid default parameters mechanizm and execute high performance insertions.
*/
nativeParameters: ObjectLiteral = {};
// -------------------------------------------------------------------------
// Constructor
// -------------------------------------------------------------------------
@ -379,6 +385,12 @@ export class QueryExpressionMap {
map.cacheDuration = this.cacheDuration;
map.relationPropertyPath = this.relationPropertyPath;
map.of = this.of;
map.insertColumns = this.insertColumns;
map.whereEntities = this.whereEntities;
map.updateEntity = this.updateEntity;
map.callListeners = this.callListeners;
map.useTransaction = this.useTransaction;
map.nativeParameters = this.nativeParameters;
return map;
}

2
src/query-builder/ReturningResultsEntityUpdator.ts
View File

@ -89,7 +89,7 @@ export class ReturningResultsEntityUpdator {
// uuid can be defined by user in a model, that's why first we get it
let uuid = generatedColumn.getEntityValue(entity);
if (!uuid) // if it was not defined by a user then InsertQueryBuilder generates it by its own, get this generated uuid value
uuid = this.expressionMap.parameters["_uuid_" + generatedColumn.databaseName + entityIndex];
uuid = this.expressionMap.nativeParameters["_uuid_" + generatedColumn.databaseName + entityIndex];
OrmUtils.mergeDeep(generatedMap, generatedColumn.createValueMap(uuid));
}

14
test/benchmark/bulk-save/bulk-save.ts → test/benchmark/bulk-save-case1/bulk-save-case1.ts
View File

@ -3,21 +3,18 @@ import {Connection} from "../../../src/connection/Connection";
import {closeTestingConnections, createTestingConnections, reloadTestingDatabases} from "../../utils/test-utils";
import {Post} from "./entity/Post";
describe.skip("benchmark > bulk-save", () => {
describe("benchmark > bulk-save > case1", () => {
let connections: Connection[];
before(async () => connections = await createTestingConnections({
entities: [__dirname + "/entity/*{.js,.ts}"],
}));
before(async () => connections = await createTestingConnections({ __dirname, enabledDrivers: ["mysql", "postgres"] }));
beforeEach(() => reloadTestingDatabases(connections));
after(() => closeTestingConnections(connections));
it("testing bulk save of 1000 objects", () => Promise.all(connections.map(async connection => {
it("testing bulk save of 10.000 objects", () => Promise.all(connections.map(async connection => {
const posts: Post[] = [];
for (let i = 1; i <= 1000; i++) {
for (let i = 1; i <= 10000; i++) {
const post = new Post();
post.title = `Post #${i}`;
post.text = `Post #${i} text`;
@ -27,7 +24,8 @@ describe.skip("benchmark > bulk-save", () => {
posts.push(post);
}
await connection.manager.save(posts);
// await connection.manager.save(posts);
await connection.manager.insert(Post, posts);
})));

0
test/benchmark/bulk-save/entity/Post.ts → test/benchmark/bulk-save-case1/entity/Post.ts
View File

53
test/benchmark/bulk-save-case2/bulk-save-case2.ts Normal file
View File

@ -0,0 +1,53 @@
import "reflect-metadata";
import {Connection} from "../../../src/connection/Connection";
import {closeTestingConnections, createTestingConnections, reloadTestingDatabases} from "../../utils/test-utils";
import {Document} from "./entity/Document";
describe("benchmark > bulk-save > case2", () => {
let connections: Connection[];
before(async () => connections = await createTestingConnections({ __dirname, enabledDrivers: ["postgres"] }));
beforeEach(() => reloadTestingDatabases(connections));
after(() => closeTestingConnections(connections));
it("testing bulk save of 5000 objects", () => Promise.all(connections.map(async connection => {
const documents: Document[] = [];
for (let i = 0; i < 5000; i++) {
const document = new Document();
document.id = i.toString();
document.docId = "label/" + i;
document.context = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent vel faucibus nunc. Etiam volutpat vel urna in scelerisque. Cras a erat ipsum. ";
document.label = "label/" + i;
document.distributions = [
{
weight: "0.9",
id: i,
docId: i
},
{
weight: "0.23123",
id: i,
docId: i
},
{
weight: "0.12312",
id: i,
docId: i
}
];
document.date = new Date();
documents.push(document);
// await connection.manager.save(document);
// await connection.manager.insert(Document, document);
}
await connection.manager.insert(Document, documents);
// await connection.manager.save(documents);
// await connection.manager.insert(Document, documents);
})));
});

31
test/benchmark/bulk-save-case2/entity/Document.ts Normal file
View File

@ -0,0 +1,31 @@
import {Entity} from "../../../../src/decorator/entity/Entity";
import {Column} from "../../../../src/decorator/columns/Column";
import {PrimaryColumn} from "../../../../src/decorator/columns/PrimaryColumn";
@Entity()
export class Document {
@PrimaryColumn("text")
id: string;
@Column("text")
docId: string;
@Column("text")
label: string;
@Column("text")
context: string;
@Column({type: "jsonb"})
distributions: Distribution[];
@Column({type: "timestamp with time zone"})
date: Date;
}
export interface Distribution {
weight: string;
id: number;
docId: number;
}