Merge remote-tracking branch 'origin/master' into bmc/search-path

This commit is contained in:
Brian Carlson 2022-05-11 13:34:15 -05:00
commit 60d6313862
17 changed files with 164 additions and 96 deletions

View File

@ -1,6 +1,6 @@
name: CI
on: [push]
on: [push, pull_request]
jobs:
build:
@ -17,12 +17,13 @@ jobs:
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
strategy:
matrix:
node: ['8', '10', '12', '14', '16', '17']
name: Node ${{ matrix.node }}
node: ['8', '10', '12', '14', '16', '18']
os: [ubuntu-latest, windows-latest, macos-latest]
name: Node.js ${{ matrix.node }} (${{ matrix.os }})
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Setup node
uses: actions/setup-node@v2
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node }}
cache: yarn

View File

@ -4,6 +4,10 @@ For richer information consult the commit log on github with referenced pull req
We do not include break-fix version release in this file.
### pg-pool@3.5.0
- Add connection [lifetime limit](https://github.com/brianc/node-postgres/pull/2698) config option.
### pg@8.7.0
- Add optional config to [pool](https://github.com/brianc/node-postgres/pull/2568) to allow process to exit if pool is idle.

View File

@ -11,7 +11,7 @@ node-postgres is made possible by the helpful contributors from the community as
- [Dataform](https://dataform.co/)
- [Eaze](https://www.eaze.com/)
- [simpleanalytics](https://simpleanalytics.com/)
- [n8n.io]https://n8n.io/
- [n8n.io](https://n8n.io/)
# Supporters

View File

@ -86,6 +86,8 @@ class Cursor extends EventEmitter {
}
_closePortal() {
if (this.state === 'done') return
// because we opened a named portal to stream results
// we need to close the same named portal. Leaving a named portal
// open can lock tables for modification if inside a transaction.
@ -97,6 +99,8 @@ class Cursor extends EventEmitter {
if (this.state !== 'error') {
this.connection.sync()
}
this.state = 'done'
}
handleRowDescription(msg) {
@ -213,7 +217,6 @@ class Cursor extends EventEmitter {
}
this._closePortal()
this.state = 'done'
this.connection.once('readyForQuery', function () {
cb()
})

View File

@ -1,6 +1,6 @@
{
"name": "pg-cursor",
"version": "2.7.1",
"version": "2.7.3",
"description": "Query cursor extension for node-postgres",
"main": "index.js",
"directories": {
@ -18,7 +18,7 @@
"license": "MIT",
"devDependencies": {
"mocha": "^7.1.2",
"pg": "^8.7.1"
"pg": "^8.7.3"
},
"peerDependencies": {
"pg": "^8"

View File

@ -136,7 +136,7 @@ because its so common to just run a query and return the client to the pool afte
var pool = new Pool()
var time = await pool.query('SELECT NOW()')
var name = await pool.query('select $1::text as name', ['brianc'])
console.log(name.rows[0].name, 'says hello at', time.rows[0].name)
console.log(name.rows[0].name, 'says hello at', time.rows[0].now)
```
you can also use a callback here if you'd like:

View File

@ -84,6 +84,7 @@ class Pool extends EventEmitter {
this.options.max = this.options.max || this.options.poolSize || 10
this.options.maxUses = this.options.maxUses || Infinity
this.options.allowExitOnIdle = this.options.allowExitOnIdle || false
this.options.maxLifetimeSeconds = this.options.maxLifetimeSeconds || 0
this.log = this.options.log || function () {}
this.Client = this.options.Client || Client || require('pg').Client
this.Promise = this.options.Promise || global.Promise
@ -94,6 +95,7 @@ class Pool extends EventEmitter {
this._clients = []
this._idle = []
this._expired = new WeakSet()
this._pendingQueue = []
this._endCallback = undefined
this.ending = false
@ -123,6 +125,7 @@ class Pool extends EventEmitter {
}
return
}
// if we don't have any waiting, do nothing
if (!this._pendingQueue.length) {
this.log('no queued requests')
@ -248,6 +251,22 @@ class Pool extends EventEmitter {
} else {
this.log('new client connected')
if (this.options.maxLifetimeSeconds !== 0) {
setTimeout(() => {
this.log('ending client due to expired lifetime')
this._expired.add(client)
const idleIndex = this._idle.findIndex((idleItem) => idleItem.client === client)
if (idleIndex !== -1) {
this._acquireClient(
client,
new PendingItem((err, client, clientRelease) => clientRelease()),
idleListener,
false
)
}
}, this.options.maxLifetimeSeconds * 1000)
}
return this._acquireClient(client, pendingItem, idleListener, true)
}
})
@ -318,6 +337,15 @@ class Pool extends EventEmitter {
return
}
const isExpired = this._expired.has(client)
if (isExpired) {
this.log('remove expired client')
this._expired.delete(client)
this._remove(client)
this._pulseQueue()
return
}
// idle timeout
let tid
if (this.options.idleTimeoutMillis) {
@ -414,6 +442,10 @@ class Pool extends EventEmitter {
return this._idle.length
}
get expiredCount() {
return this._clients.reduce((acc, client) => acc + (this._expired.has(client) ? 1 : 0), 0)
}
get totalCount() {
return this._clients.length
}

View File

@ -1,6 +1,6 @@
{
"name": "pg-pool",
"version": "3.4.1",
"version": "3.5.1",
"description": "Connection pool for node-postgres",
"main": "index.js",
"directories": {

View File

@ -0,0 +1,47 @@
'use strict'
const co = require('co')
const expect = require('expect.js')
const describe = require('mocha').describe
const it = require('mocha').it
const path = require('path')
const Pool = require('../')
describe('lifetime timeout', () => {
it('connection lifetime should expire and remove the client', (done) => {
const pool = new Pool({ maxLifetimeSeconds: 1 })
pool.query('SELECT NOW()')
pool.on('remove', () => {
console.log('expired while idle - on-remove event')
expect(pool.expiredCount).to.equal(0)
expect(pool.totalCount).to.equal(0)
done()
})
})
it('connection lifetime should expire and remove the client after the client is done working', (done) => {
const pool = new Pool({ maxLifetimeSeconds: 1 })
pool.query('SELECT pg_sleep(1.01)')
pool.on('remove', () => {
console.log('expired while busy - on-remove event')
expect(pool.expiredCount).to.equal(0)
expect(pool.totalCount).to.equal(0)
done()
})
})
it(
'can remove expired clients and recreate them',
co.wrap(function* () {
const pool = new Pool({ maxLifetimeSeconds: 1 })
let query = pool.query('SELECT pg_sleep(1)')
expect(pool.expiredCount).to.equal(0)
expect(pool.totalCount).to.equal(1)
yield query
expect(pool.expiredCount).to.equal(0)
expect(pool.totalCount).to.equal(0)
yield pool.query('SELECT NOW()')
expect(pool.expiredCount).to.equal(0)
expect(pool.totalCount).to.equal(1)
})
)
})

View File

@ -1,6 +1,6 @@
{
"name": "pg-query-stream",
"version": "4.2.1",
"version": "4.2.3",
"description": "Postgres query result returned as readable stream",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
@ -37,13 +37,12 @@
"concat-stream": "~1.0.1",
"eslint-plugin-promise": "^3.5.0",
"mocha": "^7.1.2",
"pg": "^8.7.1",
"pg": "^8.7.3",
"stream-spec": "~0.3.5",
"stream-tester": "0.0.5",
"ts-node": "^8.5.4",
"typescript": "^4.0.3"
},
"dependencies": {
"pg-cursor": "^2.7.1"
"pg-cursor": "^2.7.3"
}
}

View File

@ -117,5 +117,17 @@ if (!process.version.startsWith('v8')) {
client.release()
await pool.end()
})
it('supports breaking with low watermark', async function () {
const pool = new pg.Pool({ max: 1 })
const client = await pool.connect()
for await (const _ of client.query(new QueryStream('select TRUE', [], { highWaterMark: 1 }))) break
for await (const _ of client.query(new QueryStream('select TRUE', [], { highWaterMark: 1 }))) break
for await (const _ of client.query(new QueryStream('select TRUE', [], { highWaterMark: 1 }))) break
client.release()
await pool.end()
})
})
}

View File

@ -1,8 +1,19 @@
import helper from './helper'
import concat from 'concat-stream'
import tester from 'stream-tester'
import JSONStream from 'JSONStream'
import QueryStream from '../src'
import { Transform, TransformCallback } from 'stream'
class PauseStream extends Transform {
constructor() {
super({ objectMode: true })
}
_transform(chunk, encoding, callback): void {
this.push(chunk, encoding)
setTimeout(callback, 1)
}
}
helper('pauses', function (client) {
it('pauses', function (done) {
@ -12,7 +23,7 @@ helper('pauses', function (client) {
highWaterMark: 2,
})
const query = client.query(stream)
const pauser = tester.createPauseStream(0.1, 100)
const pauser = new PauseStream()
query
.pipe(JSONStream.stringify())
.pipe(pauser)

View File

@ -1,7 +1,6 @@
# node-postgres
[![Build Status](https://secure.travis-ci.org/brianc/node-postgres.svg?branch=master)](http://travis-ci.org/brianc/node-postgres)
[![Dependency Status](https://david-dm.org/brianc/node-postgres.svg?path=packages/pg)](https://david-dm.org/brianc/node-postgres?path=packages/pg)
<span class="badge-npmversion"><a href="https://npmjs.org/package/pg" title="View this project on NPM"><img src="https://img.shields.io/npm/v/pg.svg" alt="NPM version" /></a></span>
<span class="badge-npmdownloads"><a href="https://npmjs.org/package/pg" title="View this project on NPM"><img src="https://img.shields.io/npm/dm/pg.svg" alt="NPM downloads" /></a></span>

View File

@ -1,6 +1,6 @@
{
"name": "pg",
"version": "8.7.1",
"version": "8.7.3",
"description": "PostgreSQL client - pure javascript & libpq with the same API",
"keywords": [
"database",
@ -23,7 +23,7 @@
"buffer-writer": "2.0.0",
"packet-reader": "1.0.0",
"pg-connection-string": "^2.5.0",
"pg-pool": "^3.4.1",
"pg-pool": "^3.5.1",
"pg-protocol": "^1.5.0",
"pg-types": "^2.1.0",
"pgpass": "1.x"

View File

@ -67,7 +67,7 @@ suite.test('successful connection', (done) => {
})
suite.test('expired connection timeout', (done) => {
const opts = { ...options, port: 54322 }
const opts = { ...options, port: options.port + 1 }
serverWithConnectionTimeout(opts.port, opts.connectionTimeoutMillis * 2, (closeServer) => {
const timeoutId = setTimeout(() => {
throw new Error('Client should have emitted an error but it did not.')

View File

@ -11,7 +11,6 @@ var Server = function (response) {
this.response = response
}
let port = 54321
Server.prototype.start = function (cb) {
// this is our fake postgres server
// it responds with our specified response immediatley after receiving every buffer
@ -40,14 +39,13 @@ Server.prototype.start = function (cb) {
}.bind(this)
)
port = port + 1
var options = {
host: 'localhost',
port: port,
}
this.server.listen(options.port, options.host, function () {
cb(options)
const host = 'localhost'
this.server.listen({ host, port: 0 }, () => {
const port = this.server.address().port
cb({
host,
port,
})
})
}

View File

@ -1334,15 +1334,6 @@ assertion-error@^1.1.0:
resolved "https://registry.yarnpkg.com/assertion-error/-/assertion-error-1.1.0.tgz#e60b6b0e8f301bd97e5375215bda406c85118c0b"
integrity sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==
assertions@~2.3.0:
version "2.3.4"
resolved "https://registry.yarnpkg.com/assertions/-/assertions-2.3.4.tgz#a9433ced1fce57cc999af0965d1008e96c2796e6"
integrity sha1-qUM87R/OV8yZmvCWXRAI6WwnluY=
dependencies:
fomatto "git://github.com/BonsaiDen/Fomatto.git#468666f600b46f9067e3da7200fd9df428923ea6"
render "0.1"
traverser "1"
assign-symbols@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/assign-symbols/-/assign-symbols-1.0.0.tgz#59667f41fadd4f20ccbc2bb96b8d4f7f78ec0367"
@ -2010,11 +2001,6 @@ currently-unhandled@^0.4.1:
dependencies:
array-find-index "^1.0.1"
curry@0.0.x:
version "0.0.4"
resolved "https://registry.yarnpkg.com/curry/-/curry-0.0.4.tgz#1750d518d919c44f3d37ff44edc693de1f0d5fcb"
integrity sha1-F1DVGNkZxE89N/9E7caT3h8NX8s=
cyclist@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9"
@ -2755,10 +2741,6 @@ flush-write-stream@^1.0.0:
inherits "^2.0.3"
readable-stream "^2.3.6"
"fomatto@git://github.com/BonsaiDen/Fomatto.git#468666f600b46f9067e3da7200fd9df428923ea6":
version "0.6.0"
resolved "git://github.com/BonsaiDen/Fomatto.git#468666f600b46f9067e3da7200fd9df428923ea6"
for-in@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80"
@ -2793,11 +2775,6 @@ from2@^2.1.0:
inherits "^2.0.1"
readable-stream "^2.0.0"
from@~0.0.2:
version "0.0.2"
resolved "https://registry.yarnpkg.com/from/-/from-0.0.2.tgz#7fffac647a2f99b20d57b8e28379455cbb4189d0"
integrity sha1-f/+sZHovmbINV7jig3lFXLtBidA=
fs-extra@^8.1.0:
version "8.1.0"
resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-8.1.0.tgz#49d43c45a88cd9677668cb7be1b46efdb8d2e1c0"
@ -4323,9 +4300,11 @@ node-fetch-npm@^2.0.2:
safe-buffer "^5.1.1"
node-fetch@^2.5.0, node-fetch@^2.6.1:
version "2.6.1"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052"
integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==
version "2.6.7"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
dependencies:
whatwg-url "^5.0.0"
node-gyp@^5.0.2:
version "5.1.1"
@ -4819,9 +4798,9 @@ path-type@^4.0.0:
integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==
pathval@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/pathval/-/pathval-1.1.0.tgz#b942e6d4bde653005ef6b71361def8727d0645e0"
integrity sha1-uULm1L3mUwBe9rcTYd74cn0GReA=
version "1.1.1"
resolved "https://registry.yarnpkg.com/pathval/-/pathval-1.1.1.tgz#8534e77a77ce7ac5a2512ea21e0fdb8fcf6c3d8d"
integrity sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==
performance-now@^2.1.0:
version "2.1.0"
@ -5213,13 +5192,6 @@ regexpp@^3.0.0, regexpp@^3.1.0:
resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-3.1.0.tgz#206d0ad0a5648cffbdb8ae46438f3dc51c9f78e2"
integrity sha512-ZOIzd8yVsQQA7j8GCSlPGXwg5PfmA1mrq0JP4nGhh54LaKN3xdai/vHUDu74pKwV8OxseMS65u2NImosQcSD0Q==
render@0.1:
version "0.1.4"
resolved "https://registry.yarnpkg.com/render/-/render-0.1.4.tgz#cfb33a34e26068591d418469e23d8cc5ce1ceff5"
integrity sha1-z7M6NOJgaFkdQYRp4j2Mxc4c7/U=
dependencies:
traverser "0.0.x"
repeat-element@^1.1.2:
version "1.1.3"
resolved "https://registry.yarnpkg.com/repeat-element/-/repeat-element-1.1.3.tgz#782e0d825c0c5a3bb39731f84efee6b742e6b1ce"
@ -5681,15 +5653,6 @@ stream-spec@~0.3.5:
dependencies:
macgyver "~1.10"
stream-tester@0.0.5:
version "0.0.5"
resolved "https://registry.yarnpkg.com/stream-tester/-/stream-tester-0.0.5.tgz#4f86f2531149adaf6dd4b3ff262edf64ae9a171a"
integrity sha1-T4byUxFJra9t1LP/Ji7fZK6aFxo=
dependencies:
assertions "~2.3.0"
from "~0.0.2"
through "~0.0.3"
string-width@^1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3"
@ -5942,11 +5905,6 @@ through@2, "through@>=2.2.7 <3", through@^2.3.4, through@^2.3.6:
resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=
through@~0.0.3:
version "0.0.4"
resolved "https://registry.yarnpkg.com/through/-/through-0.0.4.tgz#0bf2f0fffafaac4bacbc533667e98aad00b588c8"
integrity sha1-C/Lw//r6rEusvFM2Z+mKrQC1iMg=
tmp@^0.0.33:
version "0.0.33"
resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9"
@ -6006,19 +5964,10 @@ tr46@^1.0.1:
dependencies:
punycode "^2.1.0"
traverser@0.0.x:
version "0.0.5"
resolved "https://registry.yarnpkg.com/traverser/-/traverser-0.0.5.tgz#c66f38c456a0c21a88014b1223580c7ebe0631eb"
integrity sha1-xm84xFagwhqIAUsSI1gMfr4GMes=
dependencies:
curry "0.0.x"
traverser@1:
version "1.0.0"
resolved "https://registry.yarnpkg.com/traverser/-/traverser-1.0.0.tgz#6f59e5813759aeeab3646b8f4513fd4a62e4fe20"
integrity sha1-b1nlgTdZruqzZGuPRRP9SmLk/iA=
dependencies:
curry "0.0.x"
tr46@~0.0.3:
version "0.0.3"
resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=
trim-newlines@^1.0.0:
version "1.0.0"
@ -6036,9 +5985,9 @@ trim-newlines@^3.0.0:
integrity sha512-C4+gOpvmxaSMKuEf9Qc134F1ZuOHVXKRbtEflf4NTtuuJDEIJ9p5PXsalL8SkeRw+qit1Mo+yuvMPAKwWg/1hA==
trim-off-newlines@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/trim-off-newlines/-/trim-off-newlines-1.0.1.tgz#9f9ba9d9efa8764c387698bcbfeb2c848f11adb3"
integrity sha1-n5up2e+odkw4dpi8v+sshI8RrbM=
version "1.0.3"
resolved "https://registry.yarnpkg.com/trim-off-newlines/-/trim-off-newlines-1.0.3.tgz#8df24847fcb821b0ab27d58ab6efec9f2fe961a1"
integrity sha512-kh6Tu6GbeSNMGfrrZh6Bb/4ZEHV1QlB4xNDBeog8Y9/QwFlKTRyWvY3Fs9tRDAMZliVUwieMgEdIeL/FtqjkJg==
ts-node@^8.5.4:
version "8.10.2"
@ -6263,11 +6212,24 @@ wcwidth@^1.0.0:
dependencies:
defaults "^1.0.3"
webidl-conversions@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"
integrity sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=
webidl-conversions@^4.0.2:
version "4.0.2"
resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad"
integrity sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==
whatwg-url@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d"
integrity sha1-lmRU6HZUYuN2RNNib2dCzotwll0=
dependencies:
tr46 "~0.0.3"
webidl-conversions "^3.0.0"
whatwg-url@^7.0.0:
version "7.1.0"
resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-7.1.0.tgz#c2c492f1eca612988efd3d2266be1b9fc6170d06"