From 24c4cd7bb81c983fb9bde4e62d66493a02733cdc Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Tue, 18 Oct 2022 16:29:22 -0700 Subject: [PATCH] grpc-js: Add more outlier detection tests and tracing --- .../src/load-balancer-outlier-detection.ts | 4 +- .../grpc-js/test/test-outlier-detection.ts | 169 ++++++++++++++++++ 2 files changed, 172 insertions(+), 1 deletion(-) diff --git a/packages/grpc-js/src/load-balancer-outlier-detection.ts b/packages/grpc-js/src/load-balancer-outlier-detection.ts index 1f1710e7..cdf58052 100644 --- a/packages/grpc-js/src/load-balancer-outlier-detection.ts +++ b/packages/grpc-js/src/load-balancer-outlier-detection.ts @@ -423,9 +423,10 @@ export class OutlierDetectionLoadBalancer implements LoadBalancer { const targetRequestVolume = successRateConfig.request_volume; let addresesWithTargetVolume = 0; const successRates: number[] = [] - for (const mapEntry of this.addressMap.values()) { + for (const [address, mapEntry] of this.addressMap) { const successes = mapEntry.counter.getLastSuccesses(); const failures = mapEntry.counter.getLastFailures(); + trace('Stats for ' + address + ': successes=' + successes + ' failures=' + failures + ' targetRequestVolume=' + targetRequestVolume); if (successes + failures >= targetRequestVolume) { addresesWithTargetVolume += 1; successRates.push(successes/(successes + failures)); @@ -545,6 +546,7 @@ export class OutlierDetectionLoadBalancer implements LoadBalancer { private startTimer(delayMs: number) { this.ejectionTimer = setTimeout(() => this.runChecks(), delayMs); + this.ejectionTimer.unref?.(); } private runChecks() { diff --git a/packages/grpc-js/test/test-outlier-detection.ts b/packages/grpc-js/test/test-outlier-detection.ts index 74e53676..c9021e60 100644 --- a/packages/grpc-js/test/test-outlier-detection.ts +++ b/packages/grpc-js/test/test-outlier-detection.ts @@ -20,6 +20,7 @@ import * as path from 'path'; import * as grpc from '../src'; import { loadProtoFile } from './common'; import { OutlierDetectionLoadBalancingConfig } from '../src/load-balancer-outlier-detection' +import { ServiceClient } from '../src/make-client'; function multiDone(done: Mocha.Done, target: number) { let count = 0; @@ -49,6 +50,54 @@ const defaultOutlierDetectionServiceConfig = { const defaultOutlierDetectionServiceConfigString = JSON.stringify(defaultOutlierDetectionServiceConfig); +const successRateOutlierDetectionServiceConfig = { + methodConfig: [], + loadBalancingConfig: [ + { + outlier_detection: { + interval: { + seconds: 1, + nanos: 0 + }, + base_ejection_time: { + seconds: 3, + nanos: 0 + }, + success_rate_ejection: { + request_volume: 5 + }, + child_policy: [{round_robin: {}}] + } + } + ] +}; + +const successRateOutlierDetectionServiceConfigString = JSON.stringify(successRateOutlierDetectionServiceConfig); + +const failurePercentageOutlierDetectionServiceConfig = { + methodConfig: [], + loadBalancingConfig: [ + { + outlier_detection: { + interval: { + seconds: 1, + nanos: 0 + }, + base_ejection_time: { + seconds: 3, + nanos: 0 + }, + failure_percentage_ejection: { + request_volume: 5 + }, + child_policy: [{round_robin: {}}] + } + } + ] +}; + +const falurePercentageOutlierDetectionServiceConfigString = JSON.stringify(failurePercentageOutlierDetectionServiceConfig); + const goodService = { echo: (call: grpc.ServerUnaryCall, callback: grpc.sendUnaryData) => { callback(null, call.request) @@ -353,6 +402,20 @@ describe('Outlier detection', () => { badServer.forceShutdown(); }); + function makeManyRequests(makeOneRequest: (callback: (error?: Error) => void) => void, total: number, callback: (error?: Error) => void) { + if (total === 0) { + callback(); + return; + } + makeOneRequest(error => { + if (error) { + callback(error); + return; + } + makeManyRequests(makeOneRequest, total - 1, callback); + }); + } + it('Should allow normal operation with one server', done => { const client = new EchoService(`localhost:${goodPorts[0]}`, grpc.credentials.createInsecure(), {'grpc.service_config': defaultOutlierDetectionServiceConfigString}); client.echo( @@ -364,4 +427,110 @@ describe('Outlier detection', () => { } ); }); + describe('Success rate', () => { + let makeCheckedRequest: (callback: () => void) => void; + let makeUncheckedRequest:(callback: (error?: Error) => void) => void; + before(() => { + const target = 'ipv4:///' + goodPorts.map(port => `127.0.0.1:${port}`).join(',') + `,127.0.0.1:${badPort}`; + const client = new EchoService(target, grpc.credentials.createInsecure(), {'grpc.service_config': successRateOutlierDetectionServiceConfigString}); + makeUncheckedRequest = (callback: () => void) => { + client.echo( + { value: 'test value', value2: 3 }, + (error: grpc.ServiceError, response: any) => { + callback(); + } + ); + }; + makeCheckedRequest = (callback: (error?: Error) => void) => { + client.echo( + { value: 'test value', value2: 3 }, + (error: grpc.ServiceError, response: any) => { + callback(error); + } + ); + }; + }); + it('Should eject a server if it is failing requests', done => { + // Make a large volume of requests + makeManyRequests(makeUncheckedRequest, 50, () => { + // Give outlier detection time to run ejection checks + setTimeout(() => { + // Make enough requests to go around all servers + makeManyRequests(makeCheckedRequest, 10, done); + }, 1000); + }); + }); + it('Should uneject a server after the ejection period', function(done) { + this.timeout(5000); + makeManyRequests(makeUncheckedRequest, 50, () => { + setTimeout(() => { + makeManyRequests(makeCheckedRequest, 10, error => { + if (error) { + done(error); + return; + } + setTimeout(() => { + makeManyRequests(makeCheckedRequest, 10, error => { + assert(error); + done(); + }); + }, 3000); + }); + }, 1000); + }) + }); + }); + describe('Failure percentage', () => { + let makeCheckedRequest: (callback: () => void) => void; + let makeUncheckedRequest:(callback: (error?: Error) => void) => void; + before(() => { + const target = 'ipv4:///' + goodPorts.map(port => `127.0.0.1:${port}`).join(',') + `,127.0.0.1:${badPort}`; + const client = new EchoService(target, grpc.credentials.createInsecure(), {'grpc.service_config': falurePercentageOutlierDetectionServiceConfigString}); + makeUncheckedRequest = (callback: () => void) => { + client.echo( + { value: 'test value', value2: 3 }, + (error: grpc.ServiceError, response: any) => { + callback(); + } + ); + }; + makeCheckedRequest = (callback: (error?: Error) => void) => { + client.echo( + { value: 'test value', value2: 3 }, + (error: grpc.ServiceError, response: any) => { + callback(error); + } + ); + }; + }); + it('Should eject a server if it is failing requests', done => { + // Make a large volume of requests + makeManyRequests(makeUncheckedRequest, 50, () => { + // Give outlier detection time to run ejection checks + setTimeout(() => { + // Make enough requests to go around all servers + makeManyRequests(makeCheckedRequest, 10, done); + }, 1000); + }); + }); + it('Should uneject a server after the ejection period', function(done) { + this.timeout(5000); + makeManyRequests(makeUncheckedRequest, 50, () => { + setTimeout(() => { + makeManyRequests(makeCheckedRequest, 10, error => { + if (error) { + done(error); + return; + } + setTimeout(() => { + makeManyRequests(makeCheckedRequest, 10, error => { + assert(error); + done(); + }); + }, 3000); + }); + }, 1000); + }) + }); + }); }); \ No newline at end of file