archive-gh-me/gitbeaker
1
0
Fork 0
mirror of https://github.com/jdalrymple/gitbeaker.git synced 2026-01-18 15:55:30 +00:00
Code Issues Projects Releases Wiki Activity

feat: obey the rate limit

Browse Source 
Wait as long as specified in the retry-after header, send out by GitLab

Fixes #73
This commit is contained in:
Max Wittig 2018-07-03 13:27:49 +02:00
parent 4458beae2e
commit 9b46250619
2 changed files with 50 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
src/infrastructure/RequestHelper.js
View File

@ -54,7 +54,11 @@ function getStream(service, endpoint, options = {}) {
return StreamableRequest.get(requestOptions);
}
async function getPaginated(service, endpoint, options = {}) {
async function wait(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
async function getPaginated(service, endpoint, options = {}, sleepOnRateLimit = true) {
const { showPagination, maxPages, ...queryOptions } = options;
const requestOptions = defaultRequest(service, endpoint, {
headers: service.headers,
@ -62,37 +66,47 @@ async function getPaginated(service, endpoint, options = {}) {
resolveWithFullResponse: true,
});
const response = await service.requester.get(requestOptions);
const links = LinkParser(response.headers.link) || {};
const page = response.headers['x-page'];
const underMaxPageLimit = maxPages ? page < maxPages : true;
let more = [];
let data;
try {
const response = await service.requester.get(requestOptions);
const links = LinkParser(response.headers.link) || {};
const page = response.headers['x-page'];
const underMaxPageLimit = maxPages ? page < maxPages : true;
let more = [];
let data;
// If not looking for a singular page and still under the max pages limit
// AND their is a next page, paginate
if (!queryOptions.page && underMaxPageLimit && links.next) {
more = await getPaginated(service, links.next.url.replace(service.url, ''), options);
data = [...response.body, ...more];
} else {
data = response.body;
// If not looking for a singular page and still under the max pages limit
// AND their is a next page, paginate
if (!queryOptions.page && underMaxPageLimit && links.next) {
more = await getPaginated(service, links.next.url.replace(service.url, ''), options);
data = [...response.body, ...more];
} else {
data = response.body;
}
if (queryOptions.page && showPagination) {
return {
data,
pagination: {
total: response.headers['x-total'],
next: response.headers['x-next-page'] || null,
current: response.headers['x-page'] || null,
previous: response.headers['x-prev-page'] || null,
perPage: response.headers['x-per-page'],
totalPages: response.headers['x-total-pages'],
},
};
}
return data;
} catch (err) {
const sleepTime = parseInt(err.response.headers['retry-after'], 10);
if (sleepOnRateLimit && parseInt(err.statusCode, 10) === 429
&& sleepTime) {
await wait(sleepTime * 1000);
return getPaginated(service, endpoint, options, sleepOnRateLimit);
}
throw err;
}
if (queryOptions.page && showPagination) {
return {
data,
pagination: {
total: response.headers['x-total'],
next: response.headers['x-next-page'] || null,
current: response.headers['x-page'] || null,
previous: response.headers['x-prev-page'] || null,
perPage: response.headers['x-per-page'],
totalPages: response.headers['x-total-pages'],
},
};
}
return data;
}
class RequestHelper {

9
src/infrastructure/XMLHttpRequester.js
View File

@ -1,14 +1,17 @@
import { StatusCodeError } from 'request-promise-core/errors';
import Promisify from 'util.promisify';
import XHR from 'xhr';
import wait from './RequestHelper';
function promisifyFn(fn) {
const promisifiedFn = Promisify(fn);
return async function (opts) {
return async function getResponse(opts) {
const response = await promisifiedFn(opts);
if (response.statusCode >= 400 && response.statusCode <= 599) {
const sleepTime = parseInt(response.headers['retry-after'], 10);
if (response.statusCode === 429 && sleepTime) {
await wait(sleepTime * 1000);
} else if (response.statusCode >= 400 && response.statusCode <= 599) {
throw new StatusCodeError(response.statusCode, response.body, {}, null);
}