diff --git a/src/infrastructure/RequestHelper.js b/src/infrastructure/RequestHelper.js index e5f021c7..1cb5f576 100644 --- a/src/infrastructure/RequestHelper.js +++ b/src/infrastructure/RequestHelper.js @@ -54,7 +54,11 @@ function getStream(service, endpoint, options = {}) { return StreamableRequest.get(requestOptions); } -async function getPaginated(service, endpoint, options = {}) { +async function wait(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function getPaginated(service, endpoint, options = {}, sleepOnRateLimit = true) { const { showPagination, maxPages, ...queryOptions } = options; const requestOptions = defaultRequest(service, endpoint, { headers: service.headers, @@ -62,37 +66,47 @@ async function getPaginated(service, endpoint, options = {}) { resolveWithFullResponse: true, }); - const response = await service.requester.get(requestOptions); - const links = LinkParser(response.headers.link) || {}; - const page = response.headers['x-page']; - const underMaxPageLimit = maxPages ? page < maxPages : true; - let more = []; - let data; + try { + const response = await service.requester.get(requestOptions); + const links = LinkParser(response.headers.link) || {}; + const page = response.headers['x-page']; + const underMaxPageLimit = maxPages ? page < maxPages : true; + let more = []; + let data; - // If not looking for a singular page and still under the max pages limit - // AND their is a next page, paginate - if (!queryOptions.page && underMaxPageLimit && links.next) { - more = await getPaginated(service, links.next.url.replace(service.url, ''), options); - data = [...response.body, ...more]; - } else { - data = response.body; + // If not looking for a singular page and still under the max pages limit + // AND their is a next page, paginate + if (!queryOptions.page && underMaxPageLimit && links.next) { + more = await getPaginated(service, links.next.url.replace(service.url, ''), options); + data = [...response.body, ...more]; + } else { + data = response.body; + } + + if (queryOptions.page && showPagination) { + return { + data, + pagination: { + total: response.headers['x-total'], + next: response.headers['x-next-page'] || null, + current: response.headers['x-page'] || null, + previous: response.headers['x-prev-page'] || null, + perPage: response.headers['x-per-page'], + totalPages: response.headers['x-total-pages'], + }, + }; + } + + return data; + } catch (err) { + const sleepTime = parseInt(err.response.headers['retry-after'], 10); + if (sleepOnRateLimit && parseInt(err.statusCode, 10) === 429 + && sleepTime) { + await wait(sleepTime * 1000); + return getPaginated(service, endpoint, options, sleepOnRateLimit); + } + throw err; } - - if (queryOptions.page && showPagination) { - return { - data, - pagination: { - total: response.headers['x-total'], - next: response.headers['x-next-page'] || null, - current: response.headers['x-page'] || null, - previous: response.headers['x-prev-page'] || null, - perPage: response.headers['x-per-page'], - totalPages: response.headers['x-total-pages'], - }, - }; - } - - return data; } class RequestHelper { diff --git a/src/infrastructure/XMLHttpRequester.js b/src/infrastructure/XMLHttpRequester.js index 83253aee..00a3b7d4 100644 --- a/src/infrastructure/XMLHttpRequester.js +++ b/src/infrastructure/XMLHttpRequester.js @@ -1,14 +1,17 @@ import { StatusCodeError } from 'request-promise-core/errors'; import Promisify from 'util.promisify'; import XHR from 'xhr'; +import wait from './RequestHelper'; function promisifyFn(fn) { const promisifiedFn = Promisify(fn); - return async function (opts) { + return async function getResponse(opts) { const response = await promisifiedFn(opts); - - if (response.statusCode >= 400 && response.statusCode <= 599) { + const sleepTime = parseInt(response.headers['retry-after'], 10); + if (response.statusCode === 429 && sleepTime) { + await wait(sleepTime * 1000); + } else if (response.statusCode >= 400 && response.statusCode <= 599) { throw new StatusCodeError(response.statusCode, response.body, {}, null); }