From 67b5ac31296d4558a26eb650a3111929527faa70 Mon Sep 17 00:00:00 2001 From: Jeff Hubbach Date: Wed, 3 Nov 2021 15:58:03 -0600 Subject: [PATCH] feat: Add Octokit Throttling to handle github rate limit/request throttling --- package-lock.json | 76 ++++++++++++++++++++++++++++++++----------- package.json | 3 +- src/github-helper.ts | 22 ++++++++++++- src/octokit-client.ts | 4 ++- 4 files changed, 83 insertions(+), 22 deletions(-) diff --git a/package-lock.json b/package-lock.json index fe50eef..01e2c55 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1090,9 +1090,9 @@ } }, "@octokit/auth-token": { - "version": "2.4.5", - "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.4.5.tgz", - "integrity": "sha512-BpGYsPgJt05M7/L/5FoE1PiAbdxXFZkX/3kDYcsvd1v6UhlnE5e96dTDr0ezX/EFwciQxf3cNV0loipsURU+WA==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.5.0.tgz", + "integrity": "sha512-r5FVUJCOLl19AxiuZD2VRZ/ORjp/4IN98Of6YJoJOkY75CIBuYfmiNHGrDwXr+aLGG55igl9QrxX3hbiXlLb+g==", "requires": { "@octokit/types": "^6.0.3" } @@ -1122,9 +1122,9 @@ } }, "@octokit/graphql": { - "version": "4.6.4", - "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.6.4.tgz", - "integrity": "sha512-SWTdXsVheRmlotWNjKzPOb6Js6tjSqA2a8z9+glDJng0Aqjzti8MEWOtuT8ZSu6wHnci7LZNuarE87+WJBG4vg==", + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.8.0.tgz", + "integrity": "sha512-0gv+qLSBLKF0z8TKaSKTsS39scVKF9dbMxJpj3U0vC7wjNWFuIpL/z76Qe2fiuCbDRcJSavkXsVtMS6/dtQQsg==", "requires": { "@octokit/request": "^5.6.0", "@octokit/types": "^6.0.3", @@ -1132,9 +1132,9 @@ } }, "@octokit/openapi-types": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-9.1.0.tgz", - "integrity": "sha512-XBP03pG4XuTU+VgeJM1ozRdmZJerMG4tk6wA+raFKycC4qV9jtD2UQroAg9bAcmI3Q0zWvifeDGtPqsFjMzkLg==" + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-11.2.0.tgz", + "integrity": "sha512-PBsVO+15KSlGmiI8QAzaqvsNlZlrDlyAJYcrXBCvVUxCp7VnXjkwPoFHgjEJXx3WF9BAwkA6nfCUA7i9sODzKA==" }, "@octokit/plugin-paginate-rest": { "version": "2.16.7", @@ -1183,10 +1183,19 @@ } } }, + "@octokit/plugin-throttling": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-3.5.2.tgz", + "integrity": "sha512-Eu7kfJxU8vmHqWGNszWpg+GVp2tnAfax3XQV5CkYPEE69C+KvInJXW9WajgSeW+cxYe0UVdouzCtcreGNuJo7A==", + "requires": { + "@octokit/types": "^6.0.1", + "bottleneck": "^2.15.3" + } + }, "@octokit/request": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.6.0.tgz", - "integrity": "sha512-4cPp/N+NqmaGQwbh3vUsYqokQIzt7VjsgTYVXiwpUP2pxd5YiZB2XuTedbb0SPtv9XS7nzAKjAuQxmY8/aZkiA==", + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.6.2.tgz", + "integrity": "sha512-je66CvSEVf0jCpRISxkUcCa0UkxmFs6eGDRSbfJtAVwbLH5ceqF+YEyC8lj8ystKyZTy8adWr0qmkY52EfOeLA==", "requires": { "@octokit/endpoint": "^6.0.1", "@octokit/request-error": "^2.1.0", @@ -1207,11 +1216,11 @@ } }, "@octokit/types": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.21.0.tgz", - "integrity": "sha512-VPSxn9uhCoOUMpxCsOAQhf8DgIx+uzFjZRYDiZS5+TvrKaEwBrWkjr/5NmUVvPbW6xdPC2n3yL3XCnoxa4rxvg==", + "version": "6.34.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.34.0.tgz", + "integrity": "sha512-s1zLBjWhdEI2zwaoSgyOFoKSl109CUcVBCc7biPJ3aAf6LGLU6szDvi31JPU7bxfla2lqfhjbbg/5DdFNxOwHw==", "requires": { - "@octokit/openapi-types": "^9.1.0" + "@octokit/openapi-types": "^11.2.0" } }, "@sinonjs/commons": { @@ -1750,6 +1759,11 @@ "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.2.tgz", "integrity": "sha512-3pZEU3NT5BFUo/AD5ERPWOgQOCZITni6iavr5AUw5AUwQjMlI0kzu5btnyD39AF0gUEsDPwJT+oY1ORBJijPjQ==" }, + "bottleneck": { + "version": "2.19.5", + "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", + "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -4561,9 +4575,33 @@ "dev": true }, "node-fetch": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", - "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + "version": "2.6.6", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", + "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", + "requires": { + "whatwg-url": "^5.0.0" + }, + "dependencies": { + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" + }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=" + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + } + } }, "node-int64": { "version": "0.4.0", diff --git a/package.json b/package.json index 0d0f1cc..4e07882 100644 --- a/package.json +++ b/package.json @@ -31,9 +31,10 @@ "dependencies": { "@actions/core": "1.6.0", "@actions/exec": "1.1.0", - "@octokit/core": "3.5.1", + "@octokit/core": "^3.5.1", "@octokit/plugin-paginate-rest": "2.16.7", "@octokit/plugin-rest-endpoint-methods": "5.11.4", + "@octokit/plugin-throttling": "3.5.2", "https-proxy-agent": "5.0.0", "uuid": "8.3.2" }, diff --git a/src/github-helper.ts b/src/github-helper.ts index d0f9a6f..08e47f1 100644 --- a/src/github-helper.ts +++ b/src/github-helper.ts @@ -20,7 +20,27 @@ export class GitHubHelper { private octokit: InstanceType constructor(token: string) { - const options: OctokitOptions = {} + const options: OctokitOptions = { + throttle: { + onRateLimit: (retryAfter, options, octokit) => { + octokit.log.warn( + `Request quota exhausted for request ${options.method} ${options.url}` + ) + + if (options.request.retryCount === 0) { + // only retries once + octokit.log.info(`Retrying after ${retryAfter} seconds!`) + return true + } + }, + onAbuseLimit: (retryAfter, options, octokit) => { + // does not retry, only logs a warning + octokit.log.warn( + `Abuse detected for request ${options.method} ${options.url}` + ) + } + } + } if (token) { options.auth = `${token}` } diff --git a/src/octokit-client.ts b/src/octokit-client.ts index adf32f0..b9bffe6 100644 --- a/src/octokit-client.ts +++ b/src/octokit-client.ts @@ -1,4 +1,5 @@ import {Octokit as Core} from '@octokit/core' +import {throttling} from '@octokit/plugin-throttling' import {paginateRest} from '@octokit/plugin-paginate-rest' import {restEndpointMethods} from '@octokit/plugin-rest-endpoint-methods' import {HttpsProxyAgent} from 'https-proxy-agent' @@ -8,7 +9,8 @@ export {OctokitOptions} from '@octokit/core/dist-types/types' export const Octokit = Core.plugin( paginateRest, restEndpointMethods, - autoProxyAgent + autoProxyAgent, + throttling ) // Octokit plugin to support the https_proxy environment variable