adding files

2021-03-02 10:13:03 -06:00 · 2021-03-02 10:13:03 -06:00 · e6f70bf7ec
--- a/.gitignore
+++ b/.gitignore
@ -1,5 +1,6 @@
 .ghec-audit-log
 .last-cursor-update
+.last-v3-cursor-update

 # Created by https://www.gitignore.io/api/node
 # Edit at https://www.gitignore.io/?templates=node
@ -110,4 +111,4 @@ dist/
 tmp/
 temp/

-# End of https://www.gitignore.io/api/node
+# End of https://www.gitignore.io/api/node
--- a/ghec-audit-log-cli.js
+++ b/ghec-audit-log-cli.js
@ -2,7 +2,9 @@
 const YAML = require('yaml')
 const fs = require('fs')
 const { graphql } = require('@octokit/graphql')
-const { octokit } = require('@octokit/rest')
+const { Octokit } = require('@octokit/rest')
+const { retry } = require('@octokit/plugin-retry')
+const { throttling } = require('@octokit/plugin-throttling')
 const { requestEntries } = require('./ghec-audit-log-client')
 const { validateInput } = require('./ghec-audit-log-utils')

@ -32,27 +34,22 @@ try {
 const { cursor, pretty, limit, api, token, org, outputFile } = validateInput(program, config)

 /**
- * Function containing all the queries
+ * Function containing the GitHub API v4 Graphql calls for the audit log
 */
 async function queryAuditLog () {
+  const graphqlWithAuth = graphql.defaults({
+    headers: {
+      authorization: `token ${token}`
+    }
+  })
  // Select the query to run
  let queryRunner
  if (cursor) {
-    if (api === 'v4') {
-      // API v4 call with cursor
-      queryRunner = () => requestEntries(graphqlWithAuth, org, limit, cursor)
-    } else {
-      // API v3 call with cursor
-      queryRunner = () => requestEntries(v3WithAuth, org, limit, cursor)
-    }
+    // API v4 call with cursor
+    queryRunner = () => requestEntries(graphqlWithAuth, org, limit, cursor)
  } else {
-    if (api === 'v4') {
-      // api v4 without cursor
-      queryRunner = () => requestEntries(graphqlWithAuth, org, limit)
-    } else {
-      // API v3 without cursor
-      queryRunner = () => requestEntries(v3WithAuth, org, limit)
-    }
+    // api v4 without cursor
+    queryRunner = () => requestEntries(graphqlWithAuth, org, limit)
  }

  // Run the query and store the most recent cursor
@ -68,31 +65,51 @@ async function queryAuditLog () {
  }
 }

-// Execute the request and print the result
-const v3WithAuth = octokit.defaults({
-  headers: {
-    authorization: `token ${token}`
-  }
-})
-queryAuditLog()
-  .then((data) => {
-    if (outputFile) {
-      fs.writeFileSync(outputFile, data)
-    } else {
-      console.log(data)
+/*
+* Function containing the GitHub v3 query for the audit-log
+*/
+async function apiV3Call (outputFile) {
+  const Octo = Octokit.plugin(retry, throttling)
+  const octokit = new Octo({
+    auth: token,
+    throttle: {
+      onRateLimit: (retryAfter, options) => {
+        octokit.log.warn(
+          `[${new Date().toISOString()}] ${program} Request quota exhausted for request, will retry in ${retryAfter}`
+        )
+        return true
+      },
+      onAbuseLimit: (retryAfter, options) => {
+        octokit.log.warn(
+          `[${new Date().toISOString()}] ${program} Abuse detected for request, will retry in ${retryAfter}`
+        )
+        return true
+      }
    }
  })
-  .catch((err) => {
-    console.error(err)
-    process.exit(1)
-  })
+  try {
+      for await (const response of octokit.paginate.iterator(`GET /orgs/{org}/audit-log?include=all&per_page=5&after=${cursor}`, {
+        org: org
+      })) {
+        console.log("link:", response.headers.link)
+        if (response.status === 200 || response.status === 201) {
+            fs.appendFileSync(outputFile, JSON.stringify(response.data), function (err) {
+              if (err) return console.log(err)
+            })
+          }
+        }
+      } catch (error) {
+        console.log(error)
+        process.exit(1)
+    }
+  }

+/*
+* Logic to see if we need to run the API v3 vs API v4
+*/
+if (api === 'v4') {
 // Execute the request and print the result
-const graphqlWithAuth = graphql.defaults({
-  headers: {
-    authorization: `token ${token}`
-  }
-})
+
 queryAuditLog()
  .then((data) => {
    if (outputFile) {
@ -105,3 +122,7 @@ queryAuditLog()
    console.error(err)
    process.exit(1)
  })
+} else {
+  // API v3 call
+  apiV3Call(outputFile)
+}
--- a/output.txt
+++ b/output.txt
--- a/package-lock.json
+++ b/package-lock.json
@ -9,6 +9,8 @@
      "license": "MIT",
      "dependencies": {
        "@octokit/graphql": "^4.3.1",
+        "@octokit/plugin-retry": "*",
+        "@octokit/plugin-throttling": "^3.3.1",
        "@octokit/rest": "^18.3.1",
        "commander": "^5.1.0",
        "validate.js": "^0.13.1",
@ -215,6 +217,43 @@
        "@octokit/openapi-types": "^5.2.1"
      }
    },
+    "node_modules/@octokit/plugin-retry": {
+      "version": "3.0.7",
+      "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-3.0.7.tgz",
+      "integrity": "sha512-n08BPfVeKj5wnyH7IaOWnuKbx+e9rSJkhDHMJWXLPv61625uWjsN8G7sAW3zWm9n9vnS4friE7LL/XLcyGeG8Q==",
+      "dependencies": {
+        "@octokit/types": "^6.0.3",
+        "bottleneck": "^2.15.3"
+      }
+    },
+    "node_modules/@octokit/plugin-retry/node_modules/@octokit/types": {
+      "version": "6.11.1",
+      "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.11.1.tgz",
+      "integrity": "sha512-UiSRTG2lrFbMUMwhKNR0uSV33Fzv4bNu1n5iFuuNOg80XCh0VYNhR4TQWgrkLhVxdDuej6s61aP3eQvVm6K6uA==",
+      "dependencies": {
+        "@octokit/openapi-types": "^5.2.1"
+      }
+    },
+    "node_modules/@octokit/plugin-throttling": {
+      "version": "3.4.1",
+      "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-3.4.1.tgz",
+      "integrity": "sha512-qCQ+Z4AnL9OrXvV59EH3GzPxsB+WyqufoCjiCJXJxTbnt3W+leXbXw5vHrMp4NG9ltw00McFWIxIxNQAzLNoTA==",
+      "dependencies": {
+        "@octokit/types": "^6.0.1",
+        "bottleneck": "^2.15.3"
+      },
+      "peerDependencies": {
+        "@octokit/core": "^3.0.0"
+      }
+    },
+    "node_modules/@octokit/plugin-throttling/node_modules/@octokit/types": {
+      "version": "6.11.1",
+      "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.11.1.tgz",
+      "integrity": "sha512-UiSRTG2lrFbMUMwhKNR0uSV33Fzv4bNu1n5iFuuNOg80XCh0VYNhR4TQWgrkLhVxdDuej6s61aP3eQvVm6K6uA==",
+      "dependencies": {
+        "@octokit/openapi-types": "^5.2.1"
+      }
+    },
    "node_modules/@octokit/request": {
      "version": "5.4.14",
      "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.4.14.tgz",
@ -420,6 +459,11 @@
      "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.1.1.tgz",
      "integrity": "sha512-5ekuQOvO04MDj7kYZJaMab2S8SPjGJbotVNyv7QYFCOAwrGZs/YnoDNlh1U+m5hl7H2D/+n0taaAV/tfyd3KMA=="
    },
+    "node_modules/bottleneck": {
+      "version": "2.19.5",
+      "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz",
+      "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw=="
+    },
    "node_modules/brace-expansion": {
      "version": "1.1.11",
      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@ -2961,6 +3005,44 @@
        }
      }
    },
+    "@octokit/plugin-retry": {
+      "version": "3.0.7",
+      "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-3.0.7.tgz",
+      "integrity": "sha512-n08BPfVeKj5wnyH7IaOWnuKbx+e9rSJkhDHMJWXLPv61625uWjsN8G7sAW3zWm9n9vnS4friE7LL/XLcyGeG8Q==",
+      "requires": {
+        "@octokit/types": "^6.0.3",
+        "bottleneck": "^2.15.3"
+      },
+      "dependencies": {
+        "@octokit/types": {
+          "version": "6.11.1",
+          "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.11.1.tgz",
+          "integrity": "sha512-UiSRTG2lrFbMUMwhKNR0uSV33Fzv4bNu1n5iFuuNOg80XCh0VYNhR4TQWgrkLhVxdDuej6s61aP3eQvVm6K6uA==",
+          "requires": {
+            "@octokit/openapi-types": "^5.2.1"
+          }
+        }
+      }
+    },
+    "@octokit/plugin-throttling": {
+      "version": "3.4.1",
+      "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-3.4.1.tgz",
+      "integrity": "sha512-qCQ+Z4AnL9OrXvV59EH3GzPxsB+WyqufoCjiCJXJxTbnt3W+leXbXw5vHrMp4NG9ltw00McFWIxIxNQAzLNoTA==",
+      "requires": {
+        "@octokit/types": "^6.0.1",
+        "bottleneck": "^2.15.3"
+      },
+      "dependencies": {
+        "@octokit/types": {
+          "version": "6.11.1",
+          "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.11.1.tgz",
+          "integrity": "sha512-UiSRTG2lrFbMUMwhKNR0uSV33Fzv4bNu1n5iFuuNOg80XCh0VYNhR4TQWgrkLhVxdDuej6s61aP3eQvVm6K6uA==",
+          "requires": {
+            "@octokit/openapi-types": "^5.2.1"
+          }
+        }
+      }
+    },
    "@octokit/request": {
      "version": "5.4.14",
      "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.4.14.tgz",
@ -3138,6 +3220,11 @@
      "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.1.1.tgz",
      "integrity": "sha512-5ekuQOvO04MDj7kYZJaMab2S8SPjGJbotVNyv7QYFCOAwrGZs/YnoDNlh1U+m5hl7H2D/+n0taaAV/tfyd3KMA=="
    },
+    "bottleneck": {
+      "version": "2.19.5",
+      "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz",
+      "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw=="
+    },
    "brace-expansion": {
      "version": "1.1.11",
      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
--- a/package.json
+++ b/package.json
@ -27,6 +27,8 @@
  "license": "MIT",
  "dependencies": {
    "@octokit/graphql": "^4.3.1",
+    "@octokit/plugin-retry": "*",
+    "@octokit/plugin-throttling": "^3.3.1",
    "@octokit/rest": "^18.3.1",
    "commander": "^5.1.0",
    "validate.js": "^0.13.1",