Fix 502 errors with retry logic and fault-tolerant batch processing

README.md

@@ -4,6 +4,13 @@
 
 If you are considering deleting a GitHub previous work-account, sunsetting an underutilized account, or simply desiring a comprehensive backup of your GitHub contributions – be it Gists, starred Gists, forked Gists, starred Repositories – you will find here scripts to archive your GitHub data using GitHub's API, enabling you to create a CSV list of your contributions or download the files themselves.
 
+## Recent Improvements
+
+- **🔄 Automatic Retry Logic**: All API requests now automatically retry up to 3 times with exponential backoff when encountering transient errors (502, 503, 429, etc.)
+- **🛡️ Better Error Handling**: Individual gist fetch failures no longer stop the entire download process - the script continues with remaining gists
+- **🔒 Security Updates**: Updated axios from 1.6.8 to 1.13.4, fixing multiple security vulnerabilities
+- **📊 Enhanced Logging**: Better error messages including HTTP status codes for easier troubleshooting
+
 ## Features
 
 - Download all your Gists

lib/axiosWithRetry.js

@@ -0,0 +1,32 @@
+const axios = require('axios');
+
+/**
+ * Makes an axios request with retry logic for transient errors
+ * @param {Object} config - Axios request configuration
+ * @param {number} maxRetries - Maximum number of retries (default: 3)
+ * @param {number} initialDelay - Initial delay in ms before first retry (default: 1000)
+ * @returns {Promise<Object>} - Axios response
+ */
+async function axiosWithRetry(config, maxRetries = 3, initialDelay = 1000) {
+  let lastError;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      return await axios(config);
+    } catch (error) {
+      lastError = error;
+      // Don't retry on 4xx errors (client errors) except 429 (rate limit)
+      if (error.response && error.response.status >= 400 && error.response.status < 500 && error.response.status !== 429) {
+        throw error;
+      }
+      // Retry on 5xx errors, network errors, and 429 (rate limit)
+      if (attempt < maxRetries) {
+        const delay = initialDelay * Math.pow(2, attempt); // Exponential backoff
+        console.log(`Request failed with ${error.message}, retrying in ${delay}ms (attempt ${attempt + 1}/${maxRetries})...`);
+        await new Promise(resolve => setTimeout(resolve, delay));
+      }
+    }
+  }
+  throw lastError;
+}
+
+module.exports = axiosWithRetry;

lib/gists.js

@@ -11,6 +11,7 @@ const {
   LIST_CSV_WITH_FILES_FILENAME,
 } = require('./dirFileNameConstants');
 const getNextLinkFromHeaders = require('./getNextLinkFromHeaders');
+const axiosWithRetry = require('./axiosWithRetry');
 
 /**
  * Runs once per page of gists listed in the response from the GitHub API
@@ -95,7 +96,7 @@ async function processGists(
 
   //get gists
   try {
-    const response = await axios({
+    const response = await axiosWithRetry({
       method: 'get',
       url: next ? next : 'https://api.github.com/gists',
       headers: {
@@ -105,18 +106,30 @@ async function processGists(
       },
     });
     //get details of each gist
-    await Promise.all(
+    const gistResults = await Promise.allSettled(
       response.data.map(async (gistFromList) => {
-        const gistDetails = await axios({
-          method: 'get',
-          url: gistFromList.url,
-          headers: {
-            Accept: 'application/vnd.github+json',
-            Authorization: `Bearer ${process.env.GITHUB_TOKEN}`,
-            'X-GitHub-Api-Version': '2022-11-28',
-          },
-        });
-        const gist = gistDetails.data;
+        try {
+          const gistDetails = await axiosWithRetry({
+            method: 'get',
+            url: gistFromList.url,
+            headers: {
+              Accept: 'application/vnd.github+json',
+              Authorization: `Bearer ${process.env.GITHUB_TOKEN}`,
+              'X-GitHub-Api-Version': '2022-11-28',
+            },
+          });
+          return { success: true, data: gistDetails.data };
+        } catch (error) {
+          console.error(`Failed to fetch gist ${gistFromList.id}: ${error.response?.status || 'network error'} - ${error.message}`);
+          return { success: false, id: gistFromList.id, error: error.message };
+        }
+      })
+    );
+
+    // Process successful gist fetches
+    for (const result of gistResults) {
+      if (result.status === 'fulfilled' && result.value.success) {
+        const gist = result.value.data;
 
         const owner_login = gist.owner.login;
 
@@ -224,8 +237,8 @@ async function processGists(
             fs.writeFileSync(filePath, file.content);
           });
         }
-      })
-    );
+      }
+    }
 
     const nextUrl = getNextLinkFromHeaders(response.headers);
 

lib/starredGists.js

@@ -11,6 +11,7 @@ const createCsvWriter = require('csv-writer').createObjectCsvWriter;
 const safeDirName = require('./safeDirName');
 const getGitHubUsername = require('./getGitHubUsername');
 const getNextLinkFromHeaders = require('./getNextLinkFromHeaders');
+const axiosWithRetry = require('./axiosWithRetry');
 
 /**
  * Runs once per page of starred gists listed in the response from the GitHub API
@@ -92,7 +93,7 @@ async function processStarredGists(
 
   //get gists
   try {
-    const response = await axios({
+    const response = await axiosWithRetry({
       method: 'get',
       url: next ? next : 'https://api.github.com/gists/starred',
       headers: {
@@ -103,22 +104,34 @@ async function processStarredGists(
     });
 
     //get details of each gist
-    await Promise.all(
+    const gistResults = await Promise.allSettled(
       response.data.map(async (gistFromList) => {
-        const gistDetails = await axios({
-          method: 'get',
-          url: gistFromList.url,
-          headers: {
-            Accept: 'application/vnd.github+json',
-            Authorization: `Bearer ${process.env.GITHUB_TOKEN}`,
-            'X-GitHub-Api-Version': '2022-11-28',
-          },
-        });
-        const gist = gistDetails.data;
+        try {
+          const gistDetails = await axiosWithRetry({
+            method: 'get',
+            url: gistFromList.url,
+            headers: {
+              Accept: 'application/vnd.github+json',
+              Authorization: `Bearer ${process.env.GITHUB_TOKEN}`,
+              'X-GitHub-Api-Version': '2022-11-28',
+            },
+          });
+          return { success: true, data: gistDetails.data };
+        } catch (error) {
+          console.error(`Failed to fetch gist ${gistFromList.id}: ${error.response?.status || 'network error'} - ${error.message}`);
+          return { success: false, id: gistFromList.id, error: error.message };
+        }
+      })
+    );
+
+    // Process successful gist fetches
+    for (const result of gistResults) {
+      if (result.status === 'fulfilled' && result.value.success) {
+        const gist = result.value.data;
         const owner_login = gist.owner.login;
 
         if (ignoringOwner === owner_login) {
-          return;
+          continue;
         }
 
         const id = gist.id;
@@ -213,8 +226,8 @@ async function processStarredGists(
             fs.writeFileSync(filePath, file.content);
           });
         }
-      })
-    );
+      }
+    }
 
     const nextUrl = getNextLinkFromHeaders(response.headers);
 

lib/starredRepositories.js

@@ -6,6 +6,7 @@ const { LIST_CSV_STARRED_REPOS_FILENAME } = require('./dirFileNameConstants');
 const getNextLinkFromHeaders = require('./getNextLinkFromHeaders');
 const getSourceRepoOfFork = require('./getSourceRepoOfFork');
 const createCsvWriter = require('csv-writer').createObjectCsvWriter;
+const axiosWithRetry = require('./axiosWithRetry');
 /**
  * Runs once per page of starred repositories listed in the response from the GitHub API
  * @param {string} next
@@ -18,15 +19,14 @@ async function processStarredRepositories(
   csvWriter,
   { parentDir = 'downloaded-gists' }
 ) {
-  const response = await axios.get(
-    next ? next : 'https://api.github.com/user/starred',
-    {
-      headers: {
-        Authorization: `token ${process.env.GITHUB_TOKEN}`,
-        Accept: 'application/vnd.github.star+json',
-      },
-    }
-  );
+  const response = await axiosWithRetry({
+    method: 'get',
+    url: next ? next : 'https://api.github.com/user/starred',
+    headers: {
+      Authorization: `token ${process.env.GITHUB_TOKEN}`,
+      Accept: 'application/vnd.github.star+json',
+    },
+  });
   //create parentDir if it doesn't exist
   if (!fs.existsSync(parentDir)) {
     fs.mkdirSync(parentDir);