Merge pull request #141 from vietthedev/upstream

mrrfv · web-flow · commit c85eca4799f8 · 2024-11-29T22:15:25.000+01:00
Refactor fetch with retry and rate limiting handler, remove `FAST_MODE`
diff --git a/.env.example b/.env.example
@@ -2,7 +2,6 @@ CLOUDFLARE_API_TOKEN=
 CLOUDFLARE_ACCOUNT_ID=
 CLOUDFLARE_LIST_ITEM_LIMIT=300000
 DRY_RUN=0
-FAST_MODE=0
 BLOCK_PAGE_ENABLED=0
 
 # Experimental SNI based filtering. Set to 1 to enable.
diff --git a/README.md b/README.md
@@ -58,7 +58,6 @@ Please note that the GitHub Action downloads the recommended blocklists and whit
    - `PING_URL`: /Optional/ The HTTP(S) URL to ping (using curl) after the GitHub Action has successfully updated your filters. Useful for monitoring.
    - `DISCORD_WEBHOOK_URL`: /Optional/ The Discord (or similar) webhook URL to send notifications to. Good for monitoring as well.
 3. Create the following GitHub Actions variables in your repository settings if you desire:
-   - `FAST_MODE`: Enable the scripts to send the requests simultaneously. Beware that there's a rate limit of 1200 requests per five minutes (<https://developers.cloudflare.com/fundamentals/api/reference/limits/>) so make sure you know what you are doing.
    - `ALLOWLIST_URLS`: Uses your own allowlists. One URL per line. Recommended allowlists will be used if this variable is not provided.
    - `BLOCKLIST_URLS`: Uses your own blocklists. One URL per line. Recommended blocklists will be used if this variable is not provided.
    - `BLOCK_PAGE_ENABLED`: Enable showing block page if host is blocked.
diff --git a/auto_update_github_action.yml b/auto_update_github_action.yml
@@ -50,7 +50,6 @@ jobs:
           CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
           CLOUDFLARE_LIST_ITEM_LIMIT: ${{ secrets.CLOUDFLARE_LIST_ITEM_LIMIT }}
           DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }}
-          FAST_MODE: ${{ vars.FAST_MODE }}
 
       - name: Create new rules and lists
         run: npm run cloudflare-create
@@ -60,7 +59,6 @@ jobs:
           CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
           CLOUDFLARE_LIST_ITEM_LIMIT: ${{ secrets.CLOUDFLARE_LIST_ITEM_LIMIT }}
           DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }}
-          FAST_MODE: ${{ vars.FAST_MODE }}
 
       - name: Send ping request
         if: env.PING_URL != ''
diff --git a/cf_gateway_rule_create.js b/cf_gateway_rule_create.js
@@ -1,6 +1,6 @@
 import { getZeroTrustLists, upsertZeroTrustRule } from "./lib/api.js";
 import { BLOCK_BASED_ON_SNI } from "./lib/constants.js";
-import { notifyWebhook } from "./lib/helpers.js";
+import { notifyWebhook } from "./lib/utils.js";
 
 const { result: lists } = await getZeroTrustLists();
 
@@ -20,10 +20,10 @@ await upsertZeroTrustRule(wirefilterDNSExpression.slice(0, -4), "CGPS Filter Lis
 if (BLOCK_BASED_ON_SNI) {
   const wirefilterSNIExpression = lists.reduce((previous, current) => {
     if (!current.name.startsWith("CGPS List")) return previous;
-  
+
     return `${previous} any(net.sni.domains[*] in \$${current.id}) or `;
   }, "");
-  
+
   console.log("Creating SNI rule...");
   // .slice removes the trailing ' or '
   await upsertZeroTrustRule(wirefilterSNIExpression.slice(0, -4), "CGPS Filter Lists - SNI Based Filtering", ["l4"]);
diff --git a/cf_gateway_rule_delete.js b/cf_gateway_rule_delete.js
@@ -1,5 +1,5 @@
 import { deleteZeroTrustRule, getZeroTrustRules } from "./lib/api.js";
-import { notifyWebhook } from "./lib/helpers.js";
+import { notifyWebhook } from "./lib/utils.js";
 
 const { result: rules } = await getZeroTrustRules();
 const cgpsRules = rules.filter(({ name }) => name.startsWith("CGPS Filter Lists"));
diff --git a/cf_list_create.js b/cf_list_create.js
@@ -1,24 +1,21 @@
 import { existsSync } from "node:fs";
 import { resolve } from "node:path";
 
-import {
-  createZeroTrustListsAtOnce,
-  createZeroTrustListsOneByOne,
-} from "./lib/api.js";
+import { createZeroTrustListsOneByOne } from "./lib/api.js";
 import {
   DEBUG,
   DRY_RUN,
-  FAST_MODE,
   LIST_ITEM_LIMIT,
   LIST_ITEM_SIZE,
   PROCESSING_FILENAME,
 } from "./lib/constants.js";
-import { normalizeDomain, notifyWebhook } from "./lib/helpers.js";
+import { normalizeDomain } from "./lib/helpers.js";
 import {
   extractDomain,
   isComment,
   isValidDomain,
   memoize,
+  notifyWebhook,
   readFile,
 } from "./lib/utils.js";
 
@@ -135,12 +132,7 @@ console.log("\n\n");
     `Creating ${numberOfLists} lists for ${domains.length} domains...`
   );
 
-  if (FAST_MODE) {
-    await createZeroTrustListsAtOnce(domains);
-  } else {
-    await createZeroTrustListsOneByOne(domains);
-  }
-
+  await createZeroTrustListsOneByOne(domains);
   await notifyWebhook(
     `CF List Create script finished running (${domains.length} domains, ${numberOfLists} lists)`
   );
diff --git a/cf_list_delete.js b/cf_list_delete.js
@@ -1,10 +1,8 @@
 import {
-  deleteZeroTrustListsAtOnce,
   deleteZeroTrustListsOneByOne,
   getZeroTrustLists,
 } from "./lib/api.js";
-import { FAST_MODE } from "./lib/constants.js";
-import { notifyWebhook } from "./lib/helpers.js";
+import { notifyWebhook } from "./lib/utils.js";
 
 (async () => {
   const { result: lists } = await getZeroTrustLists();
@@ -31,14 +29,6 @@ import { notifyWebhook } from "./lib/helpers.js";
 
   console.log(`Deleting ${cgpsLists.length} lists...`);
 
-  if (FAST_MODE) {
-    await deleteZeroTrustListsAtOnce(cgpsLists);
-    // TODO: make this less repetitive
-    await notifyWebhook(`CF List Delete script finished running (${cgpsLists.length} lists)`);
-    return;
-  }
-
   await deleteZeroTrustListsOneByOne(cgpsLists);
-
   await notifyWebhook(`CF List Delete script finished running (${cgpsLists.length} lists)`);
 })();
diff --git a/extended_guide.md b/extended_guide.md
@@ -40,10 +40,6 @@ The Cloudflare list item limit is the maximum number of items (blocked domains)
 
 Processes block/allow lists without actually adding/removing domains from Cloudflare. Avoid using this option unless you know what you are doing.
 
-#### `FAST_MODE`
-
-Sends requests much faster, but might cause rate limiting issues. Use with caution.
-
 ## Example usage
 
 These commands should be run in a terminal.
diff --git a/lib/api.js b/lib/api.js
@@ -48,6 +48,7 @@ export const createZeroTrustListsOneByOne = async (items) => {
       await createZeroTrustList(listName, chunk);
       totalListNumber--;
       listNumber++;
+
       console.log(`Created "${listName}" list - ${totalListNumber} left`);
     } catch (err) {
       console.error(`Could not create "${listName}" - ${err.toString()}`);
@@ -56,32 +57,6 @@ export const createZeroTrustListsOneByOne = async (items) => {
   }
 };
 
-/**
- * Creates all Zero Trust lists at once.
- * @param {string[]} items The domains.
- */
-export const createZeroTrustListsAtOnce = async (items) => {
-  const requests = [];
-
-  for (let i = 0, listNumber = 1; i < items.length; i += LIST_ITEM_SIZE) {
-    const chunk = items
-      .slice(i, i + LIST_ITEM_SIZE)
-      .map((item) => ({ value: item }));
-    const listName = `CGPS List - Chunk ${listNumber}`;
-
-    requests.push(createZeroTrustList(listName, chunk));
-    listNumber++;
-  }
-
-  try {
-    await Promise.all(requests);
-    console.log("Created lists successfully");
-  } catch (err) {
-    console.error(`Error occurred while creating lists - ${err.toString()}`);
-    throw err;
-  }
-};
-
 /**
  * Deletes a Zero Trust list.
  *
@@ -105,6 +80,7 @@ export const deleteZeroTrustListsOneByOne = async (lists) => {
     try {
       await deleteZeroTrustList(id);
       totalListNumber--;
+
       console.log(`Deleted ${name} list - ${totalListNumber} left`);
     } catch (err) {
       console.error(`Could not delete ${name} - ${err.toString()}`);
@@ -113,24 +89,6 @@ export const deleteZeroTrustListsOneByOne = async (lists) => {
   }
 };
 
-/**
- * Deletes all Zero Trust lists at once.
- * @param {Object[]} lists The lists to be deleted.
- * @param {number} lists[].id The ID of a list.
- * @param {string} lists[].name The name of a list.
- */
-export const deleteZeroTrustListsAtOnce = async (lists) => {
-  const requests = lists.map(({ id }) => deleteZeroTrustList(id));
-
-  try {
-    await Promise.all(requests);
-    console.log("Deleted lists successfully");
-  } catch (err) {
-    console.error(`Error occurred while deleting lists - ${err.toString()}`);
-    throw err;
-  }
-};
-
 /**
  * Gets Zero Trust rules.
  *
diff --git a/lib/constants.js b/lib/constants.js
@@ -26,14 +26,15 @@ export const API_HOST = "https://api.cloudflare.com/client/v4";
 
 export const DRY_RUN = !!parseInt(process.env.DRY_RUN, 10);
 
-export const FAST_MODE = !!parseInt(process.env.FAST_MODE, 10);
-
 export const BLOCK_PAGE_ENABLED = !!parseInt(process.env.BLOCK_PAGE_ENABLED, 10);
 
 export const BLOCK_BASED_ON_SNI = !!parseInt(process.env.BLOCK_BASED_ON_SNI, 10);
 
 export const DEBUG = !!parseInt(process.env.DEBUG, 10);
 
+export const CLOUDFLARE_RATE_LIMITING_COOLDOWN_TIME = 2 * 60 * 1000;
+export const RATE_LIMITING_HTTP_ERROR_CODE = 429;
+
 export const PROCESSING_FILENAME = {
   ALLOWLIST: "allowlist.txt",
   BLOCKLIST: "blocklist.txt",
@@ -63,7 +64,7 @@ export const RECOMMENDED_ALLOWLIST_URLS = [
   // Torrent trackers
   "https://raw.githubusercontent.com/im-sm/Pi-hole-Torrent-Blocklist/main/all-torrent-trackres.txt",
   // Banks
-  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/banks.txt", 
+  "https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/banks.txt",
   // Official Discord domains
   "https://raw.githubusercontent.com/Dogino/Discord-Phishing-URLs/main/official-domains.txt",
   // macOS specific
diff --git a/lib/helpers.js b/lib/helpers.js
@@ -5,6 +5,7 @@ import {
   API_KEY,
   API_TOKEN,
 } from "./constants.js";
+import { fetchRetry } from "./utils.js";
 
 if (!globalThis.fetch) {
   console.warn(
@@ -19,58 +20,6 @@ if (!globalThis.fetch) {
   globalThis.fetch = (await import("node-fetch")).default;
 }
 
-/**
- * Sends a message to a Discord-compatible webhook.
- * @param {url|string} url The webhook URL.
- * @param {string} message The message to be sent.
- * @returns {Promise}
- */
-async function sendMessageToWebhook(url, message) {
-  // Create the payload object with the message
-  // The message is provided as 2 different properties to improve compatibility with webhook servers outside Discord
-  const payload = { content: message, body: message };
-
-  // Send a POST request to the webhook url with the payload as JSON
-  try {
-    const response = await fetch(url, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify(payload),
-    });
-
-    // Check if the request was successful
-    if (!response.ok) {
-      throw new Error(`HTTP error! Status: ${response.status}`);
-    } else {
-      return true;
-    }
-  } catch (error) {
-    console.error('Error sending message to webhook:', error);
-    return false;
-  }
-}
-
-/**
- * Sends a CGPS notification to a Discord-compatible webhook.
- * Automatically checks if the webhook URL exists.
- * @param {string} msg The message to be sent.
- * @returns {Promise}
- */
-export async function notifyWebhook(msg) {
-  // Check if the webhook URL exists
-  const webhook_url = process.env.DISCORD_WEBHOOK_URL;
-
-  if (webhook_url && webhook_url.startsWith('http')) {
-    // Send the message to the webhook
-    try {
-      await sendMessageToWebhook(webhook_url, `CGPS: ${msg}`);
-    } catch (e) {
-      console.error('Error sending message to Discord webhook:', e);
-    }
-  }
-  // Not logging the lack of a webhook URL since it's not a feature everyone would use
-}
-
 /**
  * Fires request to the specified URL.
  * @param {string} url The URL to which the request will be fired.
@@ -87,51 +36,27 @@ const request = async (url, options) => {
   const headers = API_TOKEN
     ? {
         Authorization: `Bearer ${API_TOKEN}`,
-        "Content-Type": "application/json",
       }
     : {
         Authorization: `Bearer ${API_KEY}`,
-        "Content-Type": "application/json",
         "X-Auth-Email": ACCOUNT_EMAIL,
         "X-Auth-Key": API_KEY,
       };
-
-  /*
-	Due to the undocumented Cloudflare API rate limit, the request will be retried until Cloudflare accepts it.
-  */
-  const wait = (ms) => new Promise(resolve => setTimeout(resolve, ms));
-
-  let response;
   let data;
-  let attempts = 0;
-  let maxAttempts = 50;
 
-  while(attempts < 50) {
-    try {
-      response = await fetch(url, {
-        ...options,
-        headers: {
-          ...options.headers,
-          ...headers,
-        },
-      });
-
-      data = await response.json();
+  try {
+    const response = await fetchRetry(url, {
+      ...options,
+      headers: {
+      "Content-Type": "application/json",
+        ...options.headers,
+        ...headers,
+      },
+    });
 
-      if (!response.ok) {
-        throw new Error('Response not OK');
-      }
-      return data;
-    } catch (error) {
-      attempts++;
-      console.warn(`An error occured while making a web request: "${error}", retrying. Attempt ${attempts} of ${maxAttempts}.\nTHIS IS NORMAL IN MOST CIRCUMSTANCES. Refrain from reporting this as a bug unless the script doesn't automatically recover after several attempts.`);
-      if(attempts >= maxAttempts) {
-        // Send a message to the Discord webhook if it exists
-        await notifyWebhook(`An HTTP error has occurred (${response ? response.status : "unknown status"}) while making a web request. Please check the logs for further details.`);
-        throw new Error(`HTTP error! Status: ${response.status} - ${ (data && 'errors' in data) ? data.errors[0].message : data } - ${error}`);
-      }
-      await wait(8000);
-    }
+    return await response.json();
+  } catch (error) {
+    throw new Error(`${(data && 'errors' in data) ? data.errors[0].message : data} - ${error}`);
   }
 };
 
diff --git a/lib/utils.js b/lib/utils.js
