From b2cf31b14bb12591c1ccd42b650ca7e44b37c29a Mon Sep 17 00:00:00 2001
From: Haoyu Xu <i@halyul.dev>
Date: Thu, 15 Aug 2024 11:41:47 +0800
Subject: [PATCH] feat(aklive2d): added download parallelism

---
 libs/cf_pages.js | 18 +++++++++++-------
 package.json     |  1 +
 pnpm-lock.yaml   |  9 +++++++++
 3 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/libs/cf_pages.js b/libs/cf_pages.js
index e5878aa..6a705ff 100644
--- a/libs/cf_pages.js
+++ b/libs/cf_pages.js
@@ -2,6 +2,7 @@
 import fs from 'fs';
 import path from 'path';
 import crypto from 'crypto';
+import pThrottle from 'p-throttle';
 import { spawnSync } from 'child_process';
 import { readdirSync, fileTypeSync, writeSync, mkdir, exists } from './file.js';
 
@@ -28,20 +29,23 @@ export default class CFPages {
         let list = data.children.flatMap((child) => {
             return this.#generateDownloadList(child, this.#downloadPath);
         });
+        const throttle = pThrottle({
+            limit: 10,
+            interval: 500
+        })
         while (list.length > 0) {
             const retry = [];
-            for (const file of list) {
-                let toDownload = false;
+            await Promise.all(list.map(throttle(async (file) => {
+                let isExists = false;
                 let suppressedPath = file.target.replace(this.#downloadPath, '');
                 if (exists(file.target)) {
                     const hash = await this.#getHash(file.target);
-                    if (hash !== file.hash) {
-                        toDownload = true;
-                    } else {
+                    if (hash === file.hash) {
+                        isExists = true
                         console.log("File already exists and hash matches:", suppressedPath);
                     }
                 }
-                if (!exists(file.target) || toDownload) {
+                if (!isExists) {
                     await fetch(file.url)
                         .then(response => {
                             return response.arrayBuffer();
@@ -61,7 +65,7 @@ export default class CFPages {
                             retry.push(file);
                         });
                 }
-            }
+            })));
             list = retry;
         }
     }
diff --git a/package.json b/package.json
index 787790b..b9c527f 100644
--- a/package.json
+++ b/package.json
@@ -41,6 +41,7 @@
     "jotai": "^2.9.1",
     "jsdom": "^22.1.0",
     "node-fetch": "^3.3.2",
+    "p-throttle": "^6.2.0",
     "react": "^18.3.1",
     "react-dom": "^18.3.1",
     "react-refresh": "^0.14.2",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 1391d36..9ce835c 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -23,6 +23,9 @@ importers:
       node-fetch:
         specifier: ^3.3.2
         version: 3.3.2
+      p-throttle:
+        specifier: ^6.2.0
+        version: 6.2.0
       react:
         specifier: ^18.3.1
         version: 18.3.1
@@ -1773,6 +1776,10 @@ packages:
     resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==}
     engines: {node: '>=10'}
 
+  p-throttle@6.2.0:
+    resolution: {integrity: sha512-NCKkOVj6PZa6NiTmfvGilDdf6vO1rFCD3KDnkHko8dTOtkpk4cSR/VTAhhLMG9aiQ7/A9HYgEDNmxzf6hxzR3g==}
+    engines: {node: '>=18'}
+
   parent-module@1.0.1:
     resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==}
     engines: {node: '>=6'}
@@ -4088,6 +4095,8 @@ snapshots:
     dependencies:
       p-limit: 3.1.0
 
+  p-throttle@6.2.0: {}
+
   parent-module@1.0.1:
     dependencies:
       callsites: 3.1.0