feat(aklive2d): use cf pages to store assets (download) [2/2]

.github/workflows/cf-pages.yaml

@@ -15,6 +15,8 @@ jobs:
           version: latest
       - name: Install dependencies
         run: pnpm i
+      - name: Download Assets
+        run: pnpm run cf:download
       - name: Build all
         run: pnpm run operator:build-all
       - name: Build directory

.vscode/launch.json

@@ -83,6 +83,13 @@
       "request": "launch",
       "command": "pnpm run cf:upload",
       "cwd": "${workspaceFolder}"
+    },
+    {
+      "type": "node-terminal",
+      "name": "Run Script: cf:download",
+      "request": "launch",
+      "command": "pnpm run cf:download",
+      "cwd": "${workspaceFolder}"
     }
   ]
 }

aklive2d.js

@@ -58,10 +58,10 @@ async function main() {
       await officalInfo.update()
       process.exit(0)
     case 'cf:upload':
-      (new CFPages()).upload()
+      await (new CFPages()).upload()
       process.exit(0)
     case 'cf:download':
-      (new CFPages()).download()
+      await (new CFPages()).download()
       process.exit(0)
     default:
       break

config.yaml

@@ -1,3 +1,4 @@
+akassets_url: https://akassets.halyul.dev
 folder:
   auto_update_data: ./data/auto_update
   operator_data: ./data/operator/

libs/cf_pages.js

@@ -1,18 +1,17 @@
 /* eslint-disable no-undef */
+import fs from 'fs';
 import path from 'path';
+import crypto from 'crypto';
 import { spawnSync } from 'child_process';
-import { readdirSync, fileTypeSync, writeSync } from './file.js';
+import { readdirSync, fileTypeSync, writeSync, mkdir, exists } from './file.js';
 
 export default class CFPages {
     #uploadPath = path.join(__projectRoot, __config.folder.operator_data);
+    #downloadPath = path.join(__projectRoot, __config.folder.operator_data);
     #gitignorePath = path.join(__projectRoot, '.gitignore');
 
-    constructor() {
+    async upload() {
-        
+        const tree = await this.#generateDirTree(this.#uploadPath);
-    }
-
-    upload() {
-        const tree = this.#generateDirTree(this.#uploadPath);
         writeSync(JSON.stringify(tree, null), path.join(this.#uploadPath, 'index.json'));
         const wrangler = spawnSync('pnpm', ['wrangler', 'pages', 'deploy', this.#uploadPath]);
 
@@ -21,11 +20,77 @@ export default class CFPages {
         console.log('stderr ', wrangler.stderr.toString());
     }
 
-    download() {
+    async download() {
-
+        const indexFile = `${__config.akassets_url}/index.json`
+        const resp = await fetch(indexFile);
+        const data = await resp.json();
+        mkdir(this.#downloadPath);
+        let list = data.children.flatMap((child) => {
+            return this.#generateDownloadList(child, this.#downloadPath);
+        });
+        while (list.length > 0) {
+            const retry = [];
+            for (const file of list) {
+                let toDownload = false;
+                let suppressedPath = file.target.replace(this.#downloadPath, '');
+                if (exists(file.target)) {
+                    const hash = await this.#getHash(file.target);
+                    if (hash !== file.hash) {
+                        toDownload = true;
+                    } else {
+                        console.log("File already exists and hash matches:", suppressedPath);
+                    }
+                }
+                if (!exists(file.target) || toDownload) {
+                    await fetch(file.url)
+                        .then(response => {
+                            return response.arrayBuffer();
+                        })
+                        .then(arraybuffer => {
+                            console.log("Writing to file:", suppressedPath);
+                            writeSync(Buffer.from(arraybuffer), file.target);
+                            return this.#getHash(file.target)
+                        })
+                        .then(hash => {
+                            if (hash !== file.hash) {
+                                throw new Error(`Hash mismatch`);
+                            }
+                        })
+                        .catch(err => {
+                            console.log(`Found error for ${suppressedPath}:`, err)
+                            retry.push(file);
+                        });
+                }
+            }
+            list = retry;
+        }
     }
 
-    #generateDirTree(dir) {
+    #generateDownloadList(data, baseDir, baseUrl = "") {
+        if (data.type === 'dir') {
+            let list = [];
+            const curDir = path.join(baseDir, data.name);
+            mkdir(curDir);
+            if (data.children.length > 0) {
+                for (const child of data.children) {
+                    const filesToDownload = this.#generateDownloadList(child, curDir, baseUrl + data.name + '/');
+                    if (filesToDownload) {
+                        list = [...list, ...filesToDownload];
+                    }
+                }
+                return list;
+            }
+        } else {
+            return [{
+                url: `${__config.akassets_url}/${baseUrl + data.name.replace('#', '%23')}`,
+                target: path.join(baseDir, data.name),
+                hash: data.hash
+            }]
+        }
+        return null;
+    }
+
+    async #generateDirTree(dir) {
         const files = readdirSync(dir);
         let tree = {
             name: path.basename(dir),
@@ -39,11 +104,12 @@ export default class CFPages {
             const filePath = path.join(dir, file);
             const dirType = fileTypeSync(filePath);
             if (dirType === 'dir') {
-                tree.children.push(this.#generateDirTree(filePath))
+                tree.children.push(await this.#generateDirTree(filePath))
             } else {
                 tree.children.push({
                     name: file,
-                    type: 'file'
+                    type: 'file',
+                    hash: await this.#getHash(filePath)
                 });
             }
         }
@@ -63,4 +129,17 @@ export default class CFPages {
                 return false;
         }
     }
+
+    #getHash(filePath) {
+        return new Promise((res, rej) => {
+            const hash = crypto.createHash('md5');
+            const rStream = fs.createReadStream(filePath);
+            rStream.on('data', (data) => {
+                hash.update(data);
+            });
+            rStream.on('end', () => {
+                res(hash.digest('hex'));
+            });
+        })
+    }
 }