diff --git a/.github/workflows/cf-pages.yaml b/.github/workflows/cf-pages.yaml index 5536761..4443ca7 100644 --- a/.github/workflows/cf-pages.yaml +++ b/.github/workflows/cf-pages.yaml @@ -15,6 +15,8 @@ jobs: version: latest - name: Install dependencies run: pnpm i + - name: Download Assets + run: pnpm run cf:download - name: Build all run: pnpm run operator:build-all - name: Build directory diff --git a/.vscode/launch.json b/.vscode/launch.json index 90526c3..fc223aa 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -83,6 +83,13 @@ "request": "launch", "command": "pnpm run cf:upload", "cwd": "${workspaceFolder}" + }, + { + "type": "node-terminal", + "name": "Run Script: cf:download", + "request": "launch", + "command": "pnpm run cf:download", + "cwd": "${workspaceFolder}" } ] } diff --git a/aklive2d.js b/aklive2d.js index 63c0877..05c5661 100644 --- a/aklive2d.js +++ b/aklive2d.js @@ -58,10 +58,10 @@ async function main() { await officalInfo.update() process.exit(0) case 'cf:upload': - (new CFPages()).upload() + await (new CFPages()).upload() process.exit(0) case 'cf:download': - (new CFPages()).download() + await (new CFPages()).download() process.exit(0) default: break diff --git a/config.yaml b/config.yaml index 78b9724..0c0bd3f 100644 --- a/config.yaml +++ b/config.yaml @@ -1,3 +1,4 @@ +akassets_url: https://akassets.halyul.dev folder: auto_update_data: ./data/auto_update operator_data: ./data/operator/ diff --git a/libs/cf_pages.js b/libs/cf_pages.js index 4b35cf7..e5878aa 100644 --- a/libs/cf_pages.js +++ b/libs/cf_pages.js @@ -1,18 +1,17 @@ /* eslint-disable no-undef */ +import fs from 'fs'; import path from 'path'; +import crypto from 'crypto'; import { spawnSync } from 'child_process'; -import { readdirSync, fileTypeSync, writeSync } from './file.js'; +import { readdirSync, fileTypeSync, writeSync, mkdir, exists } from './file.js'; export default class CFPages { #uploadPath = path.join(__projectRoot, __config.folder.operator_data); + #downloadPath = path.join(__projectRoot, __config.folder.operator_data); #gitignorePath = path.join(__projectRoot, '.gitignore'); - constructor() { - - } - - upload() { - const tree = this.#generateDirTree(this.#uploadPath); + async upload() { + const tree = await this.#generateDirTree(this.#uploadPath); writeSync(JSON.stringify(tree, null), path.join(this.#uploadPath, 'index.json')); const wrangler = spawnSync('pnpm', ['wrangler', 'pages', 'deploy', this.#uploadPath]); @@ -21,11 +20,77 @@ export default class CFPages { console.log('stderr ', wrangler.stderr.toString()); } - download() { - + async download() { + const indexFile = `${__config.akassets_url}/index.json` + const resp = await fetch(indexFile); + const data = await resp.json(); + mkdir(this.#downloadPath); + let list = data.children.flatMap((child) => { + return this.#generateDownloadList(child, this.#downloadPath); + }); + while (list.length > 0) { + const retry = []; + for (const file of list) { + let toDownload = false; + let suppressedPath = file.target.replace(this.#downloadPath, ''); + if (exists(file.target)) { + const hash = await this.#getHash(file.target); + if (hash !== file.hash) { + toDownload = true; + } else { + console.log("File already exists and hash matches:", suppressedPath); + } + } + if (!exists(file.target) || toDownload) { + await fetch(file.url) + .then(response => { + return response.arrayBuffer(); + }) + .then(arraybuffer => { + console.log("Writing to file:", suppressedPath); + writeSync(Buffer.from(arraybuffer), file.target); + return this.#getHash(file.target) + }) + .then(hash => { + if (hash !== file.hash) { + throw new Error(`Hash mismatch`); + } + }) + .catch(err => { + console.log(`Found error for ${suppressedPath}:`, err) + retry.push(file); + }); + } + } + list = retry; + } } - #generateDirTree(dir) { + #generateDownloadList(data, baseDir, baseUrl = "") { + if (data.type === 'dir') { + let list = []; + const curDir = path.join(baseDir, data.name); + mkdir(curDir); + if (data.children.length > 0) { + for (const child of data.children) { + const filesToDownload = this.#generateDownloadList(child, curDir, baseUrl + data.name + '/'); + if (filesToDownload) { + list = [...list, ...filesToDownload]; + } + } + return list; + } + } else { + return [{ + url: `${__config.akassets_url}/${baseUrl + data.name.replace('#', '%23')}`, + target: path.join(baseDir, data.name), + hash: data.hash + }] + } + return null; + } + + async #generateDirTree(dir) { const files = readdirSync(dir); let tree = { name: path.basename(dir), @@ -39,11 +104,12 @@ export default class CFPages { const filePath = path.join(dir, file); const dirType = fileTypeSync(filePath); if (dirType === 'dir') { - tree.children.push(this.#generateDirTree(filePath)) + tree.children.push(await this.#generateDirTree(filePath)) } else { tree.children.push({ name: file, - type: 'file' + type: 'file', + hash: await this.#getHash(filePath) }); } } @@ -63,4 +129,17 @@ export default class CFPages { return false; } } + + #getHash(filePath) { + return new Promise((res, rej) => { + const hash = crypto.createHash('md5'); + const rStream = fs.createReadStream(filePath); + rStream.on('data', (data) => { + hash.update(data); + }); + rStream.on('end', () => { + res(hash.digest('hex')); + }); + }) + } } \ No newline at end of file