feat: migrate to turbo (#22)

* feat: migrate top turbo * ci: ci test * fix: fix codeql issues * feat: ci test * chore: lint * chore: misc changes * feat: rename vite helpers * feat: use fetch to handle assets * feat: update directory * feat: fetch charword table * feat: migrate download game data and detect missing voice files * feat: symlink relative path * feat: finish wrangler upload * feat: migrate wrangler download * feat: finish * chore: auto update * ci: update ci * ci: update ci --------- Co-authored-by: Halyul <Halyul@users.noreply.github.com>
2025-02-22 15:11:30 +08:00
parent 17c61ce5d4
commit d6e7bc20d3
352 changed files with 12911 additions and 9411 deletions
--- a/packages/downloader/index.js
+++ b/packages/downloader/index.js
@@ -0,0 +1,112 @@
+import path from 'node:path'
+import fs from 'node:fs'
+import { Buffer } from 'node:buffer'
+import { pipeline } from 'node:stream/promises'
+import pThrottle from 'p-throttle'
+import { file as fileLib } from '@aklive2d/libs'
+
+export const githubDownload = async (history_url, raw_url, filepath) => {
+    const historyResponse = await fetch(history_url)
+    const historyData = await historyResponse.json()
+    const lastCommit = historyData[0]
+    const lastCommitDate = new Date(lastCommit.commit.committer.date)
+    const ext = path.extname(filepath)
+    const basename = path.basename(filepath).replace(ext, '')
+    filepath = path.join(
+        path.dirname(filepath),
+        `${basename}_${lastCommitDate.getTime()}${ext}`
+    )
+    const dirpath = path.dirname(filepath)
+    console.log(`Last ${basename} commit date: ${lastCommitDate.getTime()}`)
+
+    if (fileLib.exists(filepath)) {
+        console.log(`${basename} is the latest version.`)
+        return JSON.parse(fileLib.readSync(filepath))
+    }
+    const response = await fetch(raw_url)
+    const data = await response.json()
+    fileLib.writeSync(JSON.stringify(data), filepath)
+    console.log(`${basename} is updated.`)
+
+    // remove old file
+    const files = fileLib.readdirSync(path.join(dirpath))
+    for (const file of files) {
+        if (file.startsWith(basename) && file !== path.basename(filepath)) {
+            fileLib.rm(path.join(dirpath, file))
+        }
+    }
+    return data
+}
+
+export const unzipDownload = async (
+    filesToDownload,
+    targetDir,
+    opts = {
+        defaultRegex: null,
+        matchRegExps: [],
+    }
+) => {
+    let retry = filesToDownload
+    const throttle = pThrottle({
+        limit: 3,
+        interval: 1000,
+    })
+    while (retry.length > 0) {
+        const newRetry = []
+        await Promise.all(
+            retry.map(
+                throttle(async (item) => {
+                    const name = item.name
+                    console.log(`Downloading ${name}`)
+                    const zip = await fetch(item.url)
+                        .then((resp) => {
+                            const status = resp.status
+                            if (status !== 200)
+                                throw new Error(`Status Code: ${status}`)
+                            return resp.arrayBuffer()
+                        })
+                        .then((arrayBuffer) => {
+                            return fileLib.unzip.fromBuffer(
+                                Buffer.from(arrayBuffer)
+                            )
+                        })
+                        .catch((err) => {
+                            console.error(err)
+                        })
+                    try {
+                        for await (const entry of zip) {
+                            if (
+                                opts.defaultRegex &&
+                                !opts.defaultRegex.test(entry.filename)
+                            ) {
+                                continue
+                            }
+                            if (opts.matchRegExps.length > 0) {
+                                let shallContinue = false
+                                for (const regex of opts.matchRegExps) {
+                                    if (!regex.test(entry.filename)) {
+                                        shallContinue = true
+                                        break
+                                    }
+                                }
+                                if (shallContinue) continue
+                            }
+                            const filePath = path.join(
+                                targetDir,
+                                entry.filename
+                            )
+                            fileLib.mkdir(path.dirname(filePath))
+                            const readStream = await entry.openReadStream()
+                            const writeStream = fs.createWriteStream(filePath)
+                            await pipeline(readStream, writeStream)
+                            console.log(`Finish Writing to ${entry.filename}`)
+                        }
+                    } finally {
+                        await zip.close()
+                    }
+                })
+            )
+        )
+        retry = newRetry
+    }
+}