aklive2d/packages/downloader/index.js

import path from 'node:path'
import fs from 'node:fs'
import { Buffer } from 'node:buffer'
import { pipeline } from 'node:stream/promises'
import pThrottle from 'p-throttle'
import { file as fileLib } from '@aklive2d/libs'

export const githubDownload = async (history_url, raw_url, filepath) => {
    const historyResponse = await fetch(history_url)
    const historyData = await historyResponse.json()
    const lastCommit = historyData[0]
    const lastCommitDate = new Date(lastCommit.commit.committer.date)
    const ext = path.extname(filepath)
    const basename = path.basename(filepath).replace(ext, '')
    filepath = path.join(
        path.dirname(filepath),
        `${basename}_${lastCommitDate.getTime()}${ext}`
    )
    const dirpath = path.dirname(filepath)
    console.log(`Last ${basename} commit date: ${lastCommitDate.getTime()}`)

    if (fileLib.exists(filepath)) {
        console.log(`${basename} is the latest version.`)
        return JSON.parse(fileLib.readSync(filepath))
    }
    const response = await fetch(raw_url)
    const data = await response.json()
    fileLib.writeSync(JSON.stringify(data), filepath)
    console.log(`${basename} is updated.`)

    // remove old file
    const files = fileLib.readdirSync(path.join(dirpath))
    for (const file of files) {
        if (file.startsWith(basename) && file !== path.basename(filepath)) {
            fileLib.rm(path.join(dirpath, file))
        }
    }
    return data
}

export const unzipDownload = async (
    filesToDownload,
    targetDir,
    opts = {
        defaultRegex: null,
        matchRegExps: [],
    }
) => {
    let retry = filesToDownload
    const throttle = pThrottle({
        limit: 3,
        interval: 1000,
    })
    while (retry.length > 0) {
        const newRetry = []
        await Promise.all(
            retry.map(
                throttle(async (item) => {
                    const name = item.name
                    console.log(`Downloading ${name}`)
                    const zip = await fetch(item.url)
                        .then((resp) => {
                            const status = resp.status
                            if (status !== 200)
                                throw new Error(`Status Code: ${status}`)
                            return resp.arrayBuffer()
                        })
                        .then((arrayBuffer) => {
                            return fileLib.unzip.fromBuffer(
                                Buffer.from(arrayBuffer)
                            )
                        })
                        .catch((err) => {
                            console.error(err)
                        })
                    try {
                        for await (const entry of zip) {
                            if (
                                opts.defaultRegex &&
                                !opts.defaultRegex.test(entry.filename)
                            ) {
                                continue
                            }
                            if (opts.matchRegExps.length > 0) {
                                let shallContinue = false
                                for (const regex of opts.matchRegExps) {
                                    if (!regex.test(entry.filename)) {
                                        shallContinue = true
                                        break
                                    }
                                }
                                if (shallContinue) continue
                            }
                            const filePath = path.join(
                                targetDir,
                                entry.filename
                            )
                            fileLib.mkdir(path.dirname(filePath))
                            const readStream = await entry.openReadStream()
                            const writeStream = fs.createWriteStream(filePath)
                            await pipeline(readStream, writeStream)
                            console.log(`Finish Writing to ${entry.filename}`)
                        }
                    } finally {
                        await zip.close()
                    }
                })
            )
        )
        retry = newRetry
    }
}