128 lines
4.5 KiB
TypeScript
128 lines
4.5 KiB
TypeScript
import path from 'node:path'
|
|
import fs from 'node:fs'
|
|
import { Buffer } from 'node:buffer'
|
|
import { pipeline } from 'node:stream/promises'
|
|
import pThrottle from 'p-throttle'
|
|
import { file as fileLib } from '@aklive2d/libs'
|
|
|
|
export type UnzipDownloadItem = {
|
|
name: string
|
|
url: string
|
|
}
|
|
|
|
export const githubDownload = async (
|
|
history_url: string,
|
|
raw_url: string,
|
|
filepath: string
|
|
) => {
|
|
const historyResponse = await fetch(history_url)
|
|
const historyData = await historyResponse.json()
|
|
const lastCommit = historyData[0]
|
|
const lastCommitDate = new Date(lastCommit.commit.committer.date)
|
|
const ext = path.extname(filepath)
|
|
const basename = path.basename(filepath).replace(ext, '')
|
|
filepath = path.join(
|
|
path.dirname(filepath),
|
|
`${basename}_${lastCommitDate.getTime()}${ext}`
|
|
)
|
|
const dirpath = path.dirname(filepath)
|
|
console.log(`Last ${basename} commit date: ${lastCommitDate.getTime()}`)
|
|
|
|
if (fileLib.exists(filepath)) {
|
|
console.log(`${basename} is the latest version.`)
|
|
const content = fileLib.readSync(filepath)
|
|
return content ? JSON.parse(content) : null
|
|
}
|
|
const response = await fetch(raw_url)
|
|
const data = await response.json()
|
|
fileLib.writeSync(JSON.stringify(data), filepath)
|
|
console.log(`${basename} is updated.`)
|
|
|
|
// remove old file
|
|
const files = fileLib.readdirSync(path.join(dirpath))
|
|
for (const file of files) {
|
|
if (file.startsWith(basename) && file !== path.basename(filepath)) {
|
|
fileLib.rm(path.join(dirpath, file))
|
|
}
|
|
}
|
|
return data
|
|
}
|
|
|
|
export const unzipDownload = async (
|
|
filesToDownload: UnzipDownloadItem[],
|
|
targetDir: string,
|
|
opts: {
|
|
defaultRegex: RegExp | null
|
|
matchRegExps: RegExp[]
|
|
} = {
|
|
defaultRegex: null,
|
|
matchRegExps: [],
|
|
}
|
|
) => {
|
|
let retry = filesToDownload
|
|
const throttle = pThrottle({
|
|
limit: 3,
|
|
interval: 1000,
|
|
})
|
|
while (retry.length > 0) {
|
|
const newRetry: UnzipDownloadItem[] = []
|
|
await Promise.all(
|
|
retry.map(
|
|
throttle(async (item) => {
|
|
const name = item.name
|
|
console.log(`Downloading ${name}`)
|
|
const zip = await fetch(item.url)
|
|
.then((resp) => {
|
|
const status = resp.status
|
|
if (status !== 200)
|
|
throw new Error(`Status Code: ${status}`)
|
|
return resp.arrayBuffer()
|
|
})
|
|
.then((arrayBuffer) => {
|
|
return fileLib.unzip.fromBuffer(
|
|
Buffer.from(arrayBuffer)
|
|
)
|
|
})
|
|
.catch((err) => {
|
|
console.error(err)
|
|
return null
|
|
})
|
|
if (!zip) return
|
|
try {
|
|
for await (const entry of zip) {
|
|
if (
|
|
opts.defaultRegex &&
|
|
!opts.defaultRegex.test(entry.filename)
|
|
) {
|
|
continue
|
|
}
|
|
if (opts.matchRegExps.length > 0) {
|
|
let shallContinue = false
|
|
for (const regex of opts.matchRegExps) {
|
|
if (!regex.test(entry.filename)) {
|
|
shallContinue = true
|
|
break
|
|
}
|
|
}
|
|
if (shallContinue) continue
|
|
}
|
|
const filePath = path.join(
|
|
targetDir,
|
|
entry.filename
|
|
)
|
|
fileLib.mkdir(path.dirname(filePath))
|
|
const readStream = await entry.openReadStream()
|
|
const writeStream = fs.createWriteStream(filePath)
|
|
await pipeline(readStream, writeStream)
|
|
console.log(`Finish Writing to ${entry.filename}`)
|
|
}
|
|
} finally {
|
|
await zip.close()
|
|
}
|
|
})
|
|
)
|
|
)
|
|
retry = newRetry
|
|
}
|
|
}
|