fix(trpc): bound scrape concurrency and surface DB errors
This commit is contained in:
parent
94d389b855
commit
c37f9dcf62
|
|
@ -5,6 +5,10 @@ import { createWebdav, isVideoFilename, joinWebdavPath, listDirectory } from "./
|
|||
import { scrapeService } from "./scrape.service.js";
|
||||
|
||||
const MAX_SCAN_FILES = 2000;
|
||||
/** 同时执行的 upsert+刮削 job 上限,避免最坏 2000 并发打 Bangumi 被限流 */
|
||||
const SCRAPE_CONCURRENCY = 4;
|
||||
|
||||
type ScanJob = () => Promise<void>;
|
||||
|
||||
async function walkVideos(
|
||||
userId: string,
|
||||
|
|
@ -12,7 +16,7 @@ async function walkVideos(
|
|||
root: string,
|
||||
rel: string,
|
||||
budget: { left: number },
|
||||
out: Promise<void>[],
|
||||
out: ScanJob[],
|
||||
): Promise<void> {
|
||||
if (budget.left <= 0) return;
|
||||
const mount = await mountDao.getByIdForUser(mountId, userId);
|
||||
|
|
@ -38,8 +42,8 @@ async function walkVideos(
|
|||
if (!isVideoFilename(entry.basename)) continue;
|
||||
budget.left -= 1;
|
||||
const title = entry.basename.replace(/\.[a-z0-9]+$/i, "");
|
||||
out.push(
|
||||
(async () => {
|
||||
// 延迟执行:扫描阶段只收集 job,随后分批限流
|
||||
out.push(async () => {
|
||||
const item = await mediaItemDao.upsertFromScan({
|
||||
userId,
|
||||
mountId,
|
||||
|
|
@ -49,8 +53,7 @@ async function walkVideos(
|
|||
size: entry.size,
|
||||
});
|
||||
await scrapeService.autoScrape(userId, item.id);
|
||||
})(),
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -78,9 +81,12 @@ export const libraryService = {
|
|||
if (!mount) throw new TRPCError({ code: "NOT_FOUND", message: "挂载不存在" });
|
||||
if (!mount.enabled) throw new TRPCError({ code: "BAD_REQUEST", message: "挂载已禁用" });
|
||||
const budget = { left: MAX_SCAN_FILES };
|
||||
const jobs: Promise<void>[] = [];
|
||||
const jobs: ScanJob[] = [];
|
||||
await walkVideos(userId, mountId, mount.rootPath || "/", "/", budget, jobs);
|
||||
await Promise.allSettled(jobs);
|
||||
for (let i = 0; i < jobs.length; i += SCRAPE_CONCURRENCY) {
|
||||
const batch = jobs.slice(i, i + SCRAPE_CONCURRENCY).map((job) => job());
|
||||
await Promise.allSettled(batch);
|
||||
}
|
||||
return { scanned: MAX_SCAN_FILES - budget.left, message: "扫描完成" };
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -104,8 +104,17 @@ export const scrapeService = {
|
|||
});
|
||||
return;
|
||||
}
|
||||
// 只包网络搜索:DB 写失败必须向外抛,避免被误标 failed 后吞掉
|
||||
let hits: BangumiSearchHit[];
|
||||
try {
|
||||
const hits = await bangumiSearch(q);
|
||||
hits = await bangumiSearch(q);
|
||||
} catch {
|
||||
await mediaItemDao.updateScrape(mediaItemId, userId, {
|
||||
scrapeStatus: "failed",
|
||||
scrapedAt: new Date(),
|
||||
});
|
||||
return;
|
||||
}
|
||||
const hit = hits[0];
|
||||
if (!hit) {
|
||||
await mediaItemDao.updateScrape(mediaItemId, userId, {
|
||||
|
|
@ -122,11 +131,5 @@ export const scrapeService = {
|
|||
title: hit.nameCn || hit.name,
|
||||
epNumber: parseEpisodeFromFilename(row.rawName),
|
||||
});
|
||||
} catch {
|
||||
await mediaItemDao.updateScrape(mediaItemId, userId, {
|
||||
scrapeStatus: "failed",
|
||||
scrapedAt: new Date(),
|
||||
});
|
||||
}
|
||||
},
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in New Issue