import { absoluteUrl } from "./url.js"; import type { RobotsTxtInput, SiteSeoConfig } from "./types.js"; /** 站点默认 robots.txt:允许全站抓取 + 指向 sitemap。 */ export function buildDefaultRobotsInput(site: Pick): RobotsTxtInput { return { rules: [ { userAgent: "*", allow: ["/"], disallow: ["/api/", "/dashboard"], }, ], sitemap: [absoluteUrl(site.siteUrl, "/sitemap.xml")], host: site.siteUrl, }; } /** 生成 robots.txt 纯文本。 */ export function buildRobotsTxt(input: RobotsTxtInput): string { const lines: string[] = []; if (input.host) { lines.push(`Host: ${input.host.replace(/\/+$/, "")}`); } const rules = input.rules ?? []; for (const rule of rules) { if (lines.length > 0) lines.push(""); lines.push(`User-agent: ${rule.userAgent}`); for (const a of rule.allow ?? []) { lines.push(`Allow: ${a}`); } for (const d of rule.disallow ?? []) { // 空 Disallow 表示允许全部,跳过空串避免误导 if (d === "") continue; lines.push(`Disallow: ${d}`); } if (rule.crawlDelay !== undefined) { lines.push(`Crawl-delay: ${rule.crawlDelay}`); } } lines.push(""); for (const s of input.sitemap ?? []) { lines.push(`Sitemap: ${s}`); } lines.push(""); return lines.join("\n"); } /** * 抓取预算辅助:分页页默认 noindex + canonical 指向首页, * 避免无限分页稀释抓取配额。 */ export function paginationRobots( pageNumber: number, ): Array<"index" | "noindex" | "follow" | "nofollow"> { if (pageNumber <= 1) return ["index", "follow"]; return ["noindex", "follow"]; } /** 内部/预发布页:noindex + nofollow。 */ export const DRAFT_ROBOTS = ["noindex", "nofollow", "noarchive"] as const;