65 lines
1.9 KiB
TypeScript
65 lines
1.9 KiB
TypeScript
import { absoluteUrl } from "./url.js";
|
||
import type { RobotsTxtInput, SiteSeoConfig } from "./types.js";
|
||
|
||
/** 站点默认 robots.txt：允许全站抓取 + 指向 sitemap。 */
|
||
export function buildDefaultRobotsInput(site: Pick<SiteSeoConfig, "siteUrl">): RobotsTxtInput {
|
||
return {
|
||
rules: [
|
||
{
|
||
userAgent: "*",
|
||
allow: ["/"],
|
||
disallow: ["/api/", "/dashboard"],
|
||
},
|
||
],
|
||
sitemap: [absoluteUrl(site.siteUrl, "/sitemap.xml")],
|
||
host: site.siteUrl,
|
||
};
|
||
}
|
||
|
||
/** 生成 robots.txt 纯文本。 */
|
||
export function buildRobotsTxt(input: RobotsTxtInput): string {
|
||
const lines: string[] = [];
|
||
|
||
if (input.host) {
|
||
lines.push(`Host: ${input.host.replace(/\/+$/, "")}`);
|
||
}
|
||
|
||
const rules = input.rules ?? [];
|
||
for (const rule of rules) {
|
||
if (lines.length > 0) lines.push("");
|
||
lines.push(`User-agent: ${rule.userAgent}`);
|
||
for (const a of rule.allow ?? []) {
|
||
lines.push(`Allow: ${a}`);
|
||
}
|
||
for (const d of rule.disallow ?? []) {
|
||
// 空 Disallow 表示允许全部，跳过空串避免误导
|
||
if (d === "") continue;
|
||
lines.push(`Disallow: ${d}`);
|
||
}
|
||
if (rule.crawlDelay !== undefined) {
|
||
lines.push(`Crawl-delay: ${rule.crawlDelay}`);
|
||
}
|
||
}
|
||
|
||
lines.push("");
|
||
for (const s of input.sitemap ?? []) {
|
||
lines.push(`Sitemap: ${s}`);
|
||
}
|
||
lines.push("");
|
||
return lines.join("\n");
|
||
}
|
||
|
||
/**
|
||
* 抓取预算辅助：分页页默认 noindex + canonical 指向首页，
|
||
* 避免无限分页稀释抓取配额。
|
||
*/
|
||
export function paginationRobots(
|
||
pageNumber: number,
|
||
): Array<"index" | "noindex" | "follow" | "nofollow"> {
|
||
if (pageNumber <= 1) return ["index", "follow"];
|
||
return ["noindex", "follow"];
|
||
}
|
||
|
||
/** 内部/预发布页：noindex + nofollow。 */
|
||
export const DRAFT_ROBOTS = ["noindex", "nofollow", "noarchive"] as const;
|