Add server-side AI bot tracking to my Lovable site so I can see which AI assistants (ChatGPT, Claude, Perplexity, Gemini, Copilot, etc.) read my pages and which ones recommend me.
Replace the contents of src/server.ts with the code below. The tracking runs inside the TanStack Start fetch handler — every incoming request is checked against a list of known AI bot User-Agents, and matching visits are POSTed to xSeek before the request is handed to the existing server handler. The POST is fire-and-forget via ctx.waitUntil() so it adds no latency to the response.
If src/server.ts already has custom logic beyond the default TanStack Start entry, preserve it. Just ensure the xSeek bot-tracking call runs FIRST inside the fetch handler, before any other request processing.
```ts
import "./lib/error-capture";
import { consumeLastCapturedError } from "./lib/error-capture";
import { renderErrorPage } from "./lib/error-page";
type ServerEntry = {
fetch: (request: Request, env: unknown, ctx: unknown) => Promise<Response> | Response;
};
interface Env {
XSEEK_API_KEY?: string;
XSEEK_WEBSITE_ID?: string;
}
// ── xSeek AI bot tracking ──────────────────────────────────────────────────
// Known AI bot User-Agent patterns. Synced from xSeek's public catalog —
// refresh from https://www.xseek.io/api/v1/robots when new crawlers ship.
const XSEEK_BOTS: { name: string; pattern: RegExp }[] = [
{ name: "anthropic-ai", pattern: /anthropic-ai/i },
{ name: "claudebot", pattern: /ClaudeBot/i },
{ name: "claude-web", pattern: /claude-web/i },
{ name: "claude-user", pattern: /Claude-User/i },
{ name: "claude-searchbot", pattern: /Claude-SearchBot/i },
{ name: "claude-code", pattern: /claude-code\//i },
{ name: "perplexitybot", pattern: /PerplexityBot/i },
{ name: "perplexity-user", pattern: /Perplexity-User/i },
{ name: "grokbot", pattern: /GrokBot(?!.*DeepSearch)/i },
{ name: "grok-search", pattern: /xAI-Grok/i },
{ name: "grok-deepsearch", pattern: /Grok-DeepSearch/i },
{ name: "GPTBot", pattern: /GPTBot/i },
{ name: "chatgpt-user", pattern: /ChatGPT-User/i },
{ name: "oai-searchbot", pattern: /OAI-SearchBot/i },
{ name: "google-extended", pattern: /Google-Extended/i },
{ name: "Google-Agent", pattern: /Google-Agent/i },
{ name: "applebot", pattern: /Applebot(?!-Extended)/i },
{ name: "applebot-extended", pattern: /Applebot-Extended/i },
{ name: "meta-external", pattern: /meta-externalagent/i },
{ name: "meta-externalfetcher", pattern: /meta-externalfetcher/i },
{ name: "bingbot", pattern: /Bingbot(?!.*AI)/i },
{ name: "bingpreview", pattern: /bingbot.*Chrome/i },
{ name: "microsoftpreview", pattern: /MicrosoftPreview/i },
{ name: "cohere-ai", pattern: /cohere-ai/i },
{ name: "cohere-training-data-crawler", pattern: /cohere-training-data-crawler/i },
{ name: "youbot", pattern: /YouBot/i },
{ name: "duckassistbot", pattern: /DuckAssistBot/i },
{ name: "semanticscholarbot", pattern: /SemanticScholarBot/i },
{ name: "ccbot", pattern: /CCBot/i },
{ name: "ai2bot", pattern: /AI2Bot/i },
{ name: "ai2bot-dolma", pattern: /AI2Bot-Dolma/i },
{ name: "aihitbot", pattern: /aiHitBot/i },
{ name: "amazonbot", pattern: /Amazonbot/i },
{ name: "novaact", pattern: /NovaAct/i },
{ name: "brightbot", pattern: /Brightbot/i },
{ name: "bytespider", pattern: /Bytespider/i },
{ name: "tiktokspider", pattern: /TikTokSpider/i },
{ name: "cotoyogi", pattern: /Cotoyogi/i },
{ name: "crawlspace", pattern: /Crawlspace/i },
{ name: "pangubot", pattern: /PanguBot/i },
{ name: "petalbot", pattern: /PetalBot/i },
{ name: "sidetrade-indexer", pattern: /Sidetrade indexer bot/i },
{ name: "timpibot", pattern: /Timpibot/i },
{ name: "omgili", pattern: /omgili/i },
{ name: "omgilibot", pattern: /omgilibot/i },
{ name: "webzio-extended", pattern: /Webzio-Extended/i },
{ name: "baiduspider", pattern: /Baiduspider/i },
{ name: "mistralai-user", pattern: /MistralAI-User/i },
];
function trackXseekBot(request: Request, env: Env): Promise<unknown> | null {
if (!env.XSEEK_API_KEY || !env.XSEEK_WEBSITE_ID) return null;
const ua = request.headers.get("user-agent") ?? "";
const bot = XSEEK_BOTS.find((b) => b.pattern.test(ua));
if (!bot) return null;
const url = new URL(request.url);
const ip =
request.headers.get("cf-connecting-ip") ??
request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ??
"";
return fetch(
`https://www.xseek.io/api/v1/websites/${env.XSEEK_WEBSITE_ID}/ai-visits`,
{
method: "POST",
headers: {
Authorization: `Bearer ${env.XSEEK_API_KEY}`,
"Content-Type": "application/json",
},
body: JSON.stringify({
botName: bot.name,
userAgent: ua,
url: url.pathname + url.search,
ip,
referer: request.headers.get("referer") ?? "",
}),
signal: AbortSignal.timeout(3000),
},
).catch((err) => {
console.error("[xseek] bot tracking failed:", err);
});
}
// ────────────────────────────────────────────────────────────────────────────
let serverEntryPromise: Promise<ServerEntry> | undefined;
async function getServerEntry(): Promise<ServerEntry> {
if (!serverEntryPromise) {
serverEntryPromise = import("@tanstack/react-start/server-entry").then(
(m) => (m.default ?? m) as ServerEntry,
);
}
return serverEntryPromise;
}
async function normalizeCatastrophicSsrResponse(response: Response): Promise<Response> {
if (response.status < 500) return response;
const contentType = response.headers.get("content-type") ?? "";
if (!contentType.includes("application/json")) return response;
const body = await response.clone().text();
if (!body.includes('"unhandled":true') || !body.includes('"message":"HTTPError"')) {
return response;
}
console.error(consumeLastCapturedError() ?? new Error(`h3 swallowed SSR error: ${body}`));
return new Response(renderErrorPage(), {
status: 500,
headers: { "content-type": "text/html; charset=utf-8" },
});
}
export default {
async fetch(request: Request, env: unknown, ctx: unknown) {
// Fire-and-forget bot tracking. ctx.waitUntil keeps the Worker alive
// long enough for the POST to finish, but doesn't block the response.
const trackPromise = trackXseekBot(request, env as Env);
if (
trackPromise &&
ctx &&
typeof (ctx as { waitUntil?: (p: Promise<unknown>) => void }).waitUntil === "function"
) {
(ctx as { waitUntil: (p: Promise<unknown>) => void }).waitUntil(trackPromise);
}
try {
const handler = await getServerEntry();
const response = await handler.fetch(request, env, ctx);
return await normalizeCatastrophicSsrResponse(response);
} catch (error) {
console.error(error);
return new Response(renderErrorPage(), {
status: 500,
headers: { "content-type": "text/html; charset=utf-8" },
});
}
},
};
```
Required environment variables (add in Lovable → Project Settings → Environment Variables):
- XSEEK_API_KEY — must include both articles:read and ai_visits:push privileges
- XSEEK_WEBSITE_ID — find it in your xSeek dashboard URL
After deploying, verify with a curl using a bot User-Agent:
curl -A "Mozilla/5.0 (compatible; GPTBot/1.0)" https://yoursite.com/
The visit should appear in xSeek → AI Visits within ~30 seconds.
