Cloudflare Workers

URL: https://aisearchindex.com/api/v1/logs/custom
Headers:
  x-api-key: bot_YOUR_BOT_TRACKING_KEY

/**
 * Cloudflare Worker – stream HTML page views to AI Search Index
 * Excludes common static assets and only logs text/html + text/plain (robots.txt).
 */
const IGNORE_EXT = /\.(?:png|jpe?g|gif|svg|webp|ico|mp4|webm|mp3|wav|ogg|css|js|mjs|json|map|pdf|zip|rar|7z|tar|gz|woff2?|ttf|eot)$/i;

export default {
  async fetch(request, env, ctx) {
    const url = new URL(request.url);
    
    // Skip static assets
    if (IGNORE_EXT.test(url.pathname)) {
      return fetch(request);
    }

    const response = await fetch(request);
    const contentType = response.headers.get("content-type")?.split(";")[0];
    
    // Only log HTML and plain text (robots.txt) responses
    if (!contentType || !["text/html", "text/plain"].includes(contentType) || response.status >= 500) {
      return response;
    }

    const log = {
      timestamp: Date.now(),
      method: request.method,
      host: request.headers.get("Host") || url.hostname,
      path: url.pathname,
      status_code: response.status,
      ip: request.headers.get("CF-Connecting-IP") || "",
      user_agent: request.headers.get("User-Agent") || "",
      referer: request.headers.get("Referer") || null,
      query_params: Object.fromEntries(url.searchParams.entries()),
      content_type: contentType,
    };

    // Send log asynchronously (non-blocking)
    ctx.waitUntil(
      fetch(env.AIS_ENDPOINT, {
        method: "POST",
        headers: {
          "Content-Type": "application/json",
          "x-api-key": env.AIS_BOT_KEY,
        },
        body: JSON.stringify(log),
      }).catch(() => {}) // Fail silently
    );

    return response;
  },
};

# Deploy the worker
wrangler deploy

# Add your Bot Tracking Key as a secret
wrangler secret put AIS_BOT_KEY