# robots.txt for https://cruisinginthewake.com/ # Last updated: 2026-05-27 # Purpose: Allow full indexing of public content while preventing crawl of # internal assets, admin pages, and development files. # # ───────────────────────────────────────────────────────────────────────────── # AI-bot policy (read this before "fixing" something that looks broken) # ───────────────────────────────────────────────────────────────────────────── # At the live edge, Cloudflare's managed AI-Bot Block prepends a header block # above this file that explicitly Disallows the major AI training crawlers: # GPTBot, ClaudeBot, Google-Extended, CCBot, Bytespider, Amazonbot, # Applebot-Extended, meta-externalagent, CloudflareBrowserRenderingCrawler. # This is intentional and reflects our content-signal policy: # "search=yes, ai-train=no" # i.e., AI engines may retrieve our content for inference-time citation in # answers, but may not use it as training data. # # Crawlers we DO allow (because they are inference-time retrieval, not # training): ChatGPT-User, OAI-SearchBot, PerplexityBot, Claude-Search-User, # Claude-Web, regular Googlebot (which feeds AI Overviews via the standard # search index). These all fall under the "User-agent: *" rules below. # # If you want to revisit the AI-Bot Block, do it in the Cloudflare dashboard # (Security → Bots → AI Audit), NOT in this file — Cloudflare's prepended # block overrides whatever this file says for those specific user agents. # ───────────────────────────────────────────────────────────────────────────── User-agent: * # === Allow key content sections === Allow: /$ Allow: /ports/ Allow: /ships/ Allow: /cruise-lines/ Allow: /restaurants/ Allow: /solo/ Allow: /tools/ Allow: /authors/ # === Allow main pages === Allow: /index.html Allow: /planning.html Allow: /travel.html Allow: /solo.html Allow: /ships.html Allow: /ports.html Allow: /articles.html Allow: /packing-lists.html Allow: /packing.html Allow: /accessibility.html Allow: /drink-packages.html Allow: /drink-calculator.html Allow: /stateroom-check.html Allow: /search.html Allow: /about-us.html Allow: /privacy.html Allow: /terms.html Allow: /internet-at-sea.html # === Disallow internal and development paths === Disallow: /solo/articles/ # article fragments, not full pages Disallow: /assets/ Disallow: /js/ Disallow: /css/ Disallow: /data/ Disallow: /meta/ Disallow: /vendor/ Disallow: /vendors/ Disallow: /images/ Disallow: /videos/ # video metadata, not embeds Disallow: /__MACOSX/ Disallow: /404.html Disallow: /offline.html Disallow: /sw.js # service worker Disallow: /placeholder.md Disallow: /admin/ Disallow: /drafts/ Disallow: /staging/ Disallow: /tmp/ Disallow: /*.php$ # === Disallow legacy/phantom paths (redirects exist but should not crawl) === Disallow: /ships/carnival-cruise-line/ # Legacy path, redirects to /ships/carnival/ Disallow: /restaurants_index_stub.html # Stub page Disallow: /ships/ships.html # Phantom path Disallow: /showmenu:* # Malformed query-as-path URLs Disallow: /drinks.html # Redirect stub → /drink-packages.html Disallow: /restaurants/chops-grille* # Phantom pages → /restaurants/chops.html Disallow: /ships/template.html # Dev-only template (unfilled {{SHIP_SLUG}} placeholders) # === Disallow ship asset directories === Disallow: /ships/*/assets/ Disallow: /ships/*/images/ Disallow: /ships/rcl/assets/ Disallow: /ships/rcl/images/ Disallow: /ships/carnival/assets/ Disallow: /ships/celebrity-cruises/assets/ Disallow: /ships/costa/assets/ Disallow: /ships/cunard/assets/ Disallow: /ships/explora-journeys/assets/ Disallow: /ships/holland-america-line/assets/ Disallow: /ships/msc/assets/ Disallow: /ships/norwegian/assets/ Disallow: /ships/oceania/assets/ Disallow: /ships/princess/assets/ Disallow: /ships/regent/assets/ Disallow: /ships/seabourn/assets/ Disallow: /ships/silversea/assets/ Disallow: /ships/virgin-voyages/assets/ # === Disallow JSON data files (not public content) === Disallow: /*.json$ Allow: /sitemap*.json # === Link previews (Facebook / Messenger) === # facebookexternalhit fetches a page only when someone shares its link, to build the # preview card. It is not Meta's training crawler (meta-externalagent, blocked at the # edge). A named group replaces the * group for this bot, so the voyage companions and # their share images open to it while the rest of /admin/ stays closed. Without this, # Facebook's Sharing Debugger reports a 403 for every companion link. User-agent: facebookexternalhit Allow: /admin/voyage-pwa/ Allow: /assets/social/ Disallow: /admin/ # === Sitemap references === Sitemap: https://cruisinginthewake.com/sitemap.xml # Notes: # - All 388 port pages are indexed (updated 2026-05-27) # - All cruise line ship pages are indexed (15 cruise lines, 296 ships): # Carnival, Celebrity, Costa, Cunard, Explora Journeys, HAL, # MSC, Norwegian, Oceania, Princess, RCL, Regent, Seabourn, # Silversea, Virgin Voyages # - 472 restaurant/venue pages indexed (RCL root + NCL + Virgin + Carnival + MSC subdirs) # - Solo main pages indexed, but /solo/articles/ fragments blocked # - Tools (port tracker, ship tracker, ship size atlas, budget calculator, port day planner) indexed # - Asset directories (ships/*/assets/, /videos/) blocked to prevent JSON indexing # - Updated 2026-01-03: Expanded to 15 cruise lines, 372 ports, 294 ships # - Updated 2026-01-07: Added blocks for legacy carnival-cruise-line path, stub pages, # and other phantom paths causing 404 errors in Google Search Console # - Updated 2026-01-31: Updated counts (380 ports, 297 ships, 404 restaurants, 1,154 sitemap URLs) # - Updated 2026-03-27: GSC audit — added 7 Alaska ports, 68 Carnival/MSC restaurants, # 2 tools to sitemap (1,150 → 1,227 URLs). Updated counts to match actual repo. # - Updated 2026-05-03: Added explicit AI-bot policy header explaining the # Cloudflare-managed training-bot block and the inference-bot allowance. # - Updated 2026-10-04: Let facebookexternalhit read the voyage companions under # /admin/voyage-pwa/ and /assets/social/ so shared links get a preview card.