# ---- Google Search (Googlebot also fetches pages for Google News) ----
User-agent: Googlebot
# Full, fast access to every public article so fresh news indexes instantly.
# No Crawl-delay: Google ignores it, and throttling would only slow indexing.
Disallow: /admin
Disallow: /livewire/
Disallow: /api/
Disallow: /manage/
Disallow: /search
Disallow: /*/search
Disallow: /up


# ---- Google News crawler (kept explicit for forward compatibility) ---
User-agent: Googlebot-News
# Never block article discovery paths — articles must be crawlable the
# instant they publish. Same private exclusions, nothing more.
Disallow: /admin
Disallow: /livewire/
Disallow: /api/
Disallow: /manage/
Disallow: /search
Disallow: /*/search
Disallow: /up


# ---- Google Images (let hero/article imagery rank in Search & News) --
User-agent: Googlebot-Image
# Media under /storage is intentionally open; only admin areas are blocked.
Disallow: /admin
Disallow: /manage/


# ---- Bing / Bing News ------------------------------------------------
User-agent: Bingbot
# Bing honors Crawl-delay, but news needs rapid crawling — leave unthrottled.
Disallow: /admin
Disallow: /livewire/
Disallow: /api/
Disallow: /manage/
Disallow: /search
Disallow: /*/search
Disallow: /up


# ---- Resource-heavy third-party crawlers (SEO/backlink tools) --------
# Throttled (these bots DO honor Crawl-delay) to protect origin capacity
# at scale. This has no SEO impact — they are not search indexers.
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
Crawl-delay: 10
Disallow: /admin
Disallow: /livewire/
Disallow: /api/
Disallow: /manage/
Disallow: /search
Disallow: /*/search
Disallow: /up


# ---- Every other crawler ---------------------------------------------
User-agent: *
Disallow: /admin
Disallow: /livewire/
Disallow: /api/
Disallow: /manage/
Disallow: /search
Disallow: /*/search
Disallow: /up
# Duplicate/thin variants (tracking params, sort orders, etc.) are handled
# by rel=canonical and on-page noindex — NOT blocked here, to avoid sending
# accidental deindexing signals or hiding canonical news URLs.

# ---- Sitemaps (resolved dynamically from the live request host) ------
# The news sitemap should list only fresh articles (last ~48h) per Google News.
Sitemap: https://energy.auis.edu.krd/sitemap.xml
Sitemap: https://energy.auis.edu.krd/news-sitemap.xml