# robots.txt — Портал «Наш Север»
# Last reviewed: 2026-07-09

# --- Search engines ---
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Bingbot
User-agent: YandexBot
User-agent: YandexImages
User-agent: Applebot
Allow: /
Disallow: /api/
Disallow: /frame/
Disallow: /profile/
Disallow: /impersonate
Disallow: /magic
Disallow: /mobile/

# Collapses tracking and referral params onto the canonical URL so Yandex
# does not treat every shared link as a separate document.
User-agent: Yandex
Clean-param: utm_source&utm_medium&utm_campaign&utm_term&utm_content&ref&referral&linkstat&hotspot

# --- AI search and citation crawlers ---
# The portal is a public information service; being quotable in AI answers is
# the point. These bots must be able to reach us to cite us.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /frame/
Disallow: /profile/

# --- Social preview fetchers ---
User-agent: facebookexternalhit
User-agent: Twitterbot
User-agent: TelegramBot
User-agent: vkShare
Allow: /

# --- Pure extractors: no citation, no traffic, only load ---
User-agent: CCBot
User-agent: Bytespider
User-agent: PetalBot
User-agent: Diffbot
User-agent: AI2Bot
User-agent: Omgilibot
User-agent: cohere-ai
Disallow: /

# --- Everyone else ---
User-agent: *
Allow: /
Disallow: /api/
Disallow: /frame/
Disallow: /profile/
Disallow: /impersonate
Disallow: /magic
Disallow: /mobile/

Sitemap: https://nashsever51.ru/sitemap.xml