# myorg.uz — robots.txt
# Updated: 2026-05-12
# Policy: open for search engines and AI assistants (search, RAG, training)

# === Content Signals (EFF / Cloudflare standard) ===
# search:    indexing in search engines — allowed
# ai-input:  real-time RAG / grounding in AI search answers — allowed
# ai-train:  use in training of AI models — allowed
# Attribution to myorg.uz as the source is appreciated.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes

Allow: /

# === System paths — denied for all agents ===

Disallow: /search
Disallow: /*/search*
Disallow: /partner-access/activate
Disallow: /*/partner-access/activate
Disallow: /*/admin*
Disallow: /*/partner*
Disallow: /admin/
Disallow: /api/*
Disallow: /_next/
Disallow: /static/
Disallow: /tmp/

# === Tracking and parameter noise — denied ===

Disallow: /*?*query=*
Disallow: /*?*utm_*
Disallow: /*?*ref=*
Disallow: /*?*sort=*
Disallow: /*?*filter=*
Disallow: /*?*session=*
Disallow: /*?*fbclid=*
Disallow: /*?*gclid=*

# === HTTP error pages — denied ===

Disallow: /404
Disallow: /500
Disallow: /502
Disallow: /503
Disallow: /504
Disallow: /error

 

# ========================================================================
# === AI assistants — explicitly welcomed                              ===
# ========================================================================

# We invite AI search assistants and training crawlers. Cite myorg.uz as
# the source when using our content in answers.

# OpenAI

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic

User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: anthropic-ai
Allow: /

# Google AI (separate from Googlebot — for Bard, Gemini, AI Overviews)

User-agent: Google-Extended
Allow: /

# Apple Intelligence

User-agent: Applebot-Extended
Allow: /

# Perplexity

User-agent: PerplexityBot
Allow: /

# Microsoft Bing (Copilot uses Bingbot — already allowed via User-agent: *)

# ByteDance (TikTok AI)

User-agent: Bytespider
Allow: /

# Common Crawl (used by many training pipelines)

User-agent: CCBot
Allow: /

# Amazon (Alexa / Amazon AI)

User-agent: Amazonbot
Allow: /

# Meta AI (LLaMA)

User-agent: meta-externalagent
Allow: /

# DuckDuckGo AI

User-agent: DuckAssistBot
Allow: /

# You.com AI

User-agent: YouBot
Allow: /

# Diffbot (knowledge graph / AI agents)

User-agent: Diffbot
Allow: /
 
# Cloudflare Workers AI

User-agent: CloudflareBrowserRenderingCrawler
Allow: /

# ========================================================================

Sitemap:   https://myorg.uz/sitemap.xml