#––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––
# VegaProductions.in robots.txt — Optimized for AI & SEO
#––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––
User-agent: *
# Disallow administrative and non-public areas to save crawl budget.
# These paths should also be secured via other means (e.g., authentication).
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-includes/
Disallow: /wp-content/cache/
Disallow: /wp-content/plugins/*/
Disallow: /wp-content/themes/*/
Disallow: /trackback/
Disallow: /comments/feed/
Disallow: /*/feed/$
Disallow: /*/comments/$
Disallow: /*/attachment/$
Disallow: /?s=
Disallow: /search/
Disallow: /author/
Disallow: /tag/
Disallow: /category/
Disallow: /page/
Disallow: /embed/
Disallow: /xmlrpc.php

# Allow necessary files within disallowed directories for proper rendering/functionality
# For /wp-admin/, only admin-ajax.php is typically needed for frontend functionality.
Allow: /wp-admin/admin-ajax.php

# Allow essential WordPress content directories
Allow: /wp-content/uploads/
Allow: /wp-content/themes/*.css
Allow: /wp-content/themes/*.js
Allow: /wp-content/plugins/*.css
Allow: /wp-content/plugins/*.js

# Explicitly allow wp-json and feed for API and content syndication (good for AI/apps)
Allow: /wp-json/
Allow: /feed/

# Allow sitemaps for discovery
Allow: /sitemap.xml
Allow: /sitemap_index.xml

# Enhanced crawlability for structured data and AI agents (Googlebot, GPTBot, CCbot)
# These bots are specifically allowed to crawl broadly where not explicitly disallowed above.
User-agent: Googlebot
Disallow: *?lightbox= # Prevent indexing of lightbox variations if they cause duplicate content
Allow: /wp-json/ # Redundant with general allow, but good to be explicit
Allow: /structured-data/ # Assuming you have specific structured data files/paths

User-agent: GPTBot
Allow: / # Explicitly allowing GPTBot to crawl everything not generally disallowed

User-agent: CCbot
Allow: / # Explicitly allowing Common Crawl bot to crawl everything not generally disallowed

# Specific directives for other bots that may not need full access or need crawl delays
User-agent: AdsBot-Google-Mobile
User-agent: AdsBot-Google
# If these paths are truly sensitive, they need server-side protection.
# This merely suggests not to crawl them for ads-related purposes.
Disallow: /_api/*
Disallow: /_partials*
Disallow: /pro-gallery-webapp/v1/galleries/*

User-agent: PetalBot
Disallow: / # Restricting PetalBot if not desired

# Sitemap declarations
Sitemap: https://www.vegaproductions.in/sitemap.xml
Sitemap: https://www.vegaproductions.in/sitemap_index.xml