# OpenLuxe Robots.txt # Last updated: 2026-06-25 # =========================================== # Google Crawlers # =========================================== # Google does NOT support Crawl-delay directive. # Use Google Search Console to control crawl rate. User-agent: Googlebot Allow: / Disallow: /api/ Disallow: /admin/ Disallow: /dashboard/ Disallow: /settings/ Disallow: /account/ Disallow: /checkout/ Disallow: /cart/ Disallow: /login Disallow: /register Disallow: /password/ Disallow: /email/ Disallow: /notifications Disallow: /messages Disallow: /inbox Disallow: /billing Disallow: /user/ Disallow: /*.json$ Disallow: /*?*sort= Disallow: /*?*filter= Disallow: /admin-login Disallow: /wp-admin Disallow: /wp-login.php Disallow: /administrator Disallow: /phpmyadmin Disallow: /.env Disallow: /config.php Disallow: /backup Disallow: /database.sql Disallow: /api-keys Disallow: /secret-data User-agent: Googlebot-Image Allow: / User-agent: Googlebot-News Allow: /news/ Disallow: /news/feed/ User-agent: Google-Extended Allow: / # =========================================== # Other Search Engine Crawlers # =========================================== User-agent: Bingbot Allow: / Crawl-delay: 1 Disallow: /api/ Disallow: /admin/ Disallow: /dashboard/ Disallow: /settings/ Disallow: /account/ Disallow: /checkout/ Disallow: /cart/ Disallow: /login Disallow: /register Disallow: /password/ Disallow: /email/ Disallow: /notifications Disallow: /messages Disallow: /inbox Disallow: /billing Disallow: /user/ Disallow: /*.json$ Disallow: /*?*sort= Disallow: /*?*filter= Disallow: /admin-login Disallow: /wp-admin Disallow: /wp-login.php Disallow: /administrator Disallow: /phpmyadmin Disallow: /.env Disallow: /config.php Disallow: /backup Disallow: /database.sql Disallow: /api-keys Disallow: /secret-data User-agent: Slurp Allow: / Crawl-delay: 1 Disallow: /api/ Disallow: /admin/ Disallow: /dashboard/ Disallow: /settings/ Disallow: /account/ Disallow: /checkout/ Disallow: /cart/ Disallow: /login Disallow: /register Disallow: /password/ Disallow: /email/ Disallow: /notifications Disallow: /messages Disallow: /inbox Disallow: /billing Disallow: /user/ Disallow: /*.json$ Disallow: /*?*sort= Disallow: /*?*filter= User-agent: DuckDuckBot Allow: / Crawl-delay: 1 Disallow: /api/ Disallow: /admin/ Disallow: /dashboard/ Disallow: /settings/ Disallow: /account/ Disallow: /checkout/ Disallow: /cart/ Disallow: /login Disallow: /register Disallow: /password/ Disallow: /email/ Disallow: /notifications Disallow: /messages Disallow: /inbox Disallow: /billing Disallow: /user/ Disallow: /*.json$ Disallow: /*?*sort= Disallow: /*?*filter= User-agent: Baiduspider Allow: / Crawl-delay: 2 Disallow: /api/ Disallow: /admin/ Disallow: /dashboard/ Disallow: /settings/ Disallow: /account/ Disallow: /checkout/ Disallow: /cart/ Disallow: /login Disallow: /register Disallow: /password/ Disallow: /email/ Disallow: /notifications Disallow: /messages Disallow: /inbox Disallow: /billing Disallow: /user/ Disallow: /*.json$ Disallow: /*?*sort= Disallow: /*?*filter= User-agent: YandexBot Allow: / Crawl-delay: 2 Disallow: /api/ Disallow: /admin/ Disallow: /dashboard/ Disallow: /settings/ Disallow: /account/ Disallow: /checkout/ Disallow: /cart/ Disallow: /login Disallow: /register Disallow: /password/ Disallow: /email/ Disallow: /notifications Disallow: /messages Disallow: /inbox Disallow: /billing Disallow: /user/ Disallow: /*.json$ Disallow: /*?*sort= Disallow: /*?*filter= # =========================================== # Social Media Crawlers (for previews) # =========================================== User-agent: facebookexternalhit Allow: / User-agent: Facebot Allow: / User-agent: Twitterbot Allow: / User-agent: LinkedInBot Allow: / User-agent: Pinterest Allow: / User-agent: WhatsApp Allow: / User-agent: TelegramBot Allow: / User-agent: Slackbot Allow: / User-agent: Discordbot Allow: / # =========================================== # AI Training Bots - ALLOWED # =========================================== User-agent: GPTBot Allow: / Crawl-delay: 2 Disallow: /api/ Disallow: /admin/ Disallow: /dashboard/ Disallow: /settings/ Disallow: /account/ Disallow: /checkout/ Disallow: /cart/ Disallow: /login Disallow: /register Disallow: /password/ Disallow: /email/ Disallow: /notifications Disallow: /messages Disallow: /inbox Disallow: /billing Disallow: /user/ Disallow: /*.json$ Disallow: /*?*sort= Disallow: /*?*filter= User-agent: ChatGPT-User Allow: / User-agent: CCBot Allow: / Crawl-delay: 2 Disallow: /api/ Disallow: /admin/ Disallow: /dashboard/ Disallow: /settings/ Disallow: /account/ Disallow: /checkout/ Disallow: /cart/ Disallow: /login Disallow: /register Disallow: /password/ Disallow: /email/ Disallow: /notifications Disallow: /messages Disallow: /inbox Disallow: /billing Disallow: /user/ Disallow: /*.json$ Disallow: /*?*sort= Disallow: /*?*filter= User-agent: anthropic-ai Allow: / User-agent: Claude-Web Allow: / User-agent: Bytespider Allow: / Crawl-delay: 2 Disallow: /api/ Disallow: /admin/ Disallow: /dashboard/ Disallow: /settings/ Disallow: /account/ Disallow: /checkout/ Disallow: /cart/ Disallow: /login Disallow: /register Disallow: /password/ Disallow: /email/ Disallow: /notifications Disallow: /messages Disallow: /inbox Disallow: /billing Disallow: /user/ Disallow: /*.json$ Disallow: /*?*sort= Disallow: /*?*filter= User-agent: PerplexityBot Allow: / # =========================================== # SEO/Marketing Bots - ALLOWED # =========================================== User-agent: AhrefsBot Allow: / Crawl-delay: 2 Disallow: /api/ Disallow: /admin/ Disallow: /dashboard/ Disallow: /settings/ Disallow: /account/ Disallow: /checkout/ Disallow: /cart/ Disallow: /login Disallow: /register Disallow: /password/ Disallow: /email/ Disallow: /notifications Disallow: /messages Disallow: /inbox Disallow: /billing Disallow: /user/ Disallow: /*.json$ Disallow: /*?*sort= Disallow: /*?*filter= User-agent: SemrushBot Allow: / Crawl-delay: 2 Disallow: /api/ Disallow: /admin/ Disallow: /dashboard/ Disallow: /settings/ Disallow: /account/ Disallow: /checkout/ Disallow: /cart/ Disallow: /login Disallow: /register Disallow: /password/ Disallow: /email/ Disallow: /notifications Disallow: /messages Disallow: /inbox Disallow: /billing Disallow: /user/ Disallow: /*.json$ Disallow: /*?*sort= Disallow: /*?*filter= User-agent: MJ12bot Allow: / Crawl-delay: 2 Disallow: /api/ Disallow: /admin/ Disallow: /dashboard/ Disallow: /settings/ Disallow: /account/ Disallow: /checkout/ Disallow: /cart/ Disallow: /login Disallow: /register Disallow: /password/ Disallow: /email/ Disallow: /notifications Disallow: /messages Disallow: /inbox Disallow: /billing Disallow: /user/ Disallow: /*.json$ Disallow: /*?*sort= Disallow: /*?*filter= User-agent: DotBot Allow: / Crawl-delay: 2 Disallow: /api/ Disallow: /admin/ Disallow: /dashboard/ Disallow: /settings/ Disallow: /account/ Disallow: /checkout/ Disallow: /cart/ Disallow: /login Disallow: /register Disallow: /password/ Disallow: /email/ Disallow: /notifications Disallow: /messages Disallow: /inbox Disallow: /billing Disallow: /user/ Disallow: /*.json$ Disallow: /*?*sort= Disallow: /*?*filter= # =========================================== # Content Scrapers - BLOCKED # =========================================== User-agent: HTTrack Disallow: / User-agent: WebCopier Disallow: / User-agent: WebZIP Disallow: / User-agent: Teleport Disallow: / User-agent: SiteSnagger Disallow: / User-agent: WebStripper Disallow: / User-agent: Offline Explorer Disallow: / User-agent: Wget Disallow: / User-agent: curl Disallow: / # =========================================== # Catch-all for Unknown Bots # =========================================== User-agent: * Allow: / Crawl-delay: 2 Disallow: /api/ Disallow: /admin/ Disallow: /dashboard/ Disallow: /settings/ Disallow: /account/ Disallow: /checkout/ Disallow: /cart/ Disallow: /login Disallow: /register Disallow: /password/ Disallow: /email/ Disallow: /notifications Disallow: /messages Disallow: /inbox Disallow: /billing Disallow: /user/ Disallow: /*.json$ Disallow: /*?*sort= Disallow: /*?*filter= Disallow: /admin-login Disallow: /wp-admin Disallow: /wp-login.php Disallow: /administrator Disallow: /phpmyadmin Disallow: /.env Disallow: /config.php Disallow: /backup Disallow: /database.sql Disallow: /api-keys Disallow: /secret-data # =========================================== # Sitemaps # =========================================== Sitemap: https://openluxe.co/sitemap.xml Sitemap: https://openluxe.co/sitemap_news.xml