# =========================================
# robots.txt for Magento 1 / OpenMage - GearIsle
# Site: https://www.gearisle.com
# Updated 2026-07-21:
#  - Faceted /l/ filter pages, ?price= filters and pagination ?p= stay crawlable.
#  - Disallowed internal search (/catalogsearch/), product compare, product reviews,
#    wishlists, tags (/tag/), store/currency parameters, RSS, print views, and limit=all.
#  - Synchronized bot-specific rules (GPTBot, ClaudeBot, etc.) with global disallow patterns.
# =========================================

Sitemap: https://www.gearisle.com/sitemap.xml

# ========================
# Global bot instructions
# ========================
User-agent: *
# -- ALLOW critical resources for rendering --
Allow: /media/js/
Allow: /media/css/
Allow: /media/catalog/product/
Allow: /media/wysiwyg/
Allow: /skin/
Allow: /js/
Allow: /media/

# -- BLOCK sensitive system directories --
Disallow: /app/
Disallow: /downloader/
Disallow: /errors/
Disallow: /includes/
Disallow: /lib/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /shell/
Disallow: /stats/
Disallow: /var/
Disallow: /wp/
Disallow: /migrationnotice/

# -- BLOCK transactional, user, and comparison pages (No SEO Value) --
Disallow: /customer/
Disallow: /sendfriend/
Disallow: /newsletter/
Disallow: /checkout/
Disallow: /contacts/
Disallow: /wishlist/
Disallow: /*wishlist*
Disallow: /catalog/product_compare/
Disallow: /*product_compare*
Disallow: /review/
Disallow: /*review/product/*
Disallow: /tag/
Disallow: /*tag/*

# -- BLOCK internal search engines (Crawl Budget / Spam Trap) --
Disallow: /catalogsearch/
Disallow: /*catalogsearch*

# -- BLOCK unnecessary or duplicate generators --
Disallow: /control/
Disallow: /customize/
Disallow: /rss/
Disallow: /*rss/*
Disallow: /catalog/product/print/
Disallow: /*print=

# -- BLOCK internal scripts and logs --
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /STATUS.txt
Disallow: /get.php

# -- BLOCK harmful session and store URL parameters --
Disallow: /*?SID=
Disallow: /*PHPSESSID
Disallow: /*?*amp%3B
Disallow: /*?*___store=
Disallow: /*?*___from_store=
Disallow: /*?*currency=

# -- BLOCK pure sort/view-mode duplicates & resource-heavy limits --
Disallow: /*?*mode=
Disallow: /*?*dir=
Disallow: /*?*order=
Disallow: /*?*limit=
Disallow: /*limit=all

# =======================================
# AI / LLM & assistant bots
# =======================================
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: PerplexityBot
User-agent: YouBot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Google-Extended
User-agent: Amazonbot
User-agent: cohere-ai
User-agent: Meta-ExternalAgent
User-agent: CCBot
Allow: /
Disallow: /app/
Disallow: /downloader/
Disallow: /errors/
Disallow: /includes/
Disallow: /lib/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /shell/
Disallow: /stats/
Disallow: /var/
Disallow: /wp/
Disallow: /migrationnotice/
Disallow: /customer/
Disallow: /checkout/
Disallow: /newsletter/
Disallow: /sendfriend/
Disallow: /contacts/
Disallow: /wishlist/
Disallow: /*wishlist*
Disallow: /catalog/product_compare/
Disallow: /*product_compare*
Disallow: /review/
Disallow: /*review/product/*
Disallow: /tag/
Disallow: /*tag/*
Disallow: /catalogsearch/
Disallow: /*catalogsearch*
Disallow: /control/
Disallow: /customize/
Disallow: /rss/
Disallow: /*rss/*
Disallow: /catalog/product/print/
Disallow: /*print=
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /STATUS.txt
Disallow: /get.php
Disallow: /*?SID=
Disallow: /*PHPSESSID
Disallow: /*?*amp%3B
Disallow: /*?*___store=
Disallow: /*?*___from_store=
Disallow: /*?*currency=
Disallow: /*?*mode=
Disallow: /*?*dir=
Disallow: /*?*order=
Disallow: /*?*limit=
Disallow: /*limit=all

# Rate-limited but allowed
User-agent: PetalBot
Crawl-delay: 10
Allow: /
Disallow: /*?*mode=
Disallow: /*?*dir=
Disallow: /*?*order=
Disallow: /*?*limit=
Disallow: /*limit=all

# Block known bad/aggressive bots that provide no value
User-agent: Bytespider
Disallow: /

User-agent: DataForSeoBot
Disallow: /