User-agent: *
Disallow: /api/
Disallow: /_next/
Disallow: /admin/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /search/
Disallow: /*?*  # Prevents duplicate content from query parameters
Allow: /_next/static/  # Allows crawling of Next.js static assets

# Allow all language directories (except English, which is root)
Allow: /it/
Allow: /fr/
Allow: /de/
Allow: /ar/

# Sitemap
Sitemap: https://beauty.esthetichubs.com/sitemap.xml

# Prevent internal search pages from being indexed
User-agent: Googlebot
Disallow: /search/

User-agent: Bingbot
Disallow: /search/

# Allow major search engines to crawl your site
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Yandex
Allow: /

User-agent: Baiduspider
Allow: /

# Block known scrapers and bots
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

# Crawl-delay (optional, used by some bots)
Crawl-delay: 10  # Google ignores this but Bing/Yandex may respect it.