# Robots.txt for SolarWijzer.nl - Dutch Solar Panel Lead Generation Website
# Optimized for search engine crawling and indexing

User-agent: *
Allow: /

# Sitemap location
Sitemap: https://solarwijzer.nl/sitemap.xml

# Allow crawling of important assets for better page speed scores
Allow: /*.css$
Allow: /*.js$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.webp$
Allow: /*.svg$
Allow: /*.gif$
Allow: /*.woff$
Allow: /*.woff2$
Allow: /*.ttf$
Allow: /*.eot$

# Disallow admin areas and sensitive paths
Disallow: /admin/
Disallow: /api/private/
Disallow: /config/
Disallow: /tmp/
Disallow: /cache/
Disallow: /.env
Disallow: /node_modules/
Disallow: /src/
Disallow: /dist/
Disallow: /*.log$

# Disallow query parameters that don't add value for SEO
Disallow: /*?utm_*
Disallow: /*?ref=*
Disallow: /*?source=*
Disallow: /*?campaign=*
Disallow: /*?fbclid=*
Disallow: /*?gclid=*

# Disallow duplicate content patterns
Disallow: /*?sort=*
Disallow: /*?filter=*
Disallow: /*?page=*
Disallow: /*?limit=*

# Disallow test/development pages
Disallow: /test/
Disallow: /dev/
Disallow: /staging/
Disallow: /preview/

# Special instructions for major search engines

# Google
User-agent: Googlebot
Allow: /

# Bing
User-agent: Bingbot
Allow: /

# Yandex (some Dutch users may use this)
User-agent: YandexBot
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Baidu (block as not relevant for Dutch market)
User-agent: Baiduspider
Disallow: /

# Block some aggressive crawlers
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# Crawl delay for well-behaved bots (1 second is reasonable)
Crawl-delay: 1

# Host directive (helps with canonicalization)
Host: solarwijzer.nl