# robots.txt for https://automaty.net/ User-agent: * Allow: / # Sitemap location Sitemap: https://automaty.net/sitemap.xml # Crawl-delay for polite crawling Crawl-delay: 1 # Specific rules for major search engines User-agent: Googlebot Allow: / Crawl-delay: 0 User-agent: Googlebot-Image Allow: / User-agent: Bingbot Allow: / Crawl-delay: 1 User-agent: Slurp Allow: / # Block access to admin or private directories (if any in future) Disallow: /admin/ Disallow: /private/ Disallow: /*.json$ # Allow all images and CSS Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.png$ Allow: /*.webp$ Allow: /*.gif$ Allow: /*.css$ Allow: /*.js$