# Robots.txt for thinkdata.be
# Personal website and portfolio of ThinkData, company by Kenny Helsens, delivering services on AI, Software Engineering and Biotechnology. Based on Belgium, open to working remotely.

# Allow all web crawlers to access the site
User-agent: *
Allow: /

# Specify sitemap location
Sitemap: https://thinkdata.be/sitemap.xml


# Block access to common administrative and sensitive directories
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /cgi-bin/
Disallow: /private/
Disallow: /temp/
Disallow: /tmp/

# Block access to common file types that shouldn't be indexed
Disallow: /*.log$
Disallow: /*.sql$
Disallow: /*.gz$
Disallow: /*.tar$
Disallow: /*.zip$

# Allow CSS and JS files for proper rendering
Allow: /*.css$
Allow: /*.js$

# Block search and filter pages to avoid duplicate content
Disallow: /*?*
Disallow: /search/
Disallow: /*search*

# Block login and registration pages
Disallow: /login/
Disallow: /register/
Disallow: /account/

# Crawl delay to be respectful to server resources
Crawl-delay: 1