User-agent: *
Disallow: /admin/
Disallow: /data/
Disallow: /xml/
Disallow: /arch/
Disallow: /css/
Disallow: /js/
Disallow: /404.php
Disallow: /zoek/
Disallow: /zoeken/
Disallow: *.pdf$
Disallow: *.epub$
Disallow: /nieuws/xml.php # exclude .xml
Disallow: /nieuws/text.php # exclude .txt
Disallow: *.zip$
Disallow: /extern/ # exclude .csv
Disallow: *.txt$
Disallow: /titels/gebruiksvoorwaarden.php
Sitemap: https://www.dbnl.org/sitemap/sitemap_index.xml

User-agent: slurp
Crawl-delay: 5

User-agent: msnbot
Crawl-delay: 5

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /