# # robots.txt # # This file is to prevent the crawling and indexing of certain parts # of your site by web crawlers and spiders run by sites like Yahoo! # and Google. By telling these "robots" where not to go on your site, # you save bandwidth and server resources. # # This file will be ignored unless it is at the root of your host: # Used: http://example.com/robots.txt # Ignored: http://example.com/site/robots.txt # # For more information about the robots.txt standard, see: # http://www.robotstxt.org/robotstxt.html User-agent: * Disallow: /eawag/search Disallow: /empa/search Disallow: /psi/search Disallow: /wsl/search Disallow: /eawag/islandora Disallow: /empa/islandora Disallow: /psi/islandora Disallow: /wsl/islandora Disallow: /eawag/form Disallow: /empa/form Disallow: /psi/form Disallow: /wsl/form Disallow: /eawag/advanced-search Disallow: /empa/advanced-search Disallow: /psi/advanced-search Disallow: /wsl/advanced-search Crawl-delay: 10 Crawl-Delay: 10 crawl-delay: 10 User-agent: core Disallow: /eawag/search Disallow: /empa/search Disallow: /psi/search Disallow: /wsl/search Disallow: /eawag/islandora Disallow: /empa/islandora Disallow: /psi/islandora Disallow: /wsl/islandora Disallow: /eawag/form Disallow: /empa/form Disallow: /psi/form Disallow: /wsl/form Disallow: /eawag/advanced-search Disallow: /empa/advanced-search Disallow: /psi/advanced-search Disallow: /wsl/advanced-search User-agent: Amazonbot Disallow: / User-agent: Seekport Crawler Disallow: / User-agent: BLEXBot Disallow: / User-agent: SEO Crawler Disallow: / User-agent: TurnitinBot Disallow: /eawag/search Disallow: /empa/search Disallow: /psi/search Disallow: /wsl/search Disallow: /eawag/form Disallow: /empa/form Disallow: /psi/form Disallow: /wsl/form Disallow: /eawag/advanced-search Disallow: /empa/advanced-search Disallow: /psi/advanced-search Disallow: wsl/advanced-search Disallow: /eawag/islandora Disallow: /empa/islandora Disallow: /psi/islandora Disallow: /wsl/islandora User-agent: GoogleOther Disallow: /eawag/search Disallow: /empa/search Disallow: /psi/search Disallow: /wsl/search Disallow: /eawag/form Disallow: /empa/form Disallow: /psi/form Disallow: /wsl/form Disallow: /eawag/advanced-search Disallow: /empa/advanced-search Disallow: /psi/advanced-search Disallow: /wsl/advanced-search Disallow: /eawag/islandora Disallow: /empa/islandora Disallow: /psi/islandora Disallow: /wsl/islandora User-agent: Googlebot Disallow: /eawag/search Disallow: /empa/search Disallow: /psi/search Disallow: /wsl/search Disallow: /eawag/form Disallow: /empa/form Disallow: /psi/form Disallow: /wsl/form Disallow: /eawag/advanced-search Disallow: /empa/advanced-search Disallow: /psi/advanced-search Disallow: /wsl/advanced-search Disallow: /eawag/islandora Disallow: /empa/islandora Disallow: /psi/islandora Disallow: /wsl/islandora User-agent: AhrefsBot Disallow: /eawag/search Disallow: /empa/search Disallow: /psi/search Disallow: /wsl/search Disallow: /eawag/form Disallow: /empa/form Disallow: /psi/form Disallow: /wsl/form Disallow: /eawag/advanced-search Disallow: /empa/advanced-search Disallow: /psi/advanced-search Disallow: /wsl/advanced-search Disallow: /eawag/islandora Disallow: /empa/islandora Disallow: /psi/islandora Disallow: /wsl/islandora Crawl-delay: 15 User-agent: AwarioBot Disallow: /eawag/search Disallow: /empa/search Disallow: /psi/search Disallow: /wsl/search Disallow: /eawag/form Disallow: /empa/form Disallow: /psi/form Disallow: /wsl/form Disallow: /eawag/advanced-search Disallow: /empa/advanced-search Disallow: /psi/advanced-search Disallow: /wsl/advanced-search Disallow: /eawag/islandora Disallow: /empa/islandora Disallow: /psi/islandora Disallow: /wsl/islandora Crawl-delay: 15 User-agent: bingbot Disallow: /eawag/search Disallow: /empa/search Disallow: /psi/search Disallow: /wsl/search Disallow: /eawag/form Disallow: /empa/form Disallow: /psi/form Disallow: /wsl/form Disallow: /eawag/advanced-search Disallow: /empa/advanced-search Disallow: /psi/advanced-search Disallow: /wsl/advanced-search Disallow: /eawag/islandora Disallow: /empa/islandora Disallow: /psi/islandora Disallow: /wsl/islandora User-agent: YandexBot Disallow: /eawag/search Disallow: /empa/search Disallow: /psi/search Disallow: /wsl/search Disallow: /eawag/form Disallow: /empa/form Disallow: /psi/form Disallow: /wsl/form Disallow: /eawag/advanced-search Disallow: /empa/advanced-search Disallow: /psi/advanced-search Disallow: /wsl/advanced-search Disallow: /eawag/islandora Disallow: /empa/islandora Disallow: /psi/islandora Disallow: /wsl/islandora Crawl-delay: 20 User-agent: TerraCotta Disallow: /eawag/search Disallow: /empa/search Disallow: /psi/search Disallow: /wsl/search Disallow: /eawag/form Disallow: /empa/form Disallow: /psi/form Disallow: /wsl/form Disallow: /eawag/advanced-search Disallow: /empa/advanced-search Disallow: /psi/advanced-search Disallow: /wsl/advanced-search Disallow: /eawag/islandora Disallow: /empa/islandora Disallow: /psi/islandora Disallow: /wsl/islandora Crawl-delay: 20 User-agent: GPTBot Disallow: / # Deny auxiliary/sub-site sitemaps (links already content of main sitemap): Disallow: /eawag/sites/eawag/files/sitemap/sitemap.xml Disallow: /empa/sites/empa/files/sitemap/sitemap.xml Disallow: /psi/sites/psi/files/sitemap/sitemap.xml Disallow: /wsl/sites/wsl/files/sitemap/sitemap.xml # Specifying Sitemap, see: https://www.sitemaps.org/protocol.html Sitemap: https://www.dora.lib4ri.ch/sitemap.xml