robots.txt was 404ing

2023-05-11 09:13:51 -05:00
parent 7dbbf80e72
commit 2f3a2ecfc1
2 changed files with 80 additions and 0 deletions
@@ -0,0 +1,74 @@
+# default
+User-agent: *
+Allow: /
+Disallow: /box/
+Disallow: /diary/entries/raw/
+
+# advertising
+User-agent: AdsBot-Google*
+User-agent: Mediapartners-Google*
+Disallow: /
+
+# trouble, referencing https://aardling.com/robots.txt and others
+User-agent: AhrefsBot
+Disallow: /
+User-agent: DOC
+Disallow: /
+User-agent: Download Ninja
+Disallow: /
+User-agent: EasouSpider
+Disallow: /
+User-agent: Exabot
+Disallow: /
+User-agent: Fetch
+Disallow: /
+User-agent: grub-client
+Disallow: /
+User-agent: HTTrack
+Disallow: /
+User-agent: larbin
+Disallow: /
+User-agent: libwww
+Disallow: /
+User-agent: linko
+Disallow: /
+User-agent: Microsoft.URL.Control
+Disallow: /
+User-agent: MSIECrawler
+Disallow: /
+User-agent: NPBot
+Disallow: /
+User-agent: Offline Explorer
+Disallow: /
+User-agent: sitecheck.internetseer.com
+Disallow: /
+User-agent: SiteSnagger
+Disallow: /
+User-agent: Teleport
+Disallow: /
+User-agent: TeleportPro
+Disallow: /
+User-agent: UbiCrawler
+Disallow: /
+User-agent: Vagabondo
+Disallow: /
+User-agent: WebCopier
+Disallow: /
+User-agent: WebReaper
+Disallow: /
+User-agent: WebStripper
+Disallow: /
+User-agent: WebZIP
+Disallow: /
+User-agent: wget
+Disallow: /
+User-agent: Xenu
+Disallow: /
+User-agent: Zao
+Disallow: /
+User-agent: Zealbot
+Disallow: /
+User-agent: ZyBORG
+Disallow: /
+
+Sitemap: http://www.blessfrey.me/sitemap.xml