robots.txt was 404ing

src/index.py

@@ -399,6 +399,7 @@ def serve_font(filename):
 def serve_img(filename):
     return static_file(filename, root='static/img')
 
+# Favico
 @route('/favicon.ico', method='GET')
 def get_favicon():
     return static_file('favicon.ico', root='static/img')
@@ -413,6 +414,11 @@ def serve_xml(filename):
 def download(filename):
     return static_file(filename, root='static/extra', download=filename)
 
+# Robots
+@route('/robots.txt', method='GET')
+def get_robots():
+    return static_file('robots.txt', root='static')
+
 ## Routes ##
 
 # Error Page

src/static/robots.txt

@@ -0,0 +1,74 @@
+# default
+User-agent: *
+Allow: /
+Disallow: /box/
+Disallow: /diary/entries/raw/
+
+# advertising
+User-agent: AdsBot-Google*
+User-agent: Mediapartners-Google*
+Disallow: /
+
+# trouble, referencing https://aardling.com/robots.txt and others
+User-agent: AhrefsBot
+Disallow: /
+User-agent: DOC
+Disallow: /
+User-agent: Download Ninja
+Disallow: /
+User-agent: EasouSpider
+Disallow: /
+User-agent: Exabot
+Disallow: /
+User-agent: Fetch
+Disallow: /
+User-agent: grub-client
+Disallow: /
+User-agent: HTTrack
+Disallow: /
+User-agent: larbin
+Disallow: /
+User-agent: libwww
+Disallow: /
+User-agent: linko
+Disallow: /
+User-agent: Microsoft.URL.Control
+Disallow: /
+User-agent: MSIECrawler
+Disallow: /
+User-agent: NPBot
+Disallow: /
+User-agent: Offline Explorer
+Disallow: /
+User-agent: sitecheck.internetseer.com
+Disallow: /
+User-agent: SiteSnagger
+Disallow: /
+User-agent: Teleport
+Disallow: /
+User-agent: TeleportPro
+Disallow: /
+User-agent: UbiCrawler
+Disallow: /
+User-agent: Vagabondo
+Disallow: /
+User-agent: WebCopier
+Disallow: /
+User-agent: WebReaper
+Disallow: /
+User-agent: WebStripper
+Disallow: /
+User-agent: WebZIP
+Disallow: /
+User-agent: wget
+Disallow: /
+User-agent: Xenu
+Disallow: /
+User-agent: Zao
+Disallow: /
+User-agent: Zealbot
+Disallow: /
+User-agent: ZyBORG
+Disallow: /
+
+Sitemap: http://www.blessfrey.me/sitemap.xml