From 2f3a2ecfc11b8f29cdbeabd4305393d98cddbaf7 Mon Sep 17 00:00:00 2001
From: chimchooree <chimchooree@mail.com>
Date: Thu, 11 May 2023 09:13:51 -0500
Subject: [PATCH] robots.txt was 404ing

---
 src/index.py          |  6 ++++
 src/static/robots.txt | 74 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+)
 create mode 100644 src/static/robots.txt

diff --git a/src/index.py b/src/index.py
index 1b08ff7..b65d692 100644
--- a/src/index.py
+++ b/src/index.py
@@ -399,6 +399,7 @@ def serve_font(filename):
 def serve_img(filename):
     return static_file(filename, root='static/img')
 
+# Favico
 @route('/favicon.ico', method='GET')
 def get_favicon():
     return static_file('favicon.ico', root='static/img')
@@ -413,6 +414,11 @@ def serve_xml(filename):
 def download(filename):
     return static_file(filename, root='static/extra', download=filename)
 
+# Robots
+@route('/robots.txt', method='GET')
+def get_robots():
+    return static_file('robots.txt', root='static')
+
 ## Routes ##
 
 # Error Page
diff --git a/src/static/robots.txt b/src/static/robots.txt
new file mode 100644
index 0000000..617187a
--- /dev/null
+++ b/src/static/robots.txt
@@ -0,0 +1,74 @@
+# default
+User-agent: *
+Allow: /
+Disallow: /box/
+Disallow: /diary/entries/raw/
+
+# advertising
+User-agent: AdsBot-Google*
+User-agent: Mediapartners-Google*
+Disallow: /
+
+# trouble, referencing https://aardling.com/robots.txt and others
+User-agent: AhrefsBot
+Disallow: /
+User-agent: DOC
+Disallow: /
+User-agent: Download Ninja
+Disallow: /
+User-agent: EasouSpider
+Disallow: /
+User-agent: Exabot
+Disallow: /
+User-agent: Fetch
+Disallow: /
+User-agent: grub-client
+Disallow: /
+User-agent: HTTrack
+Disallow: /
+User-agent: larbin
+Disallow: /
+User-agent: libwww
+Disallow: /
+User-agent: linko
+Disallow: /
+User-agent: Microsoft.URL.Control
+Disallow: /
+User-agent: MSIECrawler
+Disallow: /
+User-agent: NPBot
+Disallow: /
+User-agent: Offline Explorer
+Disallow: /
+User-agent: sitecheck.internetseer.com
+Disallow: /
+User-agent: SiteSnagger
+Disallow: /
+User-agent: Teleport
+Disallow: /
+User-agent: TeleportPro
+Disallow: /
+User-agent: UbiCrawler
+Disallow: /
+User-agent: Vagabondo
+Disallow: /
+User-agent: WebCopier
+Disallow: /
+User-agent: WebReaper
+Disallow: /
+User-agent: WebStripper
+Disallow: /
+User-agent: WebZIP
+Disallow: /
+User-agent: wget
+Disallow: /
+User-agent: Xenu
+Disallow: /
+User-agent: Zao
+Disallow: /
+User-agent: Zealbot
+Disallow: /
+User-agent: ZyBORG
+Disallow: /
+
+Sitemap: http://www.blessfrey.me/sitemap.xml