made a robots.txt, I think

2020-10-04 19:34:48 -05:00
parent 79b1ba35c2
commit 85ab32f50c
1 changed files with 16 additions and 0 deletions
@@ -0,0 +1,16 @@
+# Group 1
+User-agent: *
+Disallow: /box/
+
+# Group 2
+User-agent: AdsBot-Google
+User-agent: Mediapartners-Google
+User-agent: AdsBot-Google-Mobile
+User-agent: AdsBot-Google-Mobile-Apps
+Disallow: /
+
+# Group 3
+User-agent: *
+Allow: /
+
+Sitemap: http://www.example.com/sitemap.xml