From 85ab32f50c32eb7109ae9691d517a385a2df795d Mon Sep 17 00:00:00 2001 From: chimchooree Date: Sun, 4 Oct 2020 19:34:48 -0500 Subject: [PATCH] made a robots.txt, I think --- src/robots.txt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/robots.txt b/src/robots.txt index e69de29..f91046e 100644 --- a/src/robots.txt +++ b/src/robots.txt @@ -0,0 +1,16 @@ +# Group 1 +User-agent: * +Disallow: /box/ + +# Group 2 +User-agent: AdsBot-Google +User-agent: Mediapartners-Google +User-agent: AdsBot-Google-Mobile +User-agent: AdsBot-Google-Mobile-Apps +Disallow: / + +# Group 3 +User-agent: * +Allow: / + +Sitemap: http://www.example.com/sitemap.xml