added sitemap
parent
85ab32f50c
commit
fc43c6b711
@ -1,16 +1,74 @@
|
|||||||
# Group 1
|
# default
|
||||||
User-agent: *
|
User-agent: *
|
||||||
|
Allow: /
|
||||||
Disallow: /box/
|
Disallow: /box/
|
||||||
|
Disallow: /credits/
|
||||||
|
|
||||||
# Group 2
|
# advertising
|
||||||
User-agent: AdsBot-Google
|
User-agent: AdsBot-Google*
|
||||||
User-agent: Mediapartners-Google
|
User-agent: Mediapartners-Google*
|
||||||
User-agent: AdsBot-Google-Mobile
|
|
||||||
User-agent: AdsBot-Google-Mobile-Apps
|
|
||||||
Disallow: /
|
Disallow: /
|
||||||
|
|
||||||
# Group 3
|
# trouble, referencing https://aardling.com/robots.txt and others
|
||||||
User-agent: *
|
User-agent: AhrefsBot
|
||||||
Allow: /
|
Disallow: /
|
||||||
|
User-agent: DOC
|
||||||
|
Disallow: /
|
||||||
|
User-agent: Download Ninja
|
||||||
|
Disallow: /
|
||||||
|
User-agent: EasouSpider
|
||||||
|
Disallow: /
|
||||||
|
User-agent: Exabot
|
||||||
|
Disallow: /
|
||||||
|
User-agent: Fetch
|
||||||
|
Disallow: /
|
||||||
|
User-agent: grub-client
|
||||||
|
Disallow: /
|
||||||
|
User-agent: HTTrack
|
||||||
|
Disallow: /
|
||||||
|
User-agent: larbin
|
||||||
|
Disallow: /
|
||||||
|
User-agent: libwww
|
||||||
|
Disallow: /
|
||||||
|
User-agent: linko
|
||||||
|
Disallow: /
|
||||||
|
User-agent: Microsoft.URL.Control
|
||||||
|
Disallow: /
|
||||||
|
User-agent: MSIECrawler
|
||||||
|
Disallow: /
|
||||||
|
User-agent: NPBot
|
||||||
|
Disallow: /
|
||||||
|
User-agent: Offline Explorer
|
||||||
|
Disallow: /
|
||||||
|
User-agent: sitecheck.internetseer.com
|
||||||
|
Disallow: /
|
||||||
|
User-agent: SiteSnagger
|
||||||
|
Disallow: /
|
||||||
|
User-agent: Teleport
|
||||||
|
Disallow: /
|
||||||
|
User-agent: TeleportPro
|
||||||
|
Disallow: /
|
||||||
|
User-agent: UbiCrawler
|
||||||
|
Disallow: /
|
||||||
|
User-agent: Vagabondo
|
||||||
|
Disallow: /
|
||||||
|
User-agent: WebCopier
|
||||||
|
Disallow: /
|
||||||
|
User-agent: WebReaper
|
||||||
|
Disallow: /
|
||||||
|
User-agent: WebStripper
|
||||||
|
Disallow: /
|
||||||
|
User-agent: WebZIP
|
||||||
|
Disallow: /
|
||||||
|
User-agent: wget
|
||||||
|
Disallow: /
|
||||||
|
User-agent: Xenu
|
||||||
|
Disallow: /
|
||||||
|
User-agent: Zao
|
||||||
|
Disallow: /
|
||||||
|
User-agent: Zealbot
|
||||||
|
Disallow: /
|
||||||
|
User-agent: ZyBORG
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
Sitemap: http://www.example.com/sitemap.xml
|
Sitemap: http://www.blessfrey.me/sitemap.xml
|
||||||
|
@ -0,0 +1,47 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
|
||||||
|
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
||||||
|
|
||||||
|
<url>
|
||||||
|
|
||||||
|
<loc>http://www.blessfrey.me/</loc>
|
||||||
|
|
||||||
|
<lastmod>2020-10-04</lastmod>
|
||||||
|
|
||||||
|
<changefreq>monthly</changefreq>
|
||||||
|
|
||||||
|
<priority>0.7</priority>
|
||||||
|
|
||||||
|
</url>
|
||||||
|
|
||||||
|
<url>
|
||||||
|
|
||||||
|
<loc>http://www.blessfrey.me/diary/</loc>
|
||||||
|
|
||||||
|
<changefreq>weekly</changefreq>
|
||||||
|
|
||||||
|
<priority>0.8</priority>
|
||||||
|
|
||||||
|
</url>
|
||||||
|
|
||||||
|
<url>
|
||||||
|
|
||||||
|
<loc>http://www.blessfrey.me/game/</loc>
|
||||||
|
|
||||||
|
<changefreq>monthly</changefreq>
|
||||||
|
|
||||||
|
<priority>0.6</priority>
|
||||||
|
|
||||||
|
</url>
|
||||||
|
|
||||||
|
<url>
|
||||||
|
|
||||||
|
<loc>http://www.blessfrey.me/presskit/</loc>
|
||||||
|
|
||||||
|
<changefreq>monthly</changefreq>
|
||||||
|
|
||||||
|
<priority>0.6</priority>
|
||||||
|
|
||||||
|
</url>
|
||||||
|
|
||||||
|
</urlset>
|
Loading…
Reference in New Issue