From d4b5c4fdc4d7053137ec78e27c34716d1685787c Mon Sep 17 00:00:00 2001 From: bemyak Date: Sun, 12 Jun 2022 22:41:38 +0300 Subject: [PATCH] Make robots.txt compliant with RFC (#1895) The [RFC](http://www.robotstxt.org/orig.html) mentions only `Disallow` directive, so it must appear in the file. `Allow` is an ad hoc agreement between search engines that no all of them follow. --- components/templates/src/builtins/robots.txt | 1 + docs/content/documentation/templates/robots.md | 1 + test_site/templates/robots.txt | 1 + 3 files changed, 3 insertions(+) diff --git a/components/templates/src/builtins/robots.txt b/components/templates/src/builtins/robots.txt index b345a1a10c..b6ac790cee 100644 --- a/components/templates/src/builtins/robots.txt +++ b/components/templates/src/builtins/robots.txt @@ -1,3 +1,4 @@ User-agent: * +Disallow: Allow: / Sitemap: {{ get_url(path="sitemap.xml") }} diff --git a/docs/content/documentation/templates/robots.md b/docs/content/documentation/templates/robots.md index 40a108959b..c93ff73bbb 100644 --- a/docs/content/documentation/templates/robots.md +++ b/docs/content/documentation/templates/robots.md @@ -11,6 +11,7 @@ and the default is what most sites want: ```jinja2 User-agent: * +Disallow: Allow: / Sitemap: {{/* get_url(path="sitemap.xml") */}} ``` diff --git a/test_site/templates/robots.txt b/test_site/templates/robots.txt index 3da696f461..be62e4f6d9 100644 --- a/test_site/templates/robots.txt +++ b/test_site/templates/robots.txt @@ -1,3 +1,4 @@ User-agent: zola +Disallow: Allow: / Sitemap: {{config.base_url}}/sitemap.xml