From e1a843878e4272334869b1db4a8222ae344f5cb8 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Mon, 17 Aug 2020 09:48:17 -0400 Subject: add robots.txt to specify doc versions to appear in search engines --- docs/robots.txt | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/robots.txt (limited to 'docs') diff --git a/docs/robots.txt b/docs/robots.txt new file mode 100644 index 000000000..39a4eaeed --- /dev/null +++ b/docs/robots.txt @@ -0,0 +1,6 @@ +User-agent: * +Allow: /*/latest/ +Allow: /en/latest/ # Fallback for bots that don't understand wildcards +Allow: /*/5.3.x/ +Allow: /en/5.3.x/ # Fallback for bots that don't understand wildcards +Disallow: / -- cgit v1.2.3 From 6928de0364fa6682bd39b48b1bac9a566773f3fd Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Sun, 10 Jan 2021 13:54:08 -0500 Subject: merge from upstream and update to 6.0.x --- docs/robots.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'docs') diff --git a/docs/robots.txt b/docs/robots.txt index 39a4eaeed..ad3189d42 100644 --- a/docs/robots.txt +++ b/docs/robots.txt @@ -1,6 +1,6 @@ User-agent: * Allow: /*/latest/ Allow: /en/latest/ # Fallback for bots that don't understand wildcards -Allow: /*/5.3.x/ -Allow: /en/5.3.x/ # Fallback for bots that don't understand wildcards +Allow: /*/6.0.x/ +Allow: /en/6.0.x/ # Fallback for bots that don't understand wildcards Disallow: / -- cgit v1.2.3