Merge pull request #3291 from dhalbert/robots.txt

add robots.txt to specify doc versions to appear in search engines

2 files changed, 7 insertions, 1 deletions

diff --git a/conf.py b/conf.py
index 1a90b617b..590e5bc2c 100644
--- a/conf.py
+++ b/conf.py
@@ -276,7 +276,7 @@ html_static_path = ['docs/static']
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.
-#html_extra_path = []
+html_extra_path = ["docs/robots.txt"]
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
diff --git a/docs/robots.txt b/docs/robots.txt
new file mode 100644
index 000000000..ad3189d42
--- /dev/null
+++ b/docs/robots.txt
@@ -0,0 +1,6 @@
+User-agent: *
+Allow: /*/latest/
+Allow: /en/latest/   # Fallback for bots that don't understand wildcards
+Allow: /*/6.0.x/
+Allow: /en/6.0.x/    # Fallback for bots that don't understand wildcards
+Disallow: /