add robots.txt to specify doc versions to appear in search engines

2 files changed, 7 insertions, 1 deletions

diff --git a/conf.py b/conf.py
index 4a8b72584..f7d028c19 100644
--- a/conf.py
+++ b/conf.py
@@ -272,7 +272,7 @@ html_static_path = ['docs/static']
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.
-#html_extra_path = []
+html_extra_path = ["docs/robots.txt"]
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
diff --git a/docs/robots.txt b/docs/robots.txt
new file mode 100644
index 000000000..39a4eaeed
--- /dev/null
+++ b/docs/robots.txt
@@ -0,0 +1,6 @@
+User-agent: *
+Allow: /*/latest/
+Allow: /en/latest/   # Fallback for bots that don't understand wildcards
+Allow: /*/5.3.x/
+Allow: /en/5.3.x/    # Fallback for bots that don't understand wildcards
+Disallow: /