[feature] add robots.txt

-Resolves: #6178

1 files changed, 10 insertions, 4 deletions

diff --git a/server/src/leap/soledad/server/_resource.py b/server/src/leap/soledad/server/_resource.py
index e04c0708..7a00ad9a 100644
--- a/server/src/leap/soledad/server/_resource.py
+++ b/server/src/leap/soledad/server/_resource.py
@@ -27,21 +27,27 @@ from ._wsgi import get_sync_resource
 __all__ = ['SoledadResource', 'SoledadAnonResource']
 
 
-class Robots(Resource):
+class _Robots(Resource):
     def render_GET(self, request):
-        return 'robots, go away! please!'
+        return (
+            'User-agent: *\n'
+            'Disallow: /\n'
+            '# you are not a robot, are you???')
 
 
 class SoledadAnonResource(Resource):
+
     """
-    The parts of Soledad Server that unauthenticated users can see
+    The parts of Soledad Server that unauthenticated users can see.
+    This is nice because this means that a non-authenticated user will get 404
+    for anything that is not in this minimal resource tree.
     """
 
     def __init__(self, enable_blobs=False):
         Resource.__init__(self)
         server_info = ServerInfo(enable_blobs)
         self.putChild('', server_info)
-        self.putChild('robots.txt', Robots())
+        self.putChild('robots.txt', _Robots())
 
 
 class SoledadResource(Resource):