feat: add robots.txt and favicon by default

stapler/data_dir.py

@@ -16,7 +16,7 @@ class DataDir:
     ]
 
     PATH_REGEX = re.compile(r"^[\w-]+$")
-    NEEDED_FILES: typing.ClassVar[list[str]] = ["favicon.ico"]
+    NEEDED_FILES: typing.ClassVar[list[str]] = ["favicon.ico", "robots.txt"]
 
     def __init__(self, root_path: str) -> None:
         self.logger: logging.Logger = logging.getLogger(self.__class__.__name__)

stapler/handlers.py

@@ -272,7 +272,7 @@ class RequestHandler(http.server.SimpleHTTPRequestHandler, BaseHandler):
     UPDATE_PATH_REGEX = re.compile(r"^\/([\w-]+)\/?$")
     GET_PATH_REGEX = re.compile(r"^\/([\w-]+)($|\/)")
     HOST_PART_REGEX = re.compile(r"^([a-z0-9]|[a-z0-9][a-z0-9-]{,61}[a-z0-9])$")
-    AUTHORIZED_PATHS: typing.ClassVar[list[str]] = ["/favicon.ico"]
+    AUTHORIZED_PATHS: typing.ClassVar[list[str]] = ["/favicon.ico", "/robots.txt"]
     TOKEN_HEADER = "X-Token"  # noqa: S105
     HOST_HEADER = "X-Host"
     HOST_ONLY_HEADER = "X-Host-Only"
@@ -542,6 +542,11 @@ class RequestHandler(http.server.SimpleHTTPRequestHandler, BaseHandler):
                 return super().translate_path(path)
             return ""
         if self.host != self.default_host:
+            if (
+                not (self.root_path / page.path / path).is_file()
+                and path in self.AUTHORIZED_PATHS
+            ):
+                return super().translate_path(path)
             path = f"/{page.path}" + path
         if pathlib.Path(path).name.startswith("."):  # hidden files
             return ""

stapler/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /

tests/test_handlers.py

@@ -1192,6 +1192,21 @@ class TestRequestHandler(BaseHandlerTestCase):
                 None,
             )
 
+    def test_translate_path_with_host_favicon(self) -> None:
+        handler = self._get_handler(headers={"Host": "example.com"})
+        with (
+            self.mock_call(self.registry.get_from_host, ["example.com"], Page("path")),
+            self.patch_call(
+                "http.server.SimpleHTTPRequestHandler.translate_path",
+                ["/favicon.ico"],
+            ),
+            self.seal_mocks(),
+        ):
+            self.assertEqual(
+                handler.translate_path("/favicon.ico"),
+                None,
+            )
+
     def test_translate_path_default_host(self) -> None:
         handler = self._get_handler()
         with (