DM-48733: Another template rendering endpoint

Expose template rendering for a GitHub notebook template at a different
path. This path includes the GitHub URL path to the notebook. This is an
additional path, all existing functionality, including the existing
template rendering path, remains unchanged.

We need to deploy Times Square to environments where some users should
not have permissions to execute notebooks, but they should have
permissions to render notebook templates for certain GitHub-based
notebooks. This will let us configure that access via methods that apply
permissions based on URL paths.

src/timessquare/handlers/apitags.py

@@ -12,5 +12,6 @@ class ApiTags(Enum):
 
     pages = "Pages"
     github = "GitHub Notebooks"
+    render = "Notebook Template Rendering"
     pr = "Pull Requests"
     v1 = "v1"

src/timessquare/handlers/v1/endpoints.py

@@ -297,7 +297,7 @@ async def get_page_source(
     "/pages/{page}/rendered",
     summary="Get the unexecuted notebook source with rendered parameters",
     name="get_rendered_notebook",
-    tags=[ApiTags.pages],
+    tags=[ApiTags.pages, ApiTags.render],
     responses={
         404: {"description": "Page not found", "model": ErrorModel},
         422: {"description": "Invalid parameter", "model": ErrorModel},
@@ -307,8 +307,8 @@ async def get_rendered_notebook(
     page: Annotated[str, page_path_parameter],
     context: Annotated[RequestContext, Depends(context_dependency)],
 ) -> PlainTextResponse:
-    """Get a Jupyter Notebook with the parameter values filled in. The
-    notebook is still unexecuted.
+    """Get a Jupyter Notebook by page slug, with the parameter values filled
+    in. The notebook is still unexecuted.
     """
     page_service = context.page_service
     parameters = context.request.query_params
@@ -488,6 +488,50 @@ async def get_page_html_events(
             raise
 
 
+@v1_router.get(
+    "/rendered/github/{display_path:path}",
+    summary=(
+        "Get the unexecuted notebook source with rendered parameters for a "
+        "GitHub-based notebook."
+    ),
+    name="get_github_rendered_notebook",
+    tags=[ApiTags.pages, ApiTags.github, ApiTags.render],
+    responses={
+        404: {"description": "Page not found", "model": ErrorModel},
+        422: {"description": "Invalid parameter", "model": ErrorModel},
+    },
+)
+async def get_github_rendered_notebook(
+    display_path: Annotated[str, display_path_parameter],
+    context: Annotated[RequestContext, Depends(context_dependency)],
+) -> PlainTextResponse:
+    """Get a Jupyter Notebook by GitHub path, with the parameter values filled
+    in. The notebook is still unexecuted.
+
+    This route provides the same functionality as ``get_rendered_notebook``,
+    but exposed at a different path so that different access controls can be
+    applied upstream.
+    """
+    page_service = context.page_service
+    parameters = context.request.query_params
+    async with context.session.begin():
+        try:
+            rendered_notebook = (
+                await page_service.render_page_template_by_display_path(
+                    display_path, parameters
+                )
+            )
+        except PageNotFoundError as e:
+            e.location = ErrorLocation.path
+            e.field_path = ["page"]
+            raise
+        except ParameterSchemaValidationError as e:
+            e.location = ErrorLocation.query
+            e.field_path = [e.parameter]
+            raise
+    return PlainTextResponse(rendered_notebook, media_type="application/json")
+
+
 @v1_router.get(
     "/github",
     summary="Get a tree of GitHub-backed pages",

src/timessquare/handlers/v1/models.py

@@ -114,6 +114,16 @@ class Index(BaseModel):
     ),
 )
 
+github_page_rendered_field: AnyHttpUrl | None = Field(
+    None,
+    examples=["https://example.com/v1/rendered/org/repo/notebook.ipynb"],
+    title="Rendered notebook template URL from GitHub",
+    description=(
+        "The URL for the source notebook rendered with parameter values "
+        "(JSON-formatted), by GitHub URL path."
+    ),
+)
+
 page_html_field = Field(
     ...,
     examples=["https://example.com/v1/pages/summit-weather/html"],
@@ -228,6 +238,10 @@ class GitHubSourceMetadata(BaseModel):
 
     sidecar_path: str = Field(title="Repository path of the sidecar YAML file")
 
+    def display_path(self) -> str:
+        """Return the GitHub url path for this page's source."""
+        return f"{self.owner}/{self.repository}/{self.source_path}"
+
     @classmethod
     def from_domain(cls, *, page: PageModel) -> GitHubSourceMetadata:
         # Checks are to help mypy distinguish a GitHub-based page
@@ -281,6 +295,8 @@ class Page(BaseModel):
 
     rendered_url: AnyHttpUrl = page_rendered_field
 
+    github_rendered_url: AnyHttpUrl | None = github_page_rendered_field
+
     html_url: AnyHttpUrl = page_html_field
 
     html_status_url: AnyHttpUrl = page_html_status_field
@@ -315,8 +331,19 @@ def from_domain(cls, *, page: PageModel, request: Request) -> Page:
 
         if page.github_backed:
             github_metadata = GitHubSourceMetadata.from_domain(page=page)
+
+            display_path = github_metadata.display_path()
+            github_rendered_url = AnyHttpUrl(
+                str(
+                    request.url_for(
+                        "get_github_rendered_notebook",
+                        display_path=display_path,
+                    )
+                )
+            )
         else:
             github_metadata = None
+            github_rendered_url = None
 
         return cls(
             name=page.name,
@@ -337,6 +364,7 @@ def from_domain(cls, *, page: PageModel, request: Request) -> Page:
             rendered_url=AnyHttpUrl(
                 str(request.url_for("get_rendered_notebook", page=page.name))
             ),
+            github_rendered_url=github_rendered_url,
             html_url=AnyHttpUrl(
                 str(request.url_for("get_page_html", page=page.name))
             ),

src/timessquare/services/page.py

@@ -263,6 +263,27 @@ async def render_page_template(
             missing, the default value is used instead.
         """
         page = await self.get_page(name)
+        return await self._render_page_template(page, values)
+
+    async def render_page_template_by_display_path(
+        self, display_path: str, values: Mapping[str, Any]
+    ) -> str:
+        """Render a page's jupyter notebook, with the given parameter values.
+
+        Parameters
+        ----------
+        name : `str`
+            Name (URL slug) of the page.
+        values : `dict`
+            Parameter values, keyed by parameter names. If values are
+            missing, the default value is used instead.
+        """
+        page = await self.get_github_backed_page(display_path)
+        return await self._render_page_template(page, values)
+
+    async def _render_page_template(
+        self, page: PageModel, values: Mapping[str, Any]
+    ) -> str:
         resolved_values = page.resolve_and_validate_values(values)
         return page.render_parameters(resolved_values)
 

tests/data/times-square-demo/demo.yaml

@@ -9,7 +9,7 @@ parameters:
     description: Amplitude
     default: 4
     minimum: 0
-  lambda:
+  lambd:
     type: number
     description: Wavelength
     default: 2

tests/handlers/v1/rendered_test.py

@@ -0,0 +1,117 @@
+"""Tests for the `/v1/rendered... endpoints."""
+
+from datetime import UTC, datetime
+from pathlib import Path
+
+import nbformat
+import pytest
+from httpx import AsyncClient
+from redis.asyncio import Redis
+from safir.arq import MockArqQueue
+from safir.database import create_async_session, create_database_engine
+from structlog import get_logger
+
+from timessquare.config import config
+from timessquare.domain.githubcheckout import NotebookSidecarFile
+from timessquare.domain.page import PageModel
+from timessquare.services.page import PageService
+from timessquare.storage.nbhtmlcache import NbHtmlCacheStore
+from timessquare.storage.noteburstjobstore import NoteburstJobStore
+from timessquare.storage.page import PageStore
+
+
+@pytest.mark.asyncio
+async def test_rendered(client: AsyncClient) -> None:
+    """Test the /v1/rendered APIs with mocked sources (i.e. this does not test
+    syncing content from GitHub, only reading github-backed apges.
+    """
+    data_path = Path(__file__).parent.joinpath("../../data")
+    demo_path = data_path / "demo.ipynb"
+    sidecar_path = data_path / "times-square-demo" / "demo.yaml"
+
+    engine = create_database_engine(
+        str(config.database_url), config.database_password.get_secret_value()
+    )
+    session = await create_async_session(engine)
+
+    redis = Redis.from_url(str(config.redis_url))
+
+    page_service = PageService(
+        page_store=PageStore(session=session),
+        html_cache=NbHtmlCacheStore(redis),
+        job_store=NoteburstJobStore(redis),
+        http_client=client,
+        logger=get_logger(),
+        arq_queue=MockArqQueue(),
+    )
+
+    # In real life, parameters
+    sidecar = NotebookSidecarFile.parse_yaml(sidecar_path.read_text())
+
+    await page_service.add_page_to_store(
+        PageModel(
+            name="1",
+            ipynb=demo_path.read_text(),
+            parameters=sidecar.export_parameters(),
+            title="Demo",
+            date_added=datetime.now(UTC),
+            date_deleted=None,
+            github_owner="lsst-sqre",
+            github_repo="times-square-demo",
+            repository_path_prefix="",
+            repository_display_path_prefix="",
+            repository_path_stem="demo",
+            repository_sidecar_extension=".yaml",
+            repository_source_extension=".ipynb",
+            repository_source_sha="1" * 40,
+            repository_sidecar_sha="1" * 40,
+        )
+    )
+
+    await session.commit()
+
+    # Close the set up database
+    await session.remove()
+    await engine.dispose()
+
+    rendered_url = (
+        f"{config.path_prefix}/v1/"
+        f"rendered/github/lsst-sqre/times-square-demo/demo"
+    )
+
+    r = await client.get(rendered_url)
+    assert r.status_code == 200
+    notebook = nbformat.reads(r.text, as_version=4)
+    assert notebook.cells[0].source == (
+        "# Times Square demo\n"
+        "\n"
+        "Plot parameters:\n"
+        "\n"
+        "- Amplitude: A = 4\n"
+        "- Y offset: y0 = 0\n"
+        "- Wavelength: lambd = 2"
+    )
+    assert notebook.metadata["times-square"]["values"] == {
+        "A": 4,
+        "y0": 0,
+        "lambd": 2,
+    }
+
+    # Render the page template with some parameters set
+    r = await client.get(rendered_url, params={"A": 2})
+    assert r.status_code == 200
+    notebook = nbformat.reads(r.text, as_version=4)
+    assert notebook.cells[0].source == (
+        "# Times Square demo\n"
+        "\n"
+        "Plot parameters:\n"
+        "\n"
+        "- Amplitude: A = 2\n"
+        "- Y offset: y0 = 0\n"
+        "- Wavelength: lambd = 2"
+    )
+    assert notebook.metadata["times-square"]["values"] == {
+        "A": 2,
+        "y0": 0,
+        "lambd": 2,
+    }

tox.ini

@@ -61,7 +61,7 @@ deps =
     -r{toxinidir}/requirements/main.txt
     -r{toxinidir}/requirements/dev.txt
 commands =
-    pytest -vv --cov=timessquare --cov-branch --cov-report= {posargs}
+    pytest -s -vv --cov=timessquare --cov-branch --cov-report= {posargs}
 
 [testenv:coverage-report]
 description = Compile coverage from each test run.