Switch to relative URLs for snapshot/delta

Per mxsasha/nrtmv4#41

docs/admins/configuration.rst

@@ -125,7 +125,6 @@ This sample shows most configuration options
                     key date in base64 PEM here
                     -----END PRIVATE KEY-----
                 nrtm4_server_local_path: /var/www/nrtmv4/authdatabase/
-                nrtm4_server_base_url: https://www.example.net/nrtmv4/authdatabase/
             MIRROR-FIRST:
                 # Run a full import at first, then periodic NRTM updates.
                 authoritative: false
@@ -157,7 +156,7 @@ This sample shows most configuration options
                     - 'ftp://ftp.example.net/mirror-second.db.route.gz'
                     - 'ftp://ftp.example.net/mirror-second.db.route6.gz'
                     - 'ftp://ftp.example.net/mirror-second.db.route-set.gz'
-                nrtm4_client_notification_file_url: https://example.net/nrtmv4/MIRROR-SECOND/update-notification-file.json
+                nrtm4_client_notification_file_url: https://example.net/nrtmv4/MIRROR-SECOND/update-notification-file.jose
                 nrtm4_client_initial_public_key: |
                     -----BEGIN PUBLIC KEY-----
                     key date in base64 PEM here
@@ -637,9 +636,9 @@ Sources
   Note the use of the pipe character (``|``) to enter this multi-line data.
   |br| **Default**: not defined, no NRTMv4 server runs.
   |br| **Change takes effect**: after SIGHUP, at the next mirror update.
-* ``sources.{name}.nrtm4_server_private_key_next``: the next private Ed25519
+* ``sources.{name}.nrtm4_server_private_key_next``: the next private
   key used to sign the Update Notification File for an NRTMv4 server. This
-  setting is used for key rotation. Base64 encoded.
+  setting is used for key rotation. PEM format.
   See the :doc:`mirroring documentation </users/mirroring>` for details on
   key rotation.
   Note the use of the pipe character (``|``) to enter this multi-line data.
@@ -649,10 +648,6 @@ Sources
   writes the repository on the local file system.
   |br| **Default**: not defined, no NRTMv4 server runs.
   |br| **Change takes effect**: after SIGHUP, at the next mirror update.
-* ``sources.{name}.nrtm4_server_base_url``: the HTTPS URL where you will
-  host the files from ``nrtm4_server_local_path``.
-  |br| **Default**: not defined, no NRTMv4 server runs.
-  |br| **Change takes effect**: after SIGHUP, at the next mirror update.
 * ``sources.{name}.nrtm4_server_snapshot_frequency``: the frequency, in
   seconds, at which to generate Snapshot files. Must be between 1 and 24 hours.
   |br| **Default**: 4 hours.

docs/users/mirroring.rst

@@ -69,11 +69,11 @@ both of them at the same time for the same source.
 
 NRTMv4 mode
 ~~~~~~~~~~~
-To configure an NRTMv4 source, you set the ``nrtm4_server_private_key``,
-``nrtm4_server_local_path`` and ``nrtm4_server_base_url`` settings on the
+To configure an NRTMv4 source, you set the ``nrtm4_server_private_key``
+and ``nrtm4_server_local_path`` settings on the
 source. The local path is where IRRD will write the files, the base URL
 is the full HTTPS URL under which you will be serving the files.
-The private key must be an Ed25519 private key in base64. You can use the
+The private key must be an JWK private key in PEM. You can use the
 ``irrdctl nrtmv4 generate-private-key`` command generate such a key,
 though does not store the key in the configuration for you.
 You need to use a separate base URL and local path for each
@@ -97,9 +97,10 @@ When running the NRTMv4 server process, IRRD will:
 * Write the new Update Notification File.
 
 You need to serve the files written to ``nrtm4_server_local_path`` on
-``nrtm4_server_base_url``, so that clients can retrieve
-``{nrtm4_server_base_url}/update-notification-file.json``. This can
-be done using the same nginx instance used for other parts of IRRD,
+HTTPS, so that clients can retrieve them.
+You then tell clients the full URL to ``update-notification-file.jose``
+which IRRD will create in the provided path. You can serve them
+using the same nginx instance used for other parts of IRRD,
 or through an entirely different web server or CDN, depending on your
 scalability needs. So in a way, the actual "serving" part of an
 NRTMv4 server is not performed by IRRD, as it's just static files over HTTPS.

irrd/conf/__init__.py

@@ -450,7 +450,7 @@ def _validate_subconfig(key, value):
                     "nrtm_host, import_source, or nrtm4_client_notification_file_url are set."
                 )
 
-            nrtm4_server_keys = "nrtm4_server_private_key", "nrtm4_server_local_path", "nrtm4_server_base_url"
+            nrtm4_server_keys = "nrtm4_server_private_key", "nrtm4_server_local_path"
             nrtm4_server_enabled = any(details.get(k) for k in nrtm4_server_keys)
 
             if (nrtm4_server_enabled or details.get("nrtm4_server_private_key_next")) and not all(
@@ -480,17 +480,6 @@ def _validate_subconfig(key, value):
                         f"Invalid value for setting nrtm4_server_private_key_next for source {name}: {ve}"
                     )
 
-            url_parsed = urlparse(details.get("nrtm4_server_base_url"))
-            if nrtm4_server_enabled and not any(
-                [
-                    url_parsed.scheme == "https" and url_parsed.netloc,
-                    url_parsed.scheme == "file" and url_parsed.path,
-                ]
-            ):
-                errors.append(
-                    f"Setting nrtm4_server_base_url for source {name} is not a valid https or file URL."
-                )
-
             if details.get("nrtm4_server_local_path") and not os.path.isdir(
                 details["nrtm4_server_local_path"]
             ):
@@ -543,7 +532,7 @@ def _validate_subconfig(key, value):
             if details.get("nrtm_access_list_unfiltered"):
                 expected_access_lists.add(details.get("nrtm_access_list_unfiltered"))
 
-        source_keys_no_duplicates = ["nrtm4_server_local_path", "nrtm4_server_base_url"]
+        source_keys_no_duplicates = ["nrtm4_server_local_path"]
         for key in source_keys_no_duplicates:
             values = [s.get(key) for s in config.get("sources", {}).values()]
             duplicates = [item for item, count in collections.Counter(values).items() if item and count > 1]

irrd/conf/known_keys.py

@@ -106,7 +106,6 @@
     "nrtm4_server_private_key",
     "nrtm4_server_private_key_next",
     "nrtm4_server_local_path",
-    "nrtm4_server_base_url",
     "nrtm4_server_snapshot_frequency",
     "strict_import_keycert_objects",
     "rpki_excluded",

irrd/conf/test_conf.py

@@ -126,7 +126,6 @@ def test_load_valid_reload_valid_config(self, monkeypatch, save_yaml_config, tmp
                         "keep_journal": True,
                         "nrtm4_client_notification_file_url": "https://testhost/",
                         "nrtm4_client_initial_public_key": eckey_public_key_as_str(MOCK_UNF_PRIVATE_KEY),
-                        "nrtm4_server_base_url": "https://example.com",
                         "nrtm4_server_private_key": MOCK_UNF_PRIVATE_KEY_STR,
                         "nrtm4_server_private_key_next": MOCK_UNF_PRIVATE_KEY_OTHER_STR,
                         "nrtm4_server_local_path": str(tmpdir),
@@ -323,7 +322,6 @@ def test_load_invalid_config(self, save_yaml_config, tmpdir):
                         "suspension_enabled": True,
                         "nrtm_query_serial_range_limit": "not-a-number",
                         "nrtm4_client_initial_public_key": "invalid",
-                        "nrtm4_server_base_url": "invalid",
                         "nrtm4_server_private_key": "invalid",
                         "nrtm4_server_private_key_next": "invalid",
                         "nrtm4_server_local_path": str(tmpdir / "invalid"),
@@ -336,14 +334,14 @@ def test_load_invalid_config(self, save_yaml_config, tmpdir):
                         "nrtm_access_list": "invalid-list",
                         "nrtm_query_serial_range_limit": "not-a-number",
                         "nrtm4_client_notification_file_url": "http://invalid",
+                        "nrtm4_server_local_path": str(tmpdir / "invalid"),
                     },
                     "TESTDB3": {
                         "keep_journal": False,
                         "authoritative": True,
                         "import_source": "192.0.2.1",
                         "nrtm_access_list_unfiltered": "invalid-list",
                         "route_object_preference": "not-a-number",
-                        "nrtm4_server_base_url": "invalid",
                     },
                     # Not permitted, rpki.roa_source is set
                     "RPKI": {},
@@ -465,9 +463,6 @@ def test_load_invalid_config(self, save_yaml_config, tmpdir):
 
         assert "Invalid value for setting nrtm4_server_private_key for source TESTDB:" in str(ce.value)
         assert "Invalid value for setting nrtm4_server_private_key_next for source TESTDB:" in str(ce.value)
-        assert "Setting nrtm4_server_base_url for source TESTDB is not a valid https or file URL." in str(
-            ce.value
-        )
         assert (
             "Setting nrtm4_server_local_path for source TESTDB is required and must point to an existing"
             " directory."
@@ -478,14 +473,13 @@ def test_load_invalid_config(self, save_yaml_config, tmpdir):
         )
         assert (
             "When setting any nrtm4_server setting, all of"
-            " nrtm4_server_private_key/nrtm4_server_local_path/nrtm4_server_base_url must be set for source"
-            " TESTDB3."
+            " nrtm4_server_private_key/nrtm4_server_local_path must be set for source"
+            " TESTDB2."
             in str(ce.value)
         )
-        assert "Setting nrtm4_server_base_url for source TESTDB3 is not a valid https or file URL." in str(
+        assert f"Duplicate value(s) {tmpdir}/invalid for source setting nrtm4_server_local_path." in str(
             ce.value
         )
-        assert "Duplicate value(s) invalid for source setting nrtm4_server_base_url." in str(ce.value)
 
 
 class TestGetSetting:

irrd/integration_tests/run.py

@@ -947,7 +947,6 @@ def _start_irrds(self):
             "nrtm_access_list": "localhost",
             "nrtm4_server_private_key": eckey_private_key_as_str(self.nrtm4_private_key),
             "nrtm4_server_local_path": self.nrtm4_dir2,
-            "nrtm4_server_base_url": f"file://{self.nrtm4_dir2}",
             "nrtm4_server_snapshot_frequency": 3600,
         }
         with open(self.config_path2, "w") as yaml_file:

irrd/mirroring/nrtm4/nrtm4_client.py

@@ -1,7 +1,6 @@
 import logging
 import os
 from typing import Any, Dict, List, Optional, Tuple
-from urllib.parse import urlparse
 
 import pydantic
 from joserfc.rfc7515.model import CompactSignature
@@ -55,6 +54,7 @@ def __init__(self, source: str, database_handler: DatabaseHandler):
         self.source = source
         self.database_handler = database_handler
         self.rpki_aware = bool(get_setting("rpki.roa_source"))
+        self.notification_file_url = get_setting(f"sources.{self.source}.nrtm4_client_notification_file_url")
         self.force_reload, self.last_status = self._current_db_status()
 
     def run_client(self) -> bool:
@@ -110,12 +110,11 @@ def _retrieve_unf(self) -> Tuple[NRTM4UpdateNotificationFile, Optional[str]]:
         Retrieve, verify and parse the Update Notification File.
         Returns the UNF object and the used key in base64 string.
         """
-        notification_file_url = get_setting(f"sources.{self.source}.nrtm4_client_notification_file_url")
-        if not notification_file_url:  # pragma: no cover
+        if not self.notification_file_url:  # pragma: no cover
             raise RuntimeError("NRTM4 client called for a source without a Update Notification File URL")
 
-        unf_signed, _ = retrieve_file(notification_file_url, return_contents=True)
-        if "nrtm.db.ripe.net" in notification_file_url:  # pragma: no cover
+        unf_signed, _ = retrieve_file(self.notification_file_url, return_contents=True)
+        if "nrtm.db.ripe.net" in self.notification_file_url:  # pragma: no cover
             # When removing this, also remove Optional[] from return type
             logger.warning("Expecting raw UNF as source is RIPE*, signature not checked")
             unf_payload = unf_signed.encode("ascii")
@@ -126,7 +125,6 @@ def _retrieve_unf(self) -> Tuple[NRTM4UpdateNotificationFile, Optional[str]]:
         unf = NRTM4UpdateNotificationFile.model_validate_json(
             unf_payload,
             context={
-                "update_notification_file_scheme": urlparse(notification_file_url).scheme,
                 "expected_values": {
                     "source": self.source,
                 },
@@ -255,7 +253,9 @@ def _load_snapshot(self, unf: NRTM4UpdateNotificationFile):
         Deals with the usual things for bulk loading, like deleting old objects.
         """
         snapshot_path, should_delete = retrieve_file(
-            unf.snapshot.url, return_contents=False, expected_hash=unf.snapshot.hash
+            unf.snapshot.full_url(self.notification_file_url),
+            return_contents=False,
+            expected_hash=unf.snapshot.hash,
         )
 
         try:
@@ -308,7 +308,7 @@ def _load_deltas(self, unf: NRTM4UpdateNotificationFile, next_version: int):
             if delta.version < next_version:
                 continue
             delta_path, should_delete = retrieve_file(
-                delta.url, return_contents=False, expected_hash=delta.hash
+                delta.full_url(self.notification_file_url), return_contents=False, expected_hash=delta.hash
             )
             try:
                 delta_file = open(delta_path, "rb")

irrd/mirroring/nrtm4/nrtm4_server.py

@@ -73,7 +73,6 @@ def __init__(
         self.database_handler = database_handler
         self.source = source
         self.path = Path(get_setting(f"sources.{self.source}.nrtm4_server_local_path"))
-        self.base_url = get_setting(f"sources.{self.source}.nrtm4_server_base_url").rstrip("/") + "/"
         self.status_lockfile_path = Path(get_setting("piddir")) / f"nrtm4-server-status-{source}.lock"
         self.snapshot_lockfile_path = Path(get_setting("piddir")) / f"nrtm4-server-snapshot-{source}.lock"
         self.timestamp = datetime.datetime.now(tz=UTC)
@@ -112,7 +111,7 @@ def run(self):
             )
             return
 
-        logger.debug(f"{self.source}: NRTMv4 server preparing to update in {self.path} for {self.base_url}")
+        logger.debug(f"{self.source}: NRTMv4 server preparing to update in {self.path}")
 
         if not self._verify_integrity():
             logger.error(f"{self.source}: integrity check failed, discarding existing session")
@@ -237,13 +236,11 @@ def _write_unf(self) -> None:
             next_signing_key=next_signing_public_key,
             snapshot=NRTM4FileReference(
                 version=self.status.last_snapshot_version,
-                url=self.base_url + self.status.last_snapshot_filename,
+                url=self.status.last_snapshot_filename,
                 hash=self.status.last_snapshot_hash,
             ),
             deltas=[
-                NRTM4FileReference(
-                    version=delta["version"], url=self.base_url + delta["filename"], hash=delta["hash"]
-                )
+                NRTM4FileReference(version=delta["version"], url=delta["filename"], hash=delta["hash"])
                 for delta in self.status.previous_deltas
             ],
         )

irrd/mirroring/nrtm4/nrtm4_types.py

@@ -1,13 +1,17 @@
 import datetime
 from functools import cached_property
+from pathlib import Path
 from typing import Any, List, Literal, Optional
+from urllib.parse import urlparse, urlunparse
 from uuid import UUID
 
 import pydantic
 from joserfc.rfc7518.ec_key import ECKey
 from pytz import UTC
 from typing_extensions import Self
 
+from irrd.mirroring.nrtm4 import UPDATE_NOTIFICATION_FILENAME
+
 
 def get_from_pydantic_context(info: pydantic.ValidationInfo, key: str) -> Optional[Any]:
     """
@@ -71,21 +75,21 @@ class NRTM4FileReference(pydantic.main.BaseModel):
     """
 
     version: pydantic.PositiveInt
-    url: pydantic.AnyUrl
+    url: Path
     hash: str
 
-    @pydantic.field_validator("url")
-    @classmethod
-    def validate_url(cls, url, info: pydantic.ValidationInfo):
-        update_notification_file_scheme = get_from_pydantic_context(info, "update_notification_file_scheme")
-        if not update_notification_file_scheme:
-            return url
-        if url.scheme != update_notification_file_scheme:
-            raise ValueError(
-                f"Invalid scheme in file reference: expected {update_notification_file_scheme}, found"
-                f" {url.scheme}"
+    def full_url(self, unf_url: str) -> str:
+        unf_path = urlparse(unf_url)
+        return urlunparse(
+            (
+                unf_path.scheme,
+                unf_path.hostname,
+                unf_path.path.replace(UPDATE_NOTIFICATION_FILENAME, str(self.url)),
+                "",
+                "",
+                "",
             )
-        return url
+        )
 
 
 class NRTM4UpdateNotificationFile(NRTM4Common):

irrd/mirroring/nrtm4/tests/test_nrtm4_client.py

@@ -34,18 +34,18 @@
     "version": 4,
     "snapshot": {
         "version": 3,
-        "url": MOCK_SNAPSHOT_URL,
+        "url": MOCK_SNAPSHOT_URL.split("/")[-1],
         "hash": MOCK_SNAPSHOT_URL,
     },
     "deltas": [
         {
             "version": 3,
-            "url": MOCK_DELTA3_URL,
+            "url": MOCK_DELTA3_URL.split("/")[-1],
             "hash": MOCK_DELTA3_URL,
         },
         {
             "version": 4,
-            "url": MOCK_DELTA4_URL,
+            "url": MOCK_DELTA4_URL.split("/")[-1],
             "hash": MOCK_DELTA4_URL,
         },
     ],

irrd/mirroring/nrtm4/tests/test_nrtm4_server.py

@@ -24,8 +24,6 @@
 from irrd.utils.test_utils import MockDatabaseHandler
 from irrd.utils.text import remove_auth_hashes
 
-BASE_URL = "https://example.com/"
-
 
 class TestNRTM4Server:
     def test_server(self, monkeypatch):
@@ -65,7 +63,6 @@ def test_nrtm4_server(self, tmpdir, config_override):
                     "TEST": {
                         "nrtm4_server_private_key": MOCK_UNF_PRIVATE_KEY_STR,
                         "nrtm4_server_local_path": str(nrtm_path),
-                        "nrtm4_server_base_url": BASE_URL,
                         # "nrtm4_server_snapshot_frequency": 0,
                     }
                 },
@@ -100,7 +97,6 @@ def test_nrtm4_server(self, tmpdir, config_override):
         assert unf["deltas"] == []
         assert "next_signing_key" not in unf
         assert unf["snapshot"]["version"] == 1
-        assert unf["snapshot"]["url"].startswith(BASE_URL)
 
         snapshot_filename = unf["snapshot"]["url"].split("/")[-1]
         with gzip.open(nrtm_path / snapshot_filename, "rb") as snapshot_file:

irrd/mirroring/nrtm4/tests/test_nrtm4_types.py

@@ -169,17 +169,6 @@ def test_invalid_missing_snapshot(self):
             )
         assert "snapshot: Field required" in format_pydantic_errors(ve.value)
 
-    @pytest.mark.freeze_time("2022-03-14 12:34:56")
-    def test_invalid_url_scheme(self):
-        data = copy.deepcopy(self.valid_data)
-        data["deltas"][1]["url"] = "file:///filename"
-        with pytest.raises(pydantic.ValidationError) as ve:
-            NRTM4UpdateNotificationFile.model_validate(
-                data,
-                context=self.valid_context,
-            )
-        assert "Invalid scheme" in str(ve)
-
     @pytest.mark.freeze_time("2022-03-14 12:34:56")
     def test_invalid_unf_older_than_snapshot(self):
         data = copy.deepcopy(self.valid_data)

irrd/mirroring/tests/test_scheduler.py

@@ -326,7 +326,6 @@ def test_scheduler_runs_nrtm4_server(self, monkeypatch, config_override):
                 "rpki": {"roa_source": None},
                 "sources": {
                     "TEST": {
-                        "nrtm4_server_base_url": "https://example.com",
                         "nrtm4_server_private_key": "FalXchs8HIU22Efc3ipNcxVwYwB+Mp0x9TCM9BFtig0=",
                         "nrtm4_server_private_key_next": "4YDgaXpRDIU8vJbFYeYgPQqEa4YAdHeRF1s6SLdXCsE=",
                     }