NAS-133520 / 25.04 / Allow configurable endpoints for TNC

src/middlewared/middlewared/alembic/versions/25.04/2025-01-14_15-17_tnc_configurable_endpoints.py

@@ -0,0 +1,36 @@
+"""
+TNC Configurable Endpoints
+
+Revision ID: af75faf3a0e2
+Revises: 799718dc329e
+Create Date: 2025-01-14 15:17:35.553785+00:00
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = 'af75faf3a0e2'
+down_revision = '799718dc329e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('truenas_connect', schema=None) as batch_op:
+        batch_op.add_column(sa.Column(
+            'account_service_base_url', sa.String(length=255), nullable=False,
+            server_default='https://account-service.dev.ixsystems.net/'
+        ))
+        batch_op.add_column(sa.Column(
+            'leca_service_base_url', sa.String(length=255), nullable=False,
+            server_default='https://leca-server.dev.ixsystems.net/'
+        ))
+        batch_op.add_column(sa.Column(
+            'tnc_base_url', sa.String(length=255), nullable=False,
+            server_default='https://truenas.connect.dev.ixsystems.net/'
+        ))
+
+
+def downgrade():
+    pass

src/middlewared/middlewared/api/base/types/__init__.py

@@ -2,4 +2,5 @@
 from .filesystem import *  # noqa
 from .iscsi import *  # noqa
 from .string import *  # noqa
+from .urls import *  # noqa
 from .user import *  # noqa

src/middlewared/middlewared/api/base/types/urls.py

@@ -0,0 +1,8 @@
+from typing import Annotated
+
+from pydantic import AfterValidator, HttpUrl
+
+from middlewared.api.base.validators import https_only_check
+
+
+HttpsOnlyURL = Annotated[HttpUrl, AfterValidator(https_only_check)]

src/middlewared/middlewared/api/base/validators.py

@@ -1,6 +1,8 @@
 from datetime import time
 import re
 
+from pydantic import HttpUrl
+
 
 def match_validator(pattern: re.Pattern, explanation: str | None = None):
     def validator(value: str):
@@ -21,3 +23,9 @@ def time_validator(value: str):
         except TypeError:
             raise ValueError('Time should be in 24 hour format like "18:00"')
     return value
+
+
+def https_only_check(url: HttpUrl) -> str:
+    if url.scheme != 'https':
+        raise ValueError('URL scheme must be https')
+    return str(url)

src/middlewared/middlewared/api/v25_04_0/tn_connect.py

@@ -1,6 +1,7 @@
 from pydantic import IPvAnyAddress
 
 from middlewared.api.base import BaseModel, ForUpdateMetaclass, NonEmptyString, single_argument_args
+from middlewared.api.base.types import HttpsOnlyURL
 
 
 __all__ = [
@@ -17,12 +18,18 @@ class TNCEntry(BaseModel):
     status: NonEmptyString
     status_reason: NonEmptyString
     certificate: int | None
+    account_service_base_url: HttpsOnlyURL
+    leca_service_base_url: HttpsOnlyURL
+    tnc_base_url: HttpsOnlyURL
 
 
 @single_argument_args('tn_connect_update')
 class TNCUpdateArgs(BaseModel, metaclass=ForUpdateMetaclass):
     enabled: bool
     ips: list[IPvAnyAddress]
+    account_service_base_url: HttpsOnlyURL
+    leca_service_base_url: HttpsOnlyURL
+    tnc_base_url: HttpsOnlyURL
 
 
 class TNCUpdateResult(BaseModel):

src/middlewared/middlewared/plugins/acme_protocol_/authenticators/truenas_connect.py

@@ -4,7 +4,7 @@
 
 from middlewared.api.current import TrueNASConnectSchemaArgs
 from middlewared.plugins.truenas_connect.mixin import auth_headers
-from middlewared.plugins.truenas_connect.urls import LECA_DNS_URL, LECA_CLEANUP_URL
+from middlewared.plugins.truenas_connect.urls import get_leca_cleanup_url, get_leca_dns_url
 from middlewared.service import CallError
 
 from .base import Authenticator
@@ -37,8 +37,9 @@ def _perform_internal(self, domain, validation_name, validation_content):
             'Performing %r challenge for %r domain with %r validation name and %r validation content',
             self.NAME, domain, validation_name, validation_content,
         )
+        tnc_config = self.middleware.call_sync('tn_connect.config_internal')
         try:
-            response = requests.post(LECA_DNS_URL, data=json.dumps({
+            response = requests.post(get_leca_dns_url(tnc_config), data=json.dumps({
                 'token': validation_content,
                 'hostnames': [domain],  # We should be using validation name here
             }), headers=auth_headers(self.attributes), timeout=30)
@@ -55,9 +56,10 @@ def _perform_internal(self, domain, validation_name, validation_content):
 
     def _cleanup(self, domain, validation_name, validation_content):
         logger.debug('Cleaning up %r challenge for %r domain', self.NAME, domain)
+        tnc_config = self.middleware.call_sync('tn_connect.config_internal')
         try:
             requests.delete(
-                LECA_CLEANUP_URL, headers=auth_headers(self.attributes), timeout=30, data=json.dumps({
+                get_leca_cleanup_url(tnc_config), headers=auth_headers(self.attributes), timeout=30, data=json.dumps({
                     'hostnames': [validation_name],  # We use validation name here instead of domain as Zack advised
                 })
             )

src/middlewared/middlewared/plugins/truenas_connect/acme.py

@@ -9,7 +9,7 @@
 from .cert_utils import generate_csr, get_hostnames_from_hostname_config
 from .mixin import TNCAPIMixin
 from .status_utils import Status
-from .urls import ACME_CONFIG_URL
+from .urls import get_acme_config_url
 from .utils import CERT_RENEW_DAYS, get_account_id_and_system_id
 
 
@@ -36,7 +36,7 @@ async def config(self):
                 'acme_details': {},
             }
 
-        resp = await self.call(ACME_CONFIG_URL.format(account_id=creds['account_id']), 'get')
+        resp = await self.call(get_acme_config_url(config).format(account_id=creds['account_id']), 'get')
         resp['acme_details'] = resp.pop('response')
         if resp['error'] is None:
             resp = normalize_acme_config(resp)

src/middlewared/middlewared/plugins/truenas_connect/finalize_registration.py

@@ -7,7 +7,7 @@
 
 from .mixin import TNCAPIMixin
 from .status_utils import Status
-from .urls import REGISTRATION_FINALIZATION_URI
+from .urls import get_registration_finalization_uri
 from .utils import CLAIM_TOKEN_CACHE_KEY
 
 
@@ -45,7 +45,7 @@ async def registration(self, job):
 
             try:
                 logger.debug('Attempt %r: Polling for TNC registration finalization', try_num)
-                status = await self.poll_once(claim_token, system_id)
+                status = await self.poll_once(claim_token, system_id, config)
             except asyncio.CancelledError:
                 await self.status_update(
                     Status.REGISTRATION_FINALIZATION_TIMEOUT, 'TNC registration finalization polling has been cancelled'
@@ -98,9 +98,9 @@ async def registration(self, job):
             await asyncio.sleep(self.POLLING_GAP_MINUTES * 60)
             config = await self.middleware.call('tn_connect.config')
 
-    async def poll_once(self, claim_token, system_id):
+    async def poll_once(self, claim_token, system_id, tnc_config):
         return await self._call(
-            REGISTRATION_FINALIZATION_URI, 'post',
+            get_registration_finalization_uri(tnc_config), 'post',
             payload={'system_id': system_id, 'claim_token': claim_token},
         )
 

src/middlewared/middlewared/plugins/truenas_connect/hostname.py

@@ -3,7 +3,7 @@
 from middlewared.service import CallError, Service
 
 from .mixin import TNCAPIMixin
-from .urls import HOSTNAME_URL
+from .urls import get_hostname_url
 from .utils import get_account_id_and_system_id
 
 
@@ -32,7 +32,7 @@ async def config(self):
                 'hostname_configured': False,
             }
 
-        resp = (await self.call(HOSTNAME_URL.format(**creds), 'get')) | {'base_domain': None}
+        resp = (await self.call(get_hostname_url(config).format(**creds), 'get')) | {'base_domain': None}
         resp['hostname_details'] = resp.pop('response')
         for domain in resp['hostname_details']:
             if len(domain.rsplit('.', maxsplit=4)) == 5 and domain.startswith('*.'):
@@ -54,5 +54,5 @@ async def register_update_ips(self, ips=None):
 
         creds = get_account_id_and_system_id(tnc_config)
         return await self.call(
-            HOSTNAME_URL.format(**creds), 'put', payload={'ips': ips},
+            get_hostname_url(tnc_config).format(**creds), 'put', payload={'ips': ips},
         )

src/middlewared/middlewared/plugins/truenas_connect/register.py

@@ -10,7 +10,7 @@
 from middlewared.service import CallError, Service
 
 from .status_utils import Status
-from .urls import REGISTRATION_URI
+from .urls import get_registration_uri
 from .utils import CLAIM_TOKEN_CACHE_KEY
 
 
@@ -86,4 +86,4 @@ async def get_registration_uri(self):
             'token': claim_token,
         }
 
-        return f'{REGISTRATION_URI}?{urlencode(query_params)}'
+        return f'{get_registration_uri(config)}?{urlencode(query_params)}'

src/middlewared/middlewared/plugins/truenas_connect/update.py

@@ -8,7 +8,7 @@
 
 from .mixin import TNCAPIMixin
 from .status_utils import Status
-from .urls import ACCOUNT_SERVICE_URL
+from .urls import get_account_service_url
 from .utils import CLAIM_TOKEN_CACHE_KEY, get_account_id_and_system_id
 
 
@@ -25,6 +25,15 @@ class TrueNASConnectModel(sa.Model):
     ips = sa.Column(sa.JSON(list), nullable=False)
     status = sa.Column(sa.String(255), default=Status.DISABLED.name, nullable=False)
     certificate_id = sa.Column(sa.ForeignKey('system_certificate.id'), index=True, nullable=True)
+    account_service_base_url = sa.Column(
+        sa.String(255), nullable=False, default='https://account-service.dev.ixsystems.net/'
+    )
+    leca_service_base_url = sa.Column(
+        sa.String(255), nullable=False, default='https://leca-server.dev.ixsystems.net/'
+    )
+    tnc_base_url = sa.Column(
+        sa.String(255), nullable=False, default='https://truenas.connect.dev.ixsystems.net/'
+    )
 
 
 class TrueNASConnectService(ConfigService, TNCAPIMixin):
@@ -70,6 +79,13 @@ async def validate_data(self, old_config, data):
                 'IPs cannot be changed when TrueNAS Connect is in a state other than disabled or completely configured'
             )
 
+        if data['enabled'] and old_config['enabled']:
+            for k in ('account_service_base_url', 'leca_service_base_url', 'tnc_base_url'):
+                if data[k] != old_config[k]:
+                    verrors.add(
+                        f'tn_connect_update.{k}', 'This field cannot be changed when TrueNAS Connect is enabled'
+                    )
+
         verrors.check()
 
     @api_method(TNCUpdateArgs, TNCUpdateResult)
@@ -79,9 +95,13 @@ async def do_update(self, data):
         """
         config = await self.config()
         data = config | data
+
         await self.validate_data(config, data)
 
-        db_payload = {'enabled': data['enabled'], 'ips': data['ips']}
+        db_payload = {
+            'enabled': data['enabled'],
+            'ips': data['ips'],
+        } | {k: data[k] for k in ('account_service_base_url', 'leca_service_base_url', 'tnc_base_url')}
         if config['enabled'] is False and data['enabled'] is True:
             # Finalization registration is triggered when claim token is generated
             # We make sure there is no pending claim token
@@ -130,7 +150,7 @@ async def unset_registration_details(self):
 
         # We need to revoke the user account now
         response = await self._call(
-            ACCOUNT_SERVICE_URL.format(**creds), 'delete', headers=await self.auth_headers(config),
+            get_account_service_url(config).format(**creds), 'delete', headers=await self.auth_headers(config),
         )
         if response['error']:
             raise CallError(f'Failed to revoke account: {response["error"]}')

src/middlewared/middlewared/plugins/truenas_connect/urls.py

@@ -1,12 +1,29 @@
 import urllib.parse
 
 
-ACCOUNT_SERVICE_BASE_URL = 'https://account-service.dev.ixsystems.net/'
-ACME_CONFIG_URL = urllib.parse.urljoin(ACCOUNT_SERVICE_BASE_URL, 'v1/accounts/{account_id}/acme')
-BASE_URL = 'https://truenas.connect.dev.ixsystems.net/'
-ACCOUNT_SERVICE_URL = urllib.parse.urljoin(ACCOUNT_SERVICE_BASE_URL, 'v1/accounts/{account_id}/systems/{system_id}/')
-HOSTNAME_URL = urllib.parse.urljoin(ACCOUNT_SERVICE_URL, 'hostnames/')
-LECA_DNS_URL = 'https://leca-server.dev.ixsystems.net/v1/dns-challenge'
-LECA_CLEANUP_URL = 'https://leca-server.dev.ixsystems.net/v1/hostnames'
-REGISTRATION_URI = urllib.parse.urljoin(BASE_URL, 'system/register')
-REGISTRATION_FINALIZATION_URI = urllib.parse.urljoin(ACCOUNT_SERVICE_BASE_URL, 'v1/systems/finalize')
+def get_acme_config_url(tnc_config: dict) -> str:
+    return urllib.parse.urljoin(tnc_config['account_service_base_url'], 'v1/accounts/{account_id}/acme')
+
+
+def get_account_service_url(tnc_config: dict) -> str:
+    return urllib.parse.urljoin(tnc_config['account_service_base_url'], 'v1/accounts/{account_id}/systems/{system_id}/')
+
+
+def get_hostname_url(tnc_config: dict) -> str:
+    return urllib.parse.urljoin(get_account_service_url(tnc_config), 'hostnames/')
+
+
+def get_leca_dns_url(tnc_config: dict) -> str:
+    return urllib.parse.urljoin(tnc_config['leca_service_base_url'], 'v1/dns-challenge')
+
+
+def get_leca_cleanup_url(tnc_config: dict) -> str:
+    return urllib.parse.urljoin(tnc_config['leca_service_base_url'], 'v1/hostnames')
+
+
+def get_registration_uri(tnc_config: dict) -> str:
+    return urllib.parse.urljoin(tnc_config['tnc_base_url'], f'system/register')
+
+
+def get_registration_finalization_uri(tnc_config: dict) -> str:
+    return urllib.parse.urljoin(tnc_config['account_service_base_url'], 'v1/systems/finalize')