create _canonical_urls system table (#34431)

create a new system table `_canonical_urls` which will be the source of truth for the canonical urls of an instance.
This data is related to but stored independently from the custom/vanity domains, which are used at the routing layer.
In contrast, the canonical urls are owned by the backend/instance, so they can be used in `process.env.CONVEX_CLOUD_URL` and `process.env.CONVEX_SITE_URL`.

Note the storage of urls instead of domains, since the urls can potentially be `http://` instead of `https://`, or have a `/http` path at the end.

The overall plan is for the backend to own this "canonical url" information, and big brain doesn't care if those get out of sync with the verified custom domains (that big brain owns). These canonical urls will be used as the system env variables and also will be publically accessible so they can be used for `VITE_CONVEX_URL` by the CLI and such.

The table is empty by default, in which case the urls used are the .convex.cloud and .convex.site ones.

GitOrigin-RevId: aaa0bc6e0824d10f47d09c45bbc141d99d771d17

crates/common/src/http/mod.rs

@@ -852,6 +852,7 @@ pub async fn stats_middleware<RM: RouteMapper>(
 pub struct InstanceNameExt(pub String);
 
 #[derive(ToSchema, Copy, Clone, Debug, Eq, PartialEq, Serialize, Deserialize, Ord, PartialOrd)]
+#[cfg_attr(any(test, feature = "testing"), derive(proptest_derive::Arbitrary))]
 #[serde(rename_all = "camelCase")]
 pub enum RequestDestination {
     ConvexCloud,

crates/model/src/canonical_urls/mod.rs

@@ -0,0 +1,102 @@
+use std::{
+    collections::BTreeMap,
+    sync::LazyLock,
+};
+
+use common::{
+    document::{
+        ParsedDocument,
+        ResolvedDocument,
+    },
+    http::RequestDestination,
+    query::{
+        Order,
+        Query,
+    },
+    runtime::Runtime,
+    types::TableName,
+};
+use database::{
+    ResolvedQuery,
+    SystemMetadataModel,
+    Transaction,
+};
+use value::TableNamespace;
+
+use self::types::CanonicalUrl;
+use crate::SystemTable;
+
+pub mod types;
+
+pub static CANONICAL_URLS_TABLE: LazyLock<TableName> = LazyLock::new(|| {
+    "_canonical_urls"
+        .parse()
+        .expect("Invalid built-in table name")
+});
+
+pub struct CanonicalUrlsTable;
+
+impl SystemTable for CanonicalUrlsTable {
+    fn table_name(&self) -> &'static TableName {
+        &CANONICAL_URLS_TABLE
+    }
+
+    fn indexes(&self) -> Vec<crate::SystemIndex> {
+        vec![]
+    }
+
+    fn validate_document(&self, document: ResolvedDocument) -> anyhow::Result<()> {
+        ParsedDocument::<CanonicalUrl>::try_from(document).map(|_| ())
+    }
+}
+
+pub struct CanonicalUrlsModel<'a, RT: Runtime> {
+    tx: &'a mut Transaction<RT>,
+}
+
+impl<'a, RT: Runtime> CanonicalUrlsModel<'a, RT> {
+    pub fn new(tx: &'a mut Transaction<RT>) -> Self {
+        Self { tx }
+    }
+
+    pub async fn get_canonical_urls(
+        &mut self,
+    ) -> anyhow::Result<BTreeMap<RequestDestination, ParsedDocument<CanonicalUrl>>> {
+        let query = Query::full_table_scan(CANONICAL_URLS_TABLE.clone(), Order::Asc);
+        let mut query_stream = ResolvedQuery::new(self.tx, TableNamespace::Global, query)?;
+        let mut canonical_urls = BTreeMap::new();
+        while let Some(document) = query_stream.next(self.tx, None).await? {
+            let canonical_url = ParsedDocument::<CanonicalUrl>::try_from(document)?;
+            canonical_urls.insert(canonical_url.request_destination, canonical_url);
+        }
+        Ok(canonical_urls)
+    }
+
+    pub async fn set_canonical_url(
+        &mut self,
+        request_destination: RequestDestination,
+        url: String,
+    ) -> anyhow::Result<()> {
+        if let Some(existing_canonical_url) =
+            self.get_canonical_urls().await?.get(&request_destination)
+        {
+            if existing_canonical_url.url == url {
+                // Url isn't changing, so no-op.
+                return Ok(());
+            } else {
+                // Delete the existing canonical url.
+                SystemMetadataModel::new_global(self.tx)
+                    .delete(existing_canonical_url.id())
+                    .await?;
+            }
+        }
+        let canonical_url = CanonicalUrl {
+            request_destination,
+            url,
+        };
+        SystemMetadataModel::new_global(self.tx)
+            .insert(&CANONICAL_URLS_TABLE, canonical_url.try_into()?)
+            .await?;
+        Ok(())
+    }
+}

crates/model/src/canonical_urls/types.rs

@@ -0,0 +1,49 @@
+use common::http::RequestDestination;
+use serde::{
+    Deserialize,
+    Serialize,
+};
+use value::codegen_convex_serialization;
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(any(test, feature = "testing"), derive(proptest_derive::Arbitrary))]
+pub struct CanonicalUrl {
+    pub request_destination: RequestDestination,
+    pub url: String,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct SerializedCanonicalUrl {
+    request_destination: String,
+    url: String,
+}
+
+impl From<CanonicalUrl> for SerializedCanonicalUrl {
+    fn from(value: CanonicalUrl) -> Self {
+        Self {
+            request_destination: match value.request_destination {
+                RequestDestination::ConvexCloud => "convexCloud".to_string(),
+                RequestDestination::ConvexSite => "convexSite".to_string(),
+            },
+            url: value.url,
+        }
+    }
+}
+
+impl TryFrom<SerializedCanonicalUrl> for CanonicalUrl {
+    type Error = anyhow::Error;
+
+    fn try_from(value: SerializedCanonicalUrl) -> Result<Self, Self::Error> {
+        Ok(Self {
+            request_destination: match value.request_destination.as_str() {
+                "convexCloud" => RequestDestination::ConvexCloud,
+                "convexSite" => RequestDestination::ConvexSite,
+                _ => anyhow::bail!("Invalid request destination: {}", value.request_destination),
+            },
+            url: value.url,
+        })
+    }
+}
+
+codegen_convex_serialization!(CanonicalUrl, SerializedCanonicalUrl);

crates/model/src/lib.rs

@@ -43,6 +43,7 @@ use backend_state::{
     BackendStateTable,
     BACKEND_STATE_TABLE,
 };
+use canonical_urls::CANONICAL_URLS_TABLE;
 use common::{
     bootstrap_model::index::{
         IndexConfig,
@@ -119,6 +120,7 @@ use value::{
 use crate::{
     auth::AuthTable,
     backend_state::BackendStateModel,
+    canonical_urls::CanonicalUrlsTable,
     cron_jobs::{
         CronJobLogsTable,
         CronJobsTable,
@@ -138,6 +140,7 @@ use crate::{
 
 pub mod auth;
 pub mod backend_state;
+pub mod canonical_urls;
 pub mod components;
 pub mod config;
 pub mod cron_jobs;
@@ -188,9 +191,10 @@ enum DefaultTableNumber {
     ComponentDefinitionsTable = 31,
     ComponentsTable = 32,
     FunctionHandlesTable = 33,
+    CanonicalUrls = 34,
     // Keep this number and your user name up to date. The number makes it easy to know
     // what to use next. The username on the same line detects merge conflicts
-    // Next Number - 34 - sujayakar
+    // Next Number - 35 - lee
 }
 
 impl From<DefaultTableNumber> for TableNumber {
@@ -227,6 +231,7 @@ impl From<DefaultTableNumber> for &'static dyn SystemTable {
             DefaultTableNumber::ComponentDefinitionsTable => &ComponentDefinitionsTable,
             DefaultTableNumber::ComponentsTable => &ComponentsTable,
             DefaultTableNumber::FunctionHandlesTable => &FunctionHandlesTable,
+            DefaultTableNumber::CanonicalUrls => &CanonicalUrlsTable,
         }
     }
 }
@@ -427,6 +432,7 @@ pub fn app_system_tables() -> Vec<&'static dyn SystemTable> {
         &ExportsTable,
         &SnapshotImportsTable,
         &FunctionHandlesTable,
+        &CanonicalUrlsTable,
     ];
     system_tables.extend(component_system_tables());
     system_tables
@@ -455,6 +461,7 @@ static APP_TABLES_TO_LOAD_IN_MEMORY: LazyLock<BTreeSet<TableName>> = LazyLock::n
         ENVIRONMENT_VARIABLES_TABLE.clone(),
         CRON_JOBS_TABLE.clone(),
         BACKEND_STATE_TABLE.clone(),
+        CANONICAL_URLS_TABLE.clone(),
     }
 });
 

crates/model/src/migrations.rs

@@ -34,6 +34,7 @@ use value::{
 };
 
 use crate::{
+    canonical_urls::CANONICAL_URLS_TABLE,
     database_globals::{
         types::DatabaseVersion,
         DatabaseGlobalsModel,
@@ -77,7 +78,7 @@ impl fmt::Display for MigrationCompletionCriterion {
 // migrations unless explicitly dropping support.
 // Add a user name next to the version when you make a change to highlight merge
 // conflicts.
-pub const DATABASE_VERSION: DatabaseVersion = 115; // nipunn
+pub const DATABASE_VERSION: DatabaseVersion = 116; // lee
 
 pub struct MigrationWorker<RT: Runtime> {
     rt: RT,
@@ -372,6 +373,9 @@ impl<RT: Runtime> MigrationWorker<RT> {
                 }
                 MigrationCompletionCriterion::MigrationComplete(to_version)
             },
+            116 => MigrationCompletionCriterion::LogLine(
+                format!("Created system table: {}", *CANONICAL_URLS_TABLE).into(),
+            ),
             // NOTE: Make sure to increase DATABASE_VERSION when adding new migrations.
             _ => anyhow::bail!("Version did not define a migration! {}", to_version),
         };