036 migration (#2750)

* chore: convert to use a value, cause why not

* wip: Add the up for this going up

* wip: trait changes

* wip: Add in some more of the private transformations

* chore(wip): Adding the ssh_keys todo

* wip: Add cifs

* fix migration structure

* chore: Fix the trait for the version

* wip(feat): Notifications are in the system

* fix marker trait hell

* handle key todos

* wip: Testing the migration in a system.

* fix pubkey parser

* fix: migration works

* wip: Trying to get the migration stuff?

* fix: Can now install the packages that we wanted, yay!"

* Merge branch 'next/minor' of github.com:Start9Labs/start-os into feat/migration

---------

Co-authored-by: Aiden McClelland <[email protected]>

core/startos/src/backup/backup_bulk.rs

@@ -332,10 +332,10 @@ async fn perform_backup(
 
     let timestamp = Utc::now();
 
-    backup_guard.unencrypted_metadata.version = crate::version::Current::new().semver().into();
+    backup_guard.unencrypted_metadata.version = crate::version::Current::default().semver().into();
     backup_guard.unencrypted_metadata.hostname = ctx.account.read().await.hostname.clone();
     backup_guard.unencrypted_metadata.timestamp = timestamp.clone();
-    backup_guard.metadata.version = crate::version::Current::new().semver().into();
+    backup_guard.metadata.version = crate::version::Current::default().semver().into();
     backup_guard.metadata.timestamp = Some(timestamp);
     backup_guard.metadata.package_backups = package_backups;
 

core/startos/src/bins/container_cli.rs

@@ -8,7 +8,7 @@ use crate::util::logger::EmbassyLogger;
 use crate::version::{Current, VersionT};
 
 lazy_static::lazy_static! {
-    static ref VERSION_STRING: String = Current::new().semver().to_string();
+    static ref VERSION_STRING: String = Current::default().semver().to_string();
 }
 
 pub fn main(args: impl IntoIterator<Item = OsString>) {

core/startos/src/bins/start_cli.rs

@@ -9,7 +9,7 @@ use crate::util::logger::EmbassyLogger;
 use crate::version::{Current, VersionT};
 
 lazy_static::lazy_static! {
-    static ref VERSION_STRING: String = Current::new().semver().to_string();
+    static ref VERSION_STRING: String = Current::default().semver().to_string();
 }
 
 pub fn main(args: impl IntoIterator<Item = OsString>) {

core/startos/src/context/rpc.rs

@@ -88,6 +88,7 @@ pub struct InitRpcContextPhases {
     init_net_ctrl: PhaseProgressTrackerHandle,
     read_device_info: PhaseProgressTrackerHandle,
     cleanup_init: CleanupInitPhases,
+    // TODO: migrations
 }
 impl InitRpcContextPhases {
     pub fn new(handle: &FullProgressTracker) -> Self {
@@ -286,6 +287,7 @@ impl RpcContext {
         let res = Self(seed.clone());
         res.cleanup_and_initialize(cleanup_init).await?;
         tracing::info!("Cleaned up transient states");
+        crate::version::post_init(&res).await?;
         Ok(res)
     }
 

core/startos/src/db/model/private.rs

@@ -31,6 +31,6 @@ pub struct Private {
     pub package_stores: BTreeMap<PackageId, Value>,
 }
 
-fn generate_compat_key() -> Pem<ed25519_dalek::SigningKey> {
+pub fn generate_compat_key() -> Pem<ed25519_dalek::SigningKey> {
     Pem(ed25519_dalek::SigningKey::generate(&mut rand::thread_rng()))
 }

core/startos/src/db/model/public.rs

@@ -43,10 +43,11 @@ impl Public {
                 arch: get_arch(),
                 platform: get_platform(),
                 id: account.server_id.clone(),
-                version: Current::new().semver(),
+                version: Current::default().semver(),
                 hostname: account.hostname.no_dot_host_name(),
                 last_backup: None,
-                eos_version_compat: Current::new().compat().clone(),
+                version_compat: Current::default().compat().clone(),
+                post_init_migration_todos: BTreeSet::new(),
                 lan_address,
                 onion_address: account.tor_key.public().get_onion_address(),
                 tor_address: format!("https://{}", account.tor_key.public().get_onion_address())
@@ -112,11 +113,13 @@ pub struct ServerInfo {
     pub hostname: InternedString,
     #[ts(type = "string")]
     pub version: Version,
+    #[ts(type = "string")]
+    pub version_compat: VersionRange,
+    #[ts(type = "string[]")]
+    pub post_init_migration_todos: BTreeSet<Version>,
     #[ts(type = "string | null")]
     pub last_backup: Option<DateTime<Utc>>,
     #[ts(type = "string")]
-    pub eos_version_compat: VersionRange,
-    #[ts(type = "string")]
     pub lan_address: Url,
     #[ts(type = "string")]
     pub onion_address: OnionAddressV3,

core/startos/src/init.rs

@@ -323,7 +323,9 @@ pub async fn init(
     local_auth.complete();
 
     load_database.start();
-    let db = TypedPatchDb::<Database>::load_unchecked(cfg.db().await?);
+    let db = cfg.db().await?;
+    crate::version::Current::default().pre_init(&db).await?;
+    let db = TypedPatchDb::<Database>::load_unchecked(db);
     let peek = db.peek().await;
     load_database.complete();
     tracing::info!("Opened PatchDB");
@@ -528,8 +530,6 @@ pub async fn init(
         .await?;
     launch_service_network.complete();
 
-    crate::version::init(&db, run_migrations).await?;
-
     validate_db.start();
     db.mutate(|d| {
         let model = d.de()?;

core/startos/src/notifications.rs

@@ -256,13 +256,13 @@ impl Map for Notifications {
 #[derive(Debug, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct Notification {
-    package_id: Option<PackageId>,
-    created_at: DateTime<Utc>,
-    code: u32,
-    level: NotificationLevel,
-    title: String,
-    message: String,
-    data: Value,
+    pub package_id: Option<PackageId>,
+    pub created_at: DateTime<Utc>,
+    pub code: u32,
+    pub level: NotificationLevel,
+    pub title: String,
+    pub message: String,
+    pub data: Value,
 }
 
 #[derive(Debug, Serialize, Deserialize)]

core/startos/src/os_install/mod.rs

@@ -150,23 +150,6 @@ pub async fn execute<C: Context>(
 
     overwrite |= disk.guid.is_none() && disk.partitions.iter().all(|p| p.guid.is_none());
 
-    if !overwrite
-        && (disk
-            .guid
-            .as_ref()
-            .map_or(false, |g| g.starts_with("EMBASSY_"))
-            || disk
-                .partitions
-                .iter()
-                .flat_map(|p| p.guid.as_ref())
-                .any(|g| g.starts_with("EMBASSY_")))
-    {
-        return Err(Error::new(
-            eyre!("installing over versions before 0.3.6 is unsupported"),
-            ErrorKind::InvalidRequest,
-        ));
-    }
-
     let part_info = partition(&mut disk, overwrite).await?;
 
     if let Some(efi) = &part_info.efi {

core/startos/src/registry/device_info.rs

@@ -108,8 +108,8 @@ pub struct OsInfo {
 impl From<&RpcContext> for OsInfo {
     fn from(_: &RpcContext) -> Self {
         Self {
-            version: crate::version::Current::new().semver(),
-            compat: crate::version::Current::new().compat().clone(),
+            version: crate::version::Current::default().semver(),
+            compat: crate::version::Current::default().compat().clone(),
             platform: InternedString::intern(&*crate::PLATFORM),
         }
     }

core/startos/src/s9pk/v2/manifest.rs

@@ -22,7 +22,7 @@ use crate::util::VersionString;
 use crate::version::{Current, VersionT};
 
 fn current_version() -> Version {
-    Current::new().semver()
+    Current::default().semver()
 }
 
 #[derive(Clone, Debug, Deserialize, Serialize, HasModel, TS)]

core/startos/src/service/mod.rs

@@ -408,6 +408,7 @@ impl Service {
         let developer_key = s9pk.as_archive().signer();
         let icon = s9pk.icon_data_url().await?;
         let service = Self::new(ctx.clone(), s9pk, StartStop::Stop).await?;
+
         if let Some(recovery_source) = recovery_source {
             service
                 .actor
@@ -429,6 +430,7 @@ impl Service {
                     .clone(),
             );
         }
+
         let procedure_id = Guid::new();
         service
             .seed
@@ -441,6 +443,7 @@ impl Service {
             ) // TODO timeout
             .await
             .with_kind(ErrorKind::MigrationFailed)?; // TODO: handle cancellation
+
         if let Some(mut progress) = progress {
             progress.finalization_progress.complete();
             progress.progress.complete();

core/startos/src/service/service_map.rs

@@ -307,7 +307,6 @@ impl ServiceMap {
                     sync_progress_task.await.map_err(|_| {
                         Error::new(eyre!("progress sync task panicked"), ErrorKind::Unknown)
                     })??;
-
                     Ok(())
                 })
                 .boxed())

core/startos/src/ssh.rs

@@ -25,6 +25,12 @@ impl SshKeys {
         Self(BTreeMap::new())
     }
 }
+
+impl From<BTreeMap<InternedString, WithTimeData<SshPubKey>>> for SshKeys {
+    fn from(map: BTreeMap<InternedString, WithTimeData<SshPubKey>>) -> Self {
+        Self(map)
+    }
+}
 impl Map for SshKeys {
     type Key = InternedString;
     type Value = WithTimeData<SshPubKey>;
@@ -41,7 +47,7 @@ impl Map for SshKeys {
 pub struct SshPubKey(
     #[serde(serialize_with = "crate::util::serde::serialize_display")]
     #[serde(deserialize_with = "crate::util::serde::deserialize_from_str")]
-    openssh_keys::PublicKey,
+    pub openssh_keys::PublicKey,
 );
 impl ValueParserFactory for SshPubKey {
     type Parser = FromStrParser<Self>;