feat(hoist): hoist can create anonymous versions of specs

bin/hoist/src/args.rs

@@ -0,0 +1,96 @@
+use clap::Subcommand;
+use std::path::PathBuf;
+
+#[derive(Subcommand, Debug)]
+#[remain::sorted]
+pub enum Commands {
+    AnonymizeSpecs(AnonymizeSpecsArgs),
+    UploadAllSpecs(UploadAllSpecsArgs),
+    UploadSpec(UploadSpecArgs),
+    WriteAllSpecs(WriteAllSpecsArgs),
+    WriteExistingModulesSpec(WriteExistingModulesSpecArgs),
+    WriteSpec(WriteSpecArgs),
+}
+
+#[derive(clap::Args, Debug)]
+#[command(about = "Generate an anonymized version of target spec(s)")]
+pub struct AnonymizeSpecsArgs {
+    #[arg(long, short = 'o', required = true)]
+    pub out: PathBuf,
+
+    #[arg(
+        long,
+        short = 't',
+        required = true,
+        help = "Path to the directory containing specs to anonymize"
+    )]
+    pub target_dir: PathBuf,
+
+    #[arg(
+        long,
+        default_value = "100",
+        help = "Maximum number of concurrent uploads."
+    )]
+    pub max_concurrent: usize,
+}
+
+#[derive(clap::Args, Debug)]
+#[command(about = "Upload all specs in {target_dir} to the module index")]
+pub struct UploadAllSpecsArgs {
+    #[arg(
+        long,
+        short = 't',
+        required = true,
+        help = "Path to the directory containing specs to upload"
+    )]
+    pub target_dir: PathBuf,
+
+    #[arg(
+        long,
+        default_value = "100",
+        help = "Maximum number of concurrent uploads."
+    )]
+    pub max_concurrent: usize,
+}
+
+#[derive(clap::Args, Debug)]
+#[command(about = "Upload the spec {target} to the module index")]
+pub struct UploadSpecArgs {
+    #[arg(
+        long,
+        short = 't',
+        required = true,
+        help = "Path to the spec to upload"
+    )]
+    pub target: PathBuf,
+
+    #[arg(
+        long,
+        default_value = "100",
+        help = "Maximum number of concurrent uploads."
+    )]
+    pub max_concurrent: usize,
+}
+
+#[derive(clap::Args, Debug)]
+#[command(about = "Get all built-ins an write out a hashmap with their name and schema id")]
+pub struct WriteExistingModulesSpecArgs {
+    #[arg(long, short = 'o', required = true)]
+    pub out: PathBuf,
+}
+
+#[derive(clap::Args, Debug)]
+#[command(about = "Get {spec_name} from the module index and write it to {out}")]
+pub struct WriteSpecArgs {
+    #[arg(long, short = 's', required = true)]
+    pub spec_name: String,
+    #[arg(long, short = 'o', required = true)]
+    pub out: PathBuf,
+}
+
+#[derive(clap::Args, Debug)]
+#[command(about = "Get all specs from the module index and write them to {out}")]
+pub struct WriteAllSpecsArgs {
+    #[arg(long, short = 'o', required = true)]
+    pub out: PathBuf,
+}

bin/hoist/src/main.rs

@@ -1,19 +1,22 @@
+use args::Commands;
 use clap::CommandFactory;
 use futures::stream::StreamExt;
 use indicatif::{ProgressBar, ProgressStyle};
 use std::collections::HashMap;
-use std::fs::{self};
+use std::fs::{self, DirEntry};
 use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::Arc;
 use ulid::Ulid;
 
-use clap::{Parser, Subcommand};
+use clap::Parser;
 use color_eyre::Result;
 use module_index_client::{ModuleDetailsResponse, ModuleIndexClient};
 use si_pkg::{PkgSpec, SiPkg};
 use url::Url;
 
+mod args;
+
 const CLOVER_DEFAULT_CREATOR: &str = "Clover";
 
 #[derive(Parser, Debug)]
@@ -29,76 +32,6 @@ struct Args {
     command: Option<Commands>,
 }
 
-#[derive(Subcommand, Debug)]
-#[remain::sorted]
-enum Commands {
-    UploadAllSpecs(UploadAllSpecsArgs),
-    UploadSpec(UploadSpecArgs),
-    WriteAllSpecs(WriteAllSpecsArgs),
-    WriteExistingModulesSpec(WriteExistingModulesSpecArgs),
-    WriteSpec(WriteSpecArgs),
-}
-
-#[derive(clap::Args, Debug)]
-#[command(about = "Upload all specs in {target_dir} to the module index")]
-struct UploadAllSpecsArgs {
-    #[arg(
-        long,
-        short = 't',
-        required = true,
-        help = "Path to the directory containing specs to upload"
-    )]
-    target_dir: PathBuf,
-
-    #[arg(
-        long,
-        default_value = "100",
-        help = "Maximum number of concurrent uploads."
-    )]
-    max_concurrent: usize,
-}
-#[derive(clap::Args, Debug)]
-#[command(about = "Upload the spec {target} to the module index")]
-struct UploadSpecArgs {
-    #[arg(
-        long,
-        short = 't',
-        required = true,
-        help = "Path to the spec to upload"
-    )]
-    target: PathBuf,
-
-    #[arg(
-        long,
-        default_value = "100",
-        help = "Maximum number of concurrent uploads."
-    )]
-    max_concurrent: usize,
-}
-
-#[derive(clap::Args, Debug)]
-#[command(about = "Get all built-ins an write out a hashmap with their name and schema id")]
-struct WriteExistingModulesSpecArgs {
-    #[arg(long, short = 'o', required = true)]
-    out: PathBuf,
-}
-
-#[derive(clap::Args, Debug)]
-#[command(about = "Get {spec_name} from the module index and write it to {out}")]
-struct WriteSpecArgs {
-    #[arg(long, short = 's', required = true)]
-    spec_name: String,
-    #[arg(long, short = 'o', required = true)]
-    out: PathBuf,
-}
-
-#[derive(clap::Args, Debug)]
-#[command(about = "Get all specs from the module index and write them to {out}")]
-struct WriteAllSpecsArgs {
-    #[arg(long, short = 'o', required = true)]
-    out: PathBuf,
-}
-
 #[tokio::main]
 async fn main() -> Result<()> {
     color_eyre::install()?;
@@ -109,6 +42,7 @@ async fn main() -> Result<()> {
     let client = ModuleIndexClient::new(Url::parse(endpoint)?, token);
 
     match args.command {
+        Some(Commands::AnonymizeSpecs(args)) => anonymize_specs(args.target_dir, args.out).await?,
         Some(Commands::UploadAllSpecs(args)) => {
             upload_pkg_specs(&client, args.target_dir, args.max_concurrent).await?
         }
@@ -141,6 +75,24 @@ enum ModuleState {
     NeedsUpdate,
     New,
 }
+
+async fn anonymize_specs(target_dir: PathBuf, out: PathBuf) -> Result<()> {
+    fs::create_dir_all(&out)?;
+    let specs = spec_from_dir_or_file(target_dir)?;
+    for dir in specs {
+        let mut spec = json_to_spec(dir.path())?;
+        let spec_name = format!("{}.json", spec.name);
+        spec.anonymize();
+
+        fs::write(
+            Path::new(&out).join(spec_name),
+            serde_json::to_string_pretty(&spec)?,
+        )?;
+    }
+
+    Ok(())
+}
+
 async fn write_existing_modules_spec(client: ModuleIndexClient, out: PathBuf) -> Result<()> {
     let modules = list_specs(client.clone()).await?;
     let mut entries: HashMap<String, String> = HashMap::new();
@@ -241,23 +193,7 @@ async fn upload_pkg_specs(
     target_dir: PathBuf,
     max_concurrent: usize,
 ) -> Result<()> {
-    let specs: Vec<_> = if target_dir.is_file() {
-        if let Some(parent) = target_dir.parent() {
-            fs::read_dir(parent)?
-                .filter_map(|entry| entry.ok())
-                .filter(|entry| entry.path() == target_dir)
-                .collect()
-        } else {
-            vec![]
-        }
-    } else {
-        fs::read_dir(&target_dir)?
-            .filter_map(|entry| entry.ok())
-            .filter(|entry| {
-                entry.path().is_file() && entry.path().extension().is_some_and(|ext| ext == "json")
-            })
-            .collect()
-    };
+    let specs: Vec<_> = spec_from_dir_or_file(target_dir)?;
 
     let mut no_action_needed = 0;
     let mut new_modules = vec![];
@@ -383,9 +319,13 @@ async fn upload_pkg_specs(
 }
 
 fn json_to_pkg(spec: PathBuf) -> Result<SiPkg> {
+    Ok(SiPkg::load_from_spec(json_to_spec(spec)?)?)
+}
+
+fn json_to_spec(spec: PathBuf) -> Result<PkgSpec> {
     let buf = fs::read_to_string(&spec)?;
     let spec: PkgSpec = serde_json::from_str(&buf)?;
-    Ok(SiPkg::load_from_spec(spec)?)
+    Ok(spec)
 }
 
 async fn list_specs(client: ModuleIndexClient) -> Result<Vec<ModuleDetailsResponse>> {
@@ -431,3 +371,23 @@ fn setup_progress_bar(length: u64) -> Arc<ProgressBar> {
     );
     pb
 }
+
+fn spec_from_dir_or_file(target_dir: PathBuf) -> Result<Vec<DirEntry>> {
+    Ok(if target_dir.is_file() {
+        if let Some(parent) = target_dir.parent() {
+            fs::read_dir(parent)?
+                .filter_map(|entry| entry.ok())
+                .filter(|entry| entry.path() == target_dir)
+                .collect()
+        } else {
+            vec![]
+        }
+    } else {
+        fs::read_dir(&target_dir)?
+            .filter_map(|entry| entry.ok())
+            .filter(|entry| {
+                entry.path().is_file() && entry.path().extension().is_some_and(|ext| ext == "json")
+            })
+            .collect()
+    })
+}