Merge branch 'main' into alsh/magic-comment-transformer

crates/atlaspack/src/requests/asset_request.rs

@@ -50,6 +50,7 @@ pub struct AssetRequestOutput {
 
 #[async_trait]
 impl Request for AssetRequest {
+  #[tracing::instrument(level = "trace", skip_all)]
   async fn run(
     &self,
     request_context: RunRequestContext,
@@ -130,6 +131,7 @@ impl Request for AssetRequest {
   }
 }
 
+#[tracing::instrument(level = "trace", skip_all)]
 async fn run_pipelines(
   transform_context: TransformContext,
   input: Asset,

crates/atlaspack/src/requests/path_request.rs

@@ -41,6 +41,7 @@ pub enum PathRequestOutput {
 // TODO tracing, dev deps
 #[async_trait]
 impl Request for PathRequest {
+  #[tracing::instrument(level = "trace", skip_all)]
   async fn run(
     &self,
     request_context: RunRequestContext,

crates/atlaspack_core/src/asset_graph.rs

@@ -5,6 +5,7 @@ use petgraph::{
   visit::EdgeRef,
   Direction,
 };
+use serde::Serialize;
 
 use crate::types::{Asset, Dependency};
 
@@ -90,6 +91,24 @@ impl AssetGraph {
     }
   }
 
+  pub fn edges(&self) -> Vec<u32> {
+    let raw_edges = self.graph.raw_edges();
+    let mut edges = Vec::with_capacity(raw_edges.len() * 2);
+
+    for edge in raw_edges {
+      edges.push(edge.source().index() as u32);
+      edges.push(edge.target().index() as u32);
+    }
+
+    edges
+  }
+
+  pub fn nodes(&self) -> impl Iterator<Item = &AssetGraphNode> {
+    let nodes = self.graph.node_weights();
+
+    nodes
+  }
+
   pub fn add_asset(&mut self, parent_idx: NodeIndex, asset: Asset) -> NodeIndex {
     let idx = self.assets.len();
 
@@ -279,24 +298,13 @@ impl AssetGraph {
       }
     }
   }
-}
 
-#[derive(serde::Serialize)]
-#[serde(rename_all = "camelCase")]
-pub struct SerializedDependency {
-  id: String,
-  dependency: Dependency,
-}
+  pub fn serialize_nodes(&self, max_str_len: usize) -> serde_json::Result<Vec<String>> {
+    let mut nodes: Vec<String> = Vec::new();
+    let mut curr_node = String::default();
 
-impl serde::Serialize for AssetGraph {
-  fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-  where
-    S: serde::Serializer,
-  {
-    let nodes: Vec<_> = self
-      .graph
-      .node_weights()
-      .map(|node| match node {
+    for node in self.nodes() {
+      let serialized_node = match node {
         AssetGraphNode::Root => SerializedAssetGraphNode::Root,
         AssetGraphNode::Entry => SerializedAssetGraphNode::Entry,
         AssetGraphNode::Asset(idx) => {
@@ -314,41 +322,51 @@ impl serde::Serialize for AssetGraph {
             has_deferred: self.dependencies[*idx].state == DependencyState::Deferred,
           }
         }
-      })
-      .collect();
-    let raw_edges = self.graph.raw_edges();
-    let mut edges = Vec::with_capacity(raw_edges.len() * 2);
-    for edge in raw_edges {
-      edges.push(edge.source().index() as u32);
-      edges.push(edge.target().index() as u32);
-    }
+      };
 
-    #[derive(serde::Serialize)]
-    #[serde(tag = "type", rename_all = "camelCase")]
-    enum SerializedAssetGraphNode {
-      Root,
-      Entry,
-      Asset {
-        value: Asset,
-      },
-      Dependency {
-        value: SerializedDependency,
-        has_deferred: bool,
-      },
+      let str = serde_json::to_string(&serialized_node)?;
+      if curr_node.len() + str.len() < (max_str_len - 3) {
+        if !curr_node.is_empty() {
+          curr_node.push(',');
+        }
+        curr_node.push_str(&str);
+      } else {
+        // Add the existing node now as it has reached the max JavaScript string size
+        nodes.push(format!("[{curr_node}]"));
+        curr_node = str;
+      }
     }
 
-    #[derive(serde::Serialize)]
-    struct SerializedAssetGraph {
-      nodes: Vec<SerializedAssetGraphNode>,
-      // TODO: somehow make this a typed array?
-      edges: Vec<u32>,
+    // Add the current node if it did not overflow in size
+    if curr_node.len() < (max_str_len - 3) {
+      nodes.push(format!("[{curr_node}]"));
     }
 
-    let serialized = SerializedAssetGraph { nodes, edges };
-    serialized.serialize(serializer)
+    Ok(nodes)
   }
 }
 
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct SerializedDependency {
+  id: String,
+  dependency: Dependency,
+}
+
+#[derive(Debug, Serialize)]
+#[serde(tag = "type", rename_all = "camelCase")]
+enum SerializedAssetGraphNode {
+  Root,
+  Entry,
+  Asset {
+    value: Asset,
+  },
+  Dependency {
+    value: SerializedDependency,
+    has_deferred: bool,
+  },
+}
+
 impl std::hash::Hash for AssetGraph {
   fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
     for node in self.graph.node_weights() {
@@ -366,6 +384,8 @@ impl std::hash::Hash for AssetGraph {
 mod tests {
   use std::path::PathBuf;
 
+  use serde_json::{json, Value};
+
   use crate::types::{Symbol, Target};
 
   use super::*;
@@ -611,4 +631,100 @@ mod tests {
     // "*" should be marked as requested on the stuff dep
     assert_requested_symbols(&graph, stuff_dep, vec!["*"]);
   }
+
+  #[test]
+  fn serialize_nodes_handles_max_size() -> anyhow::Result<()> {
+    let mut graph = AssetGraph::new();
+
+    let entry = graph.add_entry_dependency(Dependency {
+      specifier: String::from("entry"),
+      ..Dependency::default()
+    });
+
+    let entry_asset = graph.add_asset(
+      entry,
+      Asset {
+        file_path: PathBuf::from("entry"),
+        ..Asset::default()
+      },
+    );
+
+    for i in 1..100 {
+      graph.add_dependency(
+        entry_asset,
+        Dependency {
+          specifier: format!("dependency-{}", i),
+          ..Dependency::default()
+        },
+      );
+    }
+
+    let max_str_len = 10000;
+    let nodes = graph.serialize_nodes(max_str_len)?;
+
+    assert_eq!(nodes.len(), 7);
+
+    // Assert each string is less than the max size
+    for node in nodes.iter() {
+      assert!(node.len() < max_str_len);
+    }
+
+    // Assert all the nodes are included and in the correct order
+    let first_entry = serde_json::from_str::<Value>(&nodes[0])?;
+    let first_entry = first_entry.as_array().unwrap();
+
+    assert_eq!(get_type(&first_entry[0]), json!("root"));
+    assert_eq!(get_dependency(&first_entry[1]), Some(json!("entry")));
+    assert_eq!(get_asset(&first_entry[2]), Some(json!("entry")));
+
+    for i in 1..first_entry.len() - 2 {
+      assert_eq!(
+        get_dependency(&first_entry[i + 2]),
+        Some(json!(format!("dependency-{}", i)))
+      );
+    }
+
+    let mut specifier = first_entry.len() - 2;
+    for node in nodes[1..].iter() {
+      let entry = serde_json::from_str::<Value>(&node)?;
+      let entry = entry.as_array().unwrap();
+
+      for value in entry {
+        assert_eq!(
+          get_dependency(&value),
+          Some(json!(format!("dependency-{}", specifier)))
+        );
+
+        specifier += 1;
+      }
+    }
+
+    Ok(())
+  }
+
+  fn get_type(node: &Value) -> Value {
+    node.get("type").unwrap().to_owned()
+  }
+
+  fn get_dependency(value: &Value) -> Option<Value> {
+    assert_eq!(get_type(&value), json!("dependency"));
+
+    value
+      .get("value")
+      .unwrap()
+      .get("dependency")
+      .unwrap()
+      .get("specifier")
+      .map(|s| s.to_owned())
+  }
+
+  fn get_asset(value: &Value) -> Option<Value> {
+    assert_eq!(get_type(&value), json!("asset"));
+
+    value
+      .get("value")
+      .unwrap()
+      .get("filePath")
+      .map(|s| s.to_owned())
+  }
 }

crates/atlaspack_plugin_transformer_js/src/js_transformer.rs

@@ -304,6 +304,7 @@ impl TransformerPlugin for AtlaspackJsTransformerPlugin {
             })
           })
           .unwrap_or_default(),
+        magic_comments: self.config.magic_comments.unwrap_or_default(),
         ..atlaspack_js_swc_core::Config::default()
       },
       None,

crates/atlaspack_plugin_transformer_js/src/js_transformer_config.rs

@@ -16,4 +16,6 @@ pub struct JsTransformerConfig {
 
   #[serde(rename = "inlineFS")]
   pub inline_fs: Option<bool>,
+
+  pub magic_comments: Option<bool>,
 }

crates/node-bindings/src/atlaspack/atlaspack.rs

@@ -53,6 +53,9 @@ pub struct AtlaspackNapi {
   tx_worker: Sender<NodejsWorker>,
 }
 
+// Refer to https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/length
+const MAX_STRING_LENGTH: usize = 268435440;
+
 #[napi]
 impl AtlaspackNapi {
   #[tracing::instrument(level = "info", skip_all)]
@@ -136,7 +139,15 @@ impl AtlaspackNapi {
         match atlaspack {
           Err(error) => deferred.reject(to_napi_error(error)),
           Ok(atlaspack) => match atlaspack.build_asset_graph() {
-            Ok(asset_graph) => deferred.resolve(move |env| env.to_js_value(&asset_graph)),
+            Ok(asset_graph) => deferred.resolve(move |env| {
+              let mut js_object = env.create_object()?;
+
+              js_object.set_named_property("edges", env.to_js_value(&asset_graph.edges())?)?;
+              js_object
+                .set_named_property("nodes", asset_graph.serialize_nodes(MAX_STRING_LENGTH)?)?;
+
+              Ok(js_object)
+            }),
             Err(error) => deferred.reject(to_napi_error(error)),
           },
         }

packages/core/cli/src/makeDebugCommand.js

@@ -84,10 +84,10 @@ export function makeDebugCommand(): commander$Command {
             useLmdbJsLite: true,
           },
         });
+
         console.log('Created atlaspack instance');
 
         await atlaspack.unstable_buildAssetGraph();
-        console.log('Done building asset graph');
         process.exit(0);
       } catch (err) {
         handleUncaughtException(err);

packages/core/core/src/Atlaspack.js

@@ -585,6 +585,8 @@ export default class Atlaspack {
     writeToCache: boolean = true,
   ): Promise<AssetGraphRequestResult> {
     await this._init();
+
+    const origin = '@atlaspack/core';
     const input = {
       optionsRef: this.#optionsRef,
       name: 'Main',
@@ -594,19 +596,26 @@ export default class Atlaspack {
       lazyExcludes: [],
       requestedAssetIds: this.#requestedAssetIds,
     };
+
+    const start = Date.now();
     const result = await this.#requestTracker.runRequest(
       this.rustAtlaspack != null
         ? createAssetGraphRequestRust(this.rustAtlaspack)(input)
         : createAssetGraphRequestJS(input),
       {force: true},
     );
 
-    logger.info({message: 'Done building asset graph!'});
+    const duration = Date.now() - start;
+
+    logger.info({
+      message: `Done building asset graph in ${duration / 1000}s!`,
+      origin,
+    });
 
     if (writeToCache) {
-      logger.info({message: 'Write request tracker to cache'});
+      logger.info({message: 'Write request tracker to cache', origin});
       await this.writeRequestTrackerToCache();
-      logger.info({message: 'Done writing request tracker to cache'});
+      logger.info({message: 'Done writing request tracker to cache', origin});
     }
 
     return result;

packages/core/core/src/requests/AssetGraphRequestRust.js

@@ -42,6 +42,9 @@ export function createAssetGraphRequestRust(
       let serializedAssetGraph;
       try {
         serializedAssetGraph = await rustAtlaspack.buildAssetGraph();
+        serializedAssetGraph.nodes = serializedAssetGraph.nodes.flatMap(
+          (node) => JSON.parse(node),
+        );
       } catch (err) {
         throw new ThrowableDiagnostic({
           diagnostic: err,

packages/core/integration-tests/test/api.js

@@ -14,8 +14,8 @@ import {
 
 import {ATLASPACK_VERSION} from '../../core/src/constants';
 
-describe.v2('JS API', function () {
-  it('should respect distEntry', async function () {
+describe('JS API', function () {
+  it.v2('should respect distEntry', async function () {
     const NAME = 'custom-name.js';
 
     let b = await bundle(

packages/core/integration-tests/test/atlaspack-link.js

@@ -19,7 +19,7 @@ function createProgram(opts: ProgramOptions) {
   return cli;
 }
 
-describe.v2('@atlaspack/link', () => {
+describe('@atlaspack/link', () => {
   let _cwd;
   let _stdout;