parallel parsing event loop, circular dependency resolution, lib api …

…surface and examples

.github/workflows/verify.yml

@@ -13,6 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - build
+      - examples
       - fmt
       - lint
       - test
@@ -28,6 +29,18 @@ jobs:
       - name: cargo build
         run: cargo build --all-features --release --workspace
 
+  examples:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: swatinem/rust-cache@v2
+      - name: cargo build
+        run: |
+          cargo run --example build_fn
+          cargo run --example parse_fn
+        working-directory: fn_build
+
   fmt:
     runs-on: ubuntu-latest
     steps:

fn_build/Cargo.toml

@@ -5,6 +5,10 @@ authors = { workspace = true }
 edition = { workspace = true }
 repository = { workspace = true }
 
+[lib]
+name = "l3_fn_build"
+path = "src/lib.rs"
+
 [dependencies]
 anyhow = { workspace = true }
 serde = { workspace = true }

fn_build/examples/build_fn.rs

@@ -0,0 +1,36 @@
+use l3_fn_build::runtime::node::NodeConfig;
+use l3_fn_build::runtime::Runtime;
+use l3_fn_build::{build_fn, BuildMode, FnBuildSpec, FnParseSpec};
+use std::env;
+use std::path::PathBuf;
+use std::sync::Arc;
+use temp_dir::TempDir;
+
+#[tokio::main]
+async fn main() {
+    let project_dir = PathBuf::from(
+        env::args()
+            .nth(1)
+            .unwrap_or_else(|| "fixtures/node/js/circular_imports".to_string()),
+    );
+    let out_dir = TempDir::new().unwrap();
+    let node_config = NodeConfig::read_node_config(&project_dir).unwrap();
+    let fn_build = build_fn(FnBuildSpec {
+        function: FnParseSpec {
+            entrypoint: PathBuf::from("routes/data/lambda.js"),
+            project_dir: Arc::new(env::current_dir().unwrap().join(&project_dir)),
+            runtime: Runtime::Node(Arc::new(node_config)),
+        },
+        mode: BuildMode::Debug,
+        output: out_dir.path().to_path_buf(),
+    })
+    .await
+    .unwrap();
+    println!(
+        "l3_fn_build::build_fn output in temp dir from project dir {}:",
+        project_dir.to_string_lossy()
+    );
+    for source in fn_build.manifest.sources {
+        println!("   {}", source.path.to_string_lossy());
+    }
+}

fn_build/examples/parse_fn.rs

@@ -0,0 +1,30 @@
+use l3_fn_build::runtime::node::NodeConfig;
+use l3_fn_build::runtime::Runtime;
+use l3_fn_build::{parse_fn, FnParseSpec};
+use std::env;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+#[tokio::main]
+async fn main() {
+    let project_dir = PathBuf::from(
+        env::args()
+            .nth(1)
+            .unwrap_or_else(|| "fixtures/node/js/circular_imports".to_string()),
+    );
+    let node_config = NodeConfig::read_node_config(&project_dir).unwrap();
+    let fn_manifest = parse_fn(FnParseSpec {
+        entrypoint: PathBuf::from("routes/data/lambda.js"),
+        project_dir: Arc::new(env::current_dir().unwrap().join(&project_dir)),
+        runtime: Runtime::Node(Arc::new(node_config)),
+    })
+    .await
+    .unwrap();
+    println!(
+        "l3_fn_build::parse_fn result for project dir {}:",
+        project_dir.to_string_lossy()
+    );
+    for source in fn_manifest.sources {
+        println!("   {}", source.path.to_string_lossy());
+    }
+}

fn_build/fixtures/node/js/circular_imports/.fixture/build_debug.json

@@ -0,0 +1,20 @@
+{
+  "files": [
+    {
+      "path": "package.json",
+      "result": "identical"
+    },
+    {
+      "path": "lib/api.js",
+      "result": "identical"
+    },
+    {
+      "path": "lib/data.js",
+      "result": "identical"
+    },
+    {
+      "path": "routes/data/lambda.js",
+      "result": "identical"
+    }
+  ]
+}

fn_build/fixtures/node/js/circular_imports/.fixture/build_release.json

@@ -0,0 +1,26 @@
+{
+  "files": [
+    {
+      "path": "package.json",
+      "result": "identical"
+    },
+    {
+      "path": "lib/api.js",
+      "result": {
+        "content": "import{getBackendData as t}from\"./data.js\";export function getData(){return t()}"
+      }
+    },
+    {
+      "path": "lib/data.js",
+      "result": {
+        "content": "import{getData as t}from\"./api.js\";export function getBackendData(){return t()}"
+      }
+    },
+    {
+      "path": "routes/data/lambda.js",
+      "result": {
+        "content": "import{getData as o}from\"../../lib/api.js\";export const GET=()=>{console.log(\"got\",o())};"
+      }
+    }
+  ]
+}

fn_build/fixtures/node/js/circular_imports/.fixture/parse.json

@@ -0,0 +1,33 @@
+{
+  "dependencies": "unused",
+  "sources": [
+    {
+      "imports": [],
+      "path": "package.json"
+    },
+    {
+      "imports": [
+        {
+          "relativeSource": "lib/data.js"
+        }
+      ],
+      "path": "lib/api.js"
+    },
+    {
+      "imports": [
+        {
+          "relativeSource": "lib/api.js"
+        }
+      ],
+      "path": "lib/data.js"
+    },
+    {
+      "imports": [
+        {
+          "relativeSource": "lib/api.js"
+        }
+      ],
+      "path": "routes/data/lambda.js"
+    }
+  ]
+}

fn_build/fixtures/node/js/circular_imports/.fixture/spec.json

@@ -0,0 +1,3 @@
+{
+  "entrypoint": "routes/data/lambda.js"
+}

fn_build/fixtures/node/js/circular_imports/lib/api.js

@@ -0,0 +1,5 @@
+import {getBackendData} from './data.js'
+
+export function getData() {
+    return getBackendData()
+}

fn_build/fixtures/node/js/circular_imports/lib/data.js

@@ -0,0 +1,5 @@
+import {getData} from './api.js'
+
+export function getBackendData() {
+    return getData()
+}

fn_build/fixtures/node/js/circular_imports/package.json

@@ -0,0 +1,3 @@
+{
+  "type": "module"
+}

fn_build/fixtures/node/js/circular_imports/routes/data/lambda.js

@@ -0,0 +1,5 @@
+import {getData} from '../../lib/api.js'
+
+export const GET = () => {
+    console.log('got', getData())
+}

fn_build/src/build_test.rs

@@ -1,9 +1,10 @@
 use crate::build_fn;
-use crate::result::FnBuildError;
 use crate::runtime::Runtime;
-use crate::spec::{BuildMode, FnBuildSpec, FnParseSpec};
+use crate::FnBuildError;
+use crate::{BuildMode, FnBuildSpec, FnParseSpec};
 use std::env;
 use std::path::PathBuf;
+use std::sync::Arc;
 use temp_dir::TempDir;
 
 #[tokio::test]
@@ -13,9 +14,11 @@ async fn build_fn_errors_for_invalid_extension() {
         let build_spec = FnBuildSpec {
             function: FnParseSpec {
                 entrypoint: PathBuf::from(entrypoint),
-                project_dir: env::current_dir()
-                    .unwrap()
-                    .join("fixtures/node/js/http_route"),
+                project_dir: Arc::new(
+                    env::current_dir()
+                        .unwrap()
+                        .join("fixtures/node/js/http_route"),
+                ),
                 runtime: Runtime::Node(Default::default()),
             },
             mode: BuildMode::Debug,

fn_build/src/lib.rs

@@ -1,7 +1,7 @@
 mod fs;
 mod paths;
 mod result;
-mod runtime;
+pub mod runtime;
 mod spec;
 mod swc;
 
@@ -17,9 +17,8 @@ mod paths_test;
 #[cfg(test)]
 mod testing;
 
-use crate::result::{FnBuild, FnBuildError, FnBuildResult, FnManifest};
-use crate::spec::{FnBuildSpec, FnParseSpec};
-use crate::swc::{build_js_fn, parse_js_fn};
+pub use crate::result::*;
+pub use crate::spec::*;
 
 pub async fn build_fn(build_spec: FnBuildSpec) -> FnBuildResult<FnBuild> {
     debug_assert!(build_spec.function.entrypoint.is_relative());
@@ -31,7 +30,7 @@ pub async fn build_fn(build_spec: FnBuildSpec) -> FnBuildResult<FnBuild> {
     match build_spec.function.entrypoint.extension() {
         None => Err(FnBuildError::InvalidFileType),
         Some(extension) => match extension.to_string_lossy().as_ref() {
-            "js" | "mjs" => build_js_fn(build_spec).await,
+            "js" | "mjs" => runtime::node::build_node_fn(build_spec).await,
             "py" => todo!(),
             "ts" => todo!(),
             &_ => Err(FnBuildError::InvalidFileType),
@@ -47,7 +46,7 @@ pub async fn parse_fn(parse_spec: FnParseSpec) -> FnBuildResult<FnManifest> {
     match parse_spec.entrypoint.extension() {
         None => Err(FnBuildError::InvalidFileType),
         Some(extension) => match extension.to_string_lossy().as_ref() {
-            "js" | "mjs" => parse_js_fn(parse_spec).await,
+            "js" | "mjs" => runtime::node::parse_node_fn(parse_spec).await,
             "py" => todo!(),
             "ts" => todo!(),
             &_ => Err(FnBuildError::InvalidFileType),

fn_build/src/parse_test.rs

@@ -1,17 +1,20 @@
 use crate::runtime::Runtime;
-use crate::spec::FnParseSpec;
+use crate::FnParseSpec;
 use crate::{parse_fn, FnBuildError};
 use std::env;
 use std::path::PathBuf;
+use std::sync::Arc;
 
 #[tokio::test]
 async fn parse_fn_errors_for_invalid_extension() {
     for entrypoint in &["README", "README.md"] {
         let parse_spec = FnParseSpec {
             entrypoint: PathBuf::from(entrypoint),
-            project_dir: env::current_dir()
-                .unwrap()
-                .join("fixtures/node/js/http_route"),
+            project_dir: Arc::new(
+                env::current_dir()
+                    .unwrap()
+                    .join("fixtures/node/js/http_route"),
+            ),
             runtime: Runtime::Node(Default::default()),
         };
         match parse_fn(parse_spec).await {

fn_build/src/result.rs

@@ -1,4 +1,4 @@
-use crate::spec::FnBuildOutput;
+use crate::FnBuildOutput;
 use serde::Deserialize;
 use std::io;
 use std::path::PathBuf;

fn_build/src/runtime/mod.rs

@@ -1,14 +1,21 @@
-use crate::result::ModuleImport;
 use crate::runtime::node::NodeConfig;
-use std::path::Path;
+use crate::{FnBuildResult, FnSource, ModuleImport};
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
 pub mod node;
+mod parse_fn;
 
-pub trait ImportResolver: Send + Sync {
+trait ImportResolver: Send + Sync {
     fn resolve(&self, project_dir: &Path, from: &Path, import: &str) -> ModuleImport;
 }
 
+trait FnSourceParser: Send + Sync {
+    fn collect_runtime_sources(&self, project_dir: &Path) -> Vec<FnSource>;
+    fn parse(&self, project_dir: &Path, source_path: PathBuf) -> FnBuildResult<FnSource>;
+}
+
+#[derive(Clone)]
 pub enum Runtime {
     Node(Arc<NodeConfig>),
 }

fn_build/src/runtime/node/build_node_fn.rs

@@ -0,0 +1,50 @@
+use crate::fs::copy_dir_all;
+use crate::runtime::node::parse_node_fn;
+use crate::swc::compiler::SwcCompiler;
+use crate::{BuildMode, FnBuild, FnBuildResult, FnBuildSpec, FnDependencies, FnSource};
+use std::fs;
+use std::path::Path;
+
+pub async fn build_node_fn(build_spec: FnBuildSpec) -> FnBuildResult<FnBuild> {
+    let manifest = parse_node_fn(build_spec.function.clone()).await?;
+    if let FnDependencies::Required = manifest.dependencies {
+        copy_dir_all(
+            &build_spec.function.project_dir.join("node_modules"),
+            &build_spec.output.join("node_modules"),
+        )?;
+    }
+    for source in &manifest.sources {
+        debug_assert!(source.path.is_relative());
+        let output_path = build_spec.output.join(&source.path);
+        fs::create_dir_all(output_path.parent().unwrap()).expect("mkdir -p");
+        if let Some(extension) = source.path.extension() {
+            if (extension == "js" || extension == "mjs") && build_spec.mode == BuildMode::Release {
+                let js_path = build_spec.function.project_dir.join(&source.path);
+                let minified_js = SwcCompiler::new().minify_js(&js_path).unwrap();
+                fs::write(output_path, minified_js)?;
+                continue;
+            }
+        }
+        fs::copy(
+            build_spec.function.project_dir.join(&source.path),
+            output_path,
+        )
+        .expect("cp");
+    }
+    Ok(FnBuild {
+        manifest,
+        output: build_spec.output,
+    })
+}
+
+#[allow(unused)]
+async fn copy_sources(
+    project_dir: &Path,
+    sources: &Vec<FnSource>,
+    output_path: &Path,
+) -> FnBuildResult<()> {
+    for source in sources {
+        fs::copy(project_dir.join(&source.path), output_path)?;
+    }
+    Ok(())
+}

fn_build/src/runtime/node/imports/resolver.rs

@@ -1,10 +1,10 @@
 use crate::paths::join_file_paths;
-use crate::result::ModuleImport;
 use crate::runtime::node::imports::{
     NodeSubpathImportAsterisks, NodeSubpathImportMapWildcardTo, NodeSubpathImportMapping,
 };
 use crate::runtime::node::NodeConfig;
 use crate::runtime::ImportResolver;
+use crate::ModuleImport;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;