From 87d1ef64ad33543becde1af9c17444943215eb08 Mon Sep 17 00:00:00 2001 From: Monica Tang Date: Mon, 9 Sep 2024 10:08:58 -0700 Subject: [PATCH] Add ability to filter out files excluded by excludes_extensions option Reviewed By: tyao1 Differential Revision: D61721120 fbshipit-source-id: 21963a58f1cf12c3b458cf97ab3f61d4a28049e4 --- compiler/crates/relay-compiler/Cargo.toml | 1 + .../relay-compiler-config-schema.json | 10 + compiler/crates/relay-compiler/src/config.rs | 17 ++ .../src/file_source/file_categorizer.rs | 176 +++++++++++++----- compiler/crates/relay-config/Cargo.toml | 1 + .../crates/relay-config/src/project_config.rs | 6 +- .../crates/relay-lsp/src/server/lsp_state.rs | 29 +-- .../src/server/lsp_state_resources.rs | 2 +- compiler/crates/relay-lsp/src/utils.rs | 4 +- 9 files changed, 181 insertions(+), 65 deletions(-) diff --git a/compiler/crates/relay-compiler/Cargo.toml b/compiler/crates/relay-compiler/Cargo.toml index 8dd8eb4d71ff2..ea1a52c26d0b8 100644 --- a/compiler/crates/relay-compiler/Cargo.toml +++ b/compiler/crates/relay-compiler/Cargo.toml @@ -39,6 +39,7 @@ extract-graphql = { path = "../extract-graphql" } fnv = "1.0" futures = { version = "0.3.30", features = ["async-await", "compat"] } glob = "0.3" +globset = { version = "0.4.13", features = ["serde1"] } graphql-cli = { path = "../graphql-cli" } graphql-ir = { path = "../graphql-ir" } graphql-syntax = { path = "../graphql-syntax" } diff --git a/compiler/crates/relay-compiler/relay-compiler-config-schema.json b/compiler/crates/relay-compiler/relay-compiler-config-schema.json index 139eae5c8571f..99363ef000cc3 100644 --- a/compiler/crates/relay-compiler/relay-compiler-config-schema.json +++ b/compiler/crates/relay-compiler/relay-compiler-config-schema.json @@ -3017,6 +3017,16 @@ "null" ] }, + "excludesExtensions": { + "description": "Some projects may need to exclude files with certain extensions.", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + } + }, "extra": { "description": "A placeholder for allowing extra information in the config file", "default": null diff --git a/compiler/crates/relay-compiler/src/config.rs b/compiler/crates/relay-compiler/src/config.rs index 733948b8d2c23..729882bbab63c 100644 --- a/compiler/crates/relay-compiler/src/config.rs +++ b/compiler/crates/relay-compiler/src/config.rs @@ -21,6 +21,8 @@ use common::ScalarName; use dunce::canonicalize; use fnv::FnvBuildHasher; use fnv::FnvHashSet; +use globset::Glob; +use globset::GlobSetBuilder; use graphql_ir::OperationDefinition; use graphql_ir::Program; use indexmap::IndexMap; @@ -385,6 +387,17 @@ impl Config { }], })?; + let excludes_extensions_set = + config_file_project + .excludes_extensions + .map(move |extensions| { + let mut builder = GlobSetBuilder::new(); + for ext in extensions { + builder.add(Glob::new(&ext).unwrap()); + } + builder.build().unwrap() + }); + let project_config = ProjectConfig { name: project_name, base: config_file_project.base, @@ -392,6 +405,7 @@ impl Config { schema_extensions: config_file_project.schema_extensions, extra_artifacts_config: None, extra: config_file_project.extra, + excludes_extensions: excludes_extensions_set, output: config_file_project.output, extra_artifacts_output: config_file_project.extra_artifacts_output, shard_output: config_file_project.shard_output, @@ -994,6 +1008,9 @@ pub struct ConfigFileProject { /// By default the will use `output` *if available extra_artifacts_output: Option, + /// Some projects may need to exclude files with certain extensions. + excludes_extensions: Option>, + /// If `output` is provided and `shard_output` is `true`, shard the files /// by putting them under `{output_dir}/{source_relative_path}` #[serde(default)] diff --git a/compiler/crates/relay-compiler/src/file_source/file_categorizer.rs b/compiler/crates/relay-compiler/src/file_source/file_categorizer.rs index 25c678b04d5ea..3514924ee0425 100644 --- a/compiler/crates/relay-compiler/src/file_source/file_categorizer.rs +++ b/compiler/crates/relay-compiler/src/file_source/file_categorizer.rs @@ -60,13 +60,16 @@ pub fn categorize_files( name: (*file.name).clone(), exists: *file.exists, }; - let file_group = categorizer.categorize(&file.name).unwrap_or_else(|err| { - panic!( - "Unexpected error in file categorizer for file `{}`: {}.", - file.name.to_string_lossy(), - err - ) - }); + let file_group = + categorizer + .categorize(&file.name, config) + .unwrap_or_else(|err| { + panic!( + "Unexpected error in file categorizer for file `{}`: {}.", + file.name.to_string_lossy(), + err + ) + }); (file_group, file) }) .filter(|(file_group, _)| { @@ -114,7 +117,7 @@ fn categorize_non_watchman_files( .filter(|file| file_filter.is_file_relevant(&file.name)) .filter_map(|file| { let file_group = categorizer - .categorize(&file.name) + .categorize(&file.name, config) .map_err(|err| { warn!( "Unexpected error in file categorizer for file `{}`: {}.", @@ -240,7 +243,7 @@ impl FileCategorizer { /// Categorizes a file. This method should be kept as cheap as possible by /// preprocessing the config in `from_config` and then re-using the /// `FileCategorizer`. - pub fn categorize(&self, path: &Path) -> Result> { + pub fn categorize(&self, path: &Path, config: &Config) -> Result> { let extension = path.extension(); let in_generated_sources = self @@ -271,29 +274,33 @@ impl FileCategorizer { .source_mapping .find(path) .ok_or(Cow::Borrowed("File is not in any source set."))?; - - let in_generated_dir = in_relative_generated_dir(self.default_generated_dir, path); - // If the path is in a generated directory and is not a generated source - // Some generated files can be treated as sources files. For example, resolver codegen. - if in_generated_dir && !in_generated_sources { - if project_set.has_multiple_projects() { - Err(Cow::Owned(format!( - "Overlapping input sources are incompatible with relative generated \ + let filtered_project_set = filter_projects_for_path(config, path, project_set); + if let Some(project_set) = filtered_project_set { + let in_generated_dir = in_relative_generated_dir(self.default_generated_dir, path); + // If the path is in a generated directory and is not a generated source + // Some generated files can be treated as sources files. For example, resolver codegen. + if in_generated_dir && !in_generated_sources { + if project_set.has_multiple_projects() { + Err(Cow::Owned(format!( + "Overlapping input sources are incompatible with relative generated \ directories. Got file in a relative generated directory with source set {:?}.", - project_set, - ))) + project_set, + ))) + } else { + let project_name = project_set.into_iter().next().unwrap(); + Ok(FileGroup::Generated { project_name }) + } } else { - let project_name = project_set.into_iter().next().unwrap(); - Ok(FileGroup::Generated { project_name }) + let is_valid_extension = + self.is_valid_extension_for_project_set(&project_set, extension, path); + if is_valid_extension { + Ok(FileGroup::Source { project_set }) + } else { + Err(Cow::Borrowed("Invalid extension for a generated file.")) + } } } else { - let is_valid_extension = - self.is_valid_extension_for_project_set(&project_set, extension, path); - if is_valid_extension { - Ok(FileGroup::Source { project_set }) - } else { - Err(Cow::Borrowed("Invalid extension for a generated file.")) - } + Ok(FileGroup::Ignore) } } else if is_schema_extension(extension) { if let Some(project_set) = self.schema_file_mapping.get(path) { @@ -359,6 +366,33 @@ impl PathMapping { } } +fn filter_projects_for_path( + config: &Config, + path: &Path, + project_set: ProjectSet, +) -> Option { + let project_names: Vec = project_set + .iter() + .filter(|project_name| { + if let Some(project_config) = config.projects.get(*project_name) { + if let Some(glob_set) = &project_config.excludes_extensions { + !glob_set.is_match(path) + } else { + true + } + } else { + true + } + }) + .cloned() + .collect(); + if project_names.is_empty() { + None + } else { + Some(ProjectSet::new(project_names)) + } +} + fn in_relative_generated_dir(default_generated_dir: &OsStr, path: &Path) -> bool { path.components().any(|comp| match comp { Component::Normal(comp) => comp == default_generated_dir, @@ -403,7 +437,9 @@ mod tests { "src/vendor": "public", "src/custom": "with_custom_generated_dir", "src/typescript": "typescript", - "src/custom_overlapping": ["with_custom_generated_dir", "overlapping_generated_dir"] + "src/custom_overlapping": ["with_custom_generated_dir", "overlapping_generated_dir"], + "src/react_native.native.js": ["public"], + "src/component.react.native.js": ["public"] }, "generatedSources": { "src/resolver_codegen/__generated__": "public" @@ -411,7 +447,10 @@ mod tests { "projects": { "public": { "schema": "graphql/public.graphql", - "language": "flow" + "language": "flow", + "excludesExtensions": [ + "*.native.js" + ] }, "internal": { "schema": "graphql/__generated__/internal.graphql", @@ -444,7 +483,7 @@ mod tests { assert_eq!( categorizer - .categorize(&PathBuf::from("src/js/a.js")) + .categorize(&PathBuf::from("src/js/a.js"), &config) .unwrap(), FileGroup::Source { project_set: ProjectSet::of("public".intern().into()), @@ -452,7 +491,7 @@ mod tests { ); assert_eq!( categorizer - .categorize(&PathBuf::from("src/js/__generated__/a.js")) + .categorize(&PathBuf::from("src/js/__generated__/a.js"), &config) .unwrap(), FileGroup::Generated { project_name: "public".intern().into(), @@ -460,9 +499,10 @@ mod tests { ); assert_eq!( categorizer - .categorize(&PathBuf::from( - "src/resolver_codegen/__generated__/resolvers.js" - )) + .categorize( + &PathBuf::from("src/resolver_codegen/__generated__/resolvers.js"), + &config + ) .unwrap(), FileGroup::Source { project_set: ProjectSet::of("public".intern().into()), @@ -470,7 +510,7 @@ mod tests { ); assert_eq!( categorizer - .categorize(&PathBuf::from("src/js/nested/b.js")) + .categorize(&PathBuf::from("src/js/nested/b.js"), &config) .unwrap(), FileGroup::Source { project_set: ProjectSet::of("public".intern().into()), @@ -478,7 +518,7 @@ mod tests { ); assert_eq!( categorizer - .categorize(&PathBuf::from("src/js/internal/nested/c.js")) + .categorize(&PathBuf::from("src/js/internal/nested/c.js"), &config) .unwrap(), FileGroup::Source { project_set: ProjectSet::of("internal".intern().into()), @@ -490,7 +530,7 @@ mod tests { // Path is only categorized as generated if it matches the absolute path // of the provided custom output. categorizer - .categorize(&PathBuf::from("src/custom/custom-generated/c.js")) + .categorize(&PathBuf::from("src/custom/custom-generated/c.js"), &config) .unwrap(), FileGroup::Source { project_set: ProjectSet::of("with_custom_generated_dir".intern().into()), @@ -498,7 +538,10 @@ mod tests { ); assert_eq!( categorizer - .categorize(&PathBuf::from("src/js/internal/nested/__generated__/c.js")) + .categorize( + &PathBuf::from("src/js/internal/nested/__generated__/c.js"), + &config + ) .unwrap(), FileGroup::Generated { project_name: "internal".intern().into() @@ -506,7 +549,7 @@ mod tests { ); assert_eq!( categorizer - .categorize(&PathBuf::from("graphql/custom-generated/c.js")) + .categorize(&PathBuf::from("graphql/custom-generated/c.js"), &config) .unwrap(), FileGroup::Generated { project_name: "with_custom_generated_dir".intern().into() @@ -514,7 +557,7 @@ mod tests { ); assert_eq!( categorizer - .categorize(&PathBuf::from("graphql/public.graphql")) + .categorize(&PathBuf::from("graphql/public.graphql"), &config) .unwrap(), FileGroup::Schema { project_set: ProjectSet::of("public".intern().into()) @@ -522,7 +565,10 @@ mod tests { ); assert_eq!( categorizer - .categorize(&PathBuf::from("graphql/__generated__/internal.graphql")) + .categorize( + &PathBuf::from("graphql/__generated__/internal.graphql"), + &config + ) .unwrap(), FileGroup::Schema { project_set: ProjectSet::of("internal".intern().into()) @@ -530,7 +576,7 @@ mod tests { ); assert_eq!( categorizer - .categorize(&PathBuf::from("src/typescript/a.ts")) + .categorize(&PathBuf::from("src/typescript/a.ts"), &config) .unwrap(), FileGroup::Source { project_set: ProjectSet::of("typescript".intern().into()), @@ -543,7 +589,7 @@ mod tests { let config = create_test_config(); let categorizer = FileCategorizer::from_config(&config); assert_eq!( - categorizer.categorize(&PathBuf::from("src/js/a.cpp")), + categorizer.categorize(&PathBuf::from("src/js/a.cpp"), &config), Err(Cow::Borrowed( "File categorizer encounter a file with unsupported extension." )) @@ -555,7 +601,7 @@ mod tests { let config = create_test_config(); let categorizer = FileCategorizer::from_config(&config); assert_eq!( - categorizer.categorize(&PathBuf::from("src/js/a.ts")), + categorizer.categorize(&PathBuf::from("src/js/a.ts"), &config), Err(Cow::Borrowed("Invalid extension for a generated file.")) ); } @@ -566,29 +612,61 @@ mod tests { let categorizer = FileCategorizer::from_config(&config); assert_eq!( - categorizer.categorize(&PathBuf::from("src/js/a.graphql")), + categorizer.categorize(&PathBuf::from("src/js/a.graphql"), &config), Err(Cow::Borrowed( "Expected *.graphql/*.gql file to be either a schema or extension." )), ); assert_eq!( - categorizer.categorize(&PathBuf::from("src/js/a.gql")), + categorizer.categorize(&PathBuf::from("src/js/a.gql"), &config), Err(Cow::Borrowed( "Expected *.graphql/*.gql file to be either a schema or extension." )), ); assert_eq!( - categorizer.categorize(&PathBuf::from("src/js/noextension")), + categorizer.categorize(&PathBuf::from("src/js/noextension"), &config), Err(Cow::Borrowed("Got unexpected path without extension.")), ); assert_eq!( - categorizer.categorize(&PathBuf::from("src/custom_overlapping/__generated__/c.js")), + categorizer.categorize( + &PathBuf::from("src/custom_overlapping/__generated__/c.js"), + &config + ), Err(Cow::Borrowed( "Overlapping input sources are incompatible with relative generated directories. Got file in a relative generated directory with source set ProjectSet([Named(\"with_custom_generated_dir\"), Named(\"overlapping_generated_dir\")])." )), ); } + + #[test] + fn test_filter_projects_for_path() { + let config = create_test_config(); + assert_eq!( + filter_projects_for_path( + &config, + &PathBuf::from("src/react_native.native.js"), + ProjectSet::new(vec![ProjectName::Named("public".intern())]) + ), + None + ); + assert_eq!( + filter_projects_for_path( + &config, + &PathBuf::from("src/component.react.native.js"), + ProjectSet::new(vec![ProjectName::Named("public".intern())]) + ), + None + ); + assert_eq!( + filter_projects_for_path( + &config, + &PathBuf::from("src/typescript"), + ProjectSet::new(vec![ProjectName::Named("public".intern())]) + ), + Some(ProjectSet::new(vec![ProjectName::Named("public".intern())])) + ); + } } diff --git a/compiler/crates/relay-config/Cargo.toml b/compiler/crates/relay-config/Cargo.toml index 330dedca2e7e0..98dd8fd24a7ee 100644 --- a/compiler/crates/relay-config/Cargo.toml +++ b/compiler/crates/relay-config/Cargo.toml @@ -11,6 +11,7 @@ license = "MIT" [dependencies] common = { path = "../common" } fnv = "1.0" +globset = { version = "0.4.13", features = ["serde1"] } indexmap = { version = "2.2.6", features = ["arbitrary", "rayon", "serde"] } intern = { path = "../intern" } pathdiff = "0.2" diff --git a/compiler/crates/relay-config/src/project_config.rs b/compiler/crates/relay-config/src/project_config.rs index 23010ea367072..c5445dac96b7a 100644 --- a/compiler/crates/relay-config/src/project_config.rs +++ b/compiler/crates/relay-config/src/project_config.rs @@ -10,7 +10,6 @@ use std::path::Path; use std::path::PathBuf; use std::path::MAIN_SEPARATOR; use std::sync::Arc; -use std::usize; use common::DirectiveName; use common::FeatureFlags; @@ -19,6 +18,7 @@ use common::SourceLocationKey; use common::WithLocation; use fmt::Debug; use fnv::FnvBuildHasher; +use globset::GlobSet; use indexmap::IndexMap; use intern::string_key::Intern; use intern::string_key::StringKey; @@ -232,6 +232,7 @@ pub struct ProjectConfig { pub base: Option, pub extra_artifacts_output: Option, pub extra_artifacts_config: Option, + pub excludes_extensions: Option, pub output: Option, pub shard_output: bool, pub shard_strip_regex: Option, @@ -261,6 +262,7 @@ impl Default for ProjectConfig { base: None, extra_artifacts_output: None, extra_artifacts_config: None, + excludes_extensions: None, output: None, shard_output: false, shard_strip_regex: None, @@ -290,6 +292,7 @@ impl Debug for ProjectConfig { base, extra_artifacts_output, extra_artifacts_config, + excludes_extensions, output, shard_output, shard_strip_regex, @@ -316,6 +319,7 @@ impl Debug for ProjectConfig { .field("output", output) .field("extra_artifacts_config", extra_artifacts_config) .field("extra_artifacts_output", extra_artifacts_output) + .field("excludes_extensions", excludes_extensions) .field("shard_output", shard_output) .field("shard_strip_regex", shard_strip_regex) .field("schema_extensions", schema_extensions) diff --git a/compiler/crates/relay-lsp/src/server/lsp_state.rs b/compiler/crates/relay-lsp/src/server/lsp_state.rs index eb304666a5268..b03b3d929fd3a 100644 --- a/compiler/crates/relay-lsp/src/server/lsp_state.rs +++ b/compiler/crates/relay-lsp/src/server/lsp_state.rs @@ -380,7 +380,7 @@ impl) { self.insert_synced_js_sources(uri, sources); - self.schedule_task(Task::ValidateSyncedSource(uri.clone())); + self.schedule_task(Task::SyncedSource(uri.clone())); } fn remove_synced_js_sources(&self, url: &Url) { @@ -396,7 +396,7 @@ impl LSPRuntimeResult { - let file_group = get_file_group_from_uri(&self.file_categorizer, url, &self.root_dir)?; + let file_group = + get_file_group_from_uri(&self.file_categorizer, url, &self.root_dir, &self.config)?; get_project_name_from_file_group(&file_group).map_err(|msg| { LSPRuntimeError::UnexpectedError(format!( @@ -575,7 +576,8 @@ impl LSPRuntimeResult<()> { - let file_group = get_file_group_from_uri(&self.file_categorizer, uri, &self.root_dir)?; + let file_group = + get_file_group_from_uri(&self.file_categorizer, uri, &self.root_dir, &self.config)?; let project_name = get_project_name_from_file_group(&file_group).map_err(|msg| { LSPRuntimeError::UnexpectedError(format!( "Could not determine project name for \"{}\": {}", @@ -609,7 +611,8 @@ impl LSPRuntimeResult<()> { - let file_group = get_file_group_from_uri(&self.file_categorizer, uri, &self.root_dir)?; + let file_group = + get_file_group_from_uri(&self.file_categorizer, uri, &self.root_dir, &self.config)?; match file_group { FileGroup::Schema { project_set: _ } | FileGroup::Extension { project_set: _ } => { @@ -677,9 +680,9 @@ pub fn build_ir_for_lsp( #[derive(Debug)] pub enum Task { - ValidateSyncedDocuments, - ValidateSyncedSource(Url), - ValidateSchemaSource(Url), + SyncedDocuments, + SyncedSource(Url), + SchemaSource(Url), } pub(crate) fn handle_lsp_state_tasks< @@ -690,19 +693,19 @@ pub(crate) fn handle_lsp_state_tasks< task: Task, ) { match task { - Task::ValidateSyncedDocuments => { + Task::SyncedDocuments => { for item in &state.synced_javascript_sources { - state.schedule_task(Task::ValidateSyncedSource(item.key().clone())); + state.schedule_task(Task::SyncedSource(item.key().clone())); } for item in &state.synced_schema_sources { - state.schedule_task(Task::ValidateSchemaSource(item.key().clone())); + state.schedule_task(Task::SchemaSource(item.key().clone())); } } - Task::ValidateSyncedSource(url) => { + Task::SyncedSource(url) => { state.validate_synced_js_sources(&url).ok(); } - Task::ValidateSchemaSource(url) => { + Task::SchemaSource(url) => { state.validate_synced_schema_source(&url).ok(); } } diff --git a/compiler/crates/relay-lsp/src/server/lsp_state_resources.rs b/compiler/crates/relay-lsp/src/server/lsp_state_resources.rs index 4ffde640aa96c..7323a323e8dfe 100644 --- a/compiler/crates/relay-lsp/src/server/lsp_state_resources.rs +++ b/compiler/crates/relay-lsp/src/server/lsp_state_resources.rs @@ -337,7 +337,7 @@ impl LSPRuntimeResult { let absolute_file_path = url.to_file_path().map_err(|_| { LSPRuntimeError::UnexpectedError(format!("Unable to convert URL to file path: {:?}", url)) @@ -94,7 +96,7 @@ pub fn get_file_group_from_uri( )) })?; - file_categorizer.categorize(file_path).map_err(|_| { + file_categorizer.categorize(file_path, config).map_err(|_| { LSPRuntimeError::UnexpectedError(format!( "Unable to categorize the file correctly: {:?}", file_path