From 3fff36881a8aabdfd13dd2f6047dabf28e6736db Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 28 Feb 2025 12:23:36 +0100 Subject: [PATCH 1/7] chore(translations): localize to languages other than English (#7633) Co-authored-by: GitHub Action --- ui/src/translations/de.json | 3 ++- ui/src/translations/es.json | 3 ++- ui/src/translations/fr.json | 3 ++- ui/src/translations/hi.json | 3 ++- ui/src/translations/it.json | 3 ++- ui/src/translations/ja.json | 3 ++- ui/src/translations/ko.json | 3 ++- ui/src/translations/pl.json | 3 ++- ui/src/translations/pt.json | 3 ++- ui/src/translations/ru.json | 3 ++- ui/src/translations/zh_CN.json | 3 ++- 11 files changed, 22 insertions(+), 11 deletions(-) diff --git a/ui/src/translations/de.json b/ui/src/translations/de.json index 99485edcf21..e2610f2f980 100644 --- a/ui/src/translations/de.json +++ b/ui/src/translations/de.json @@ -522,7 +522,8 @@ "settings": { "label": "Seiteneinstellungen", "show_chart": "Hauptdiagramm anzeigen" - } + }, + "text_search": "Nach diesem Text suchen" }, "flow": "Flow", "flow already exists": "Flow existiert bereits", diff --git a/ui/src/translations/es.json b/ui/src/translations/es.json index e9e62525bf2..d24d6654513 100644 --- a/ui/src/translations/es.json +++ b/ui/src/translations/es.json @@ -522,7 +522,8 @@ "settings": { "label": "Configuración de la página", "show_chart": "Mostrar gráfico principal" - } + }, + "text_search": "Buscar este texto" }, "flow": "Flujo", "flow already exists": "Flow ya existe", diff --git a/ui/src/translations/fr.json b/ui/src/translations/fr.json index b84dee99eed..35690c5c2be 100644 --- a/ui/src/translations/fr.json +++ b/ui/src/translations/fr.json @@ -522,7 +522,8 @@ "settings": { "label": "Paramètres de la page", "show_chart": "Afficher le graphique principal" - } + }, + "text_search": "Rechercher ce texte" }, "flow": "Flow", "flow already exists": "Flow déjà existant", diff --git a/ui/src/translations/hi.json b/ui/src/translations/hi.json index 08ec04487b6..e3528a66a82 100644 --- a/ui/src/translations/hi.json +++ b/ui/src/translations/hi.json @@ -522,7 +522,8 @@ "settings": { "label": "पृष्ठ सेटिंग्स", "show_chart": "मुख्य चार्ट दिखाएं" - } + }, + "text_search": "इस पाठ के लिए खोजें" }, "flow": "Flow", "flow already exists": "Flow पहले से मौजूद है", diff --git a/ui/src/translations/it.json b/ui/src/translations/it.json index dbc651c4183..963c0ddcf2c 100644 --- a/ui/src/translations/it.json +++ b/ui/src/translations/it.json @@ -522,7 +522,8 @@ "settings": { "label": "Impostazioni pagina", "show_chart": "Mostra il grafico principale" - } + }, + "text_search": "Cerca questo testo" }, "flow": "Flow", "flow already exists": "Flow già esistente", diff --git a/ui/src/translations/ja.json b/ui/src/translations/ja.json index f976be193a9..4f85b41cb22 100644 --- a/ui/src/translations/ja.json +++ b/ui/src/translations/ja.json @@ -522,7 +522,8 @@ "settings": { "label": "ページ設定", "show_chart": "メインチャートを表示" - } + }, + "text_search": "このテキストを検索" }, "flow": "Flow", "flow already exists": "Flowはすでに存在します", diff --git a/ui/src/translations/ko.json b/ui/src/translations/ko.json index 899c3f3452e..bc43c326bf5 100644 --- a/ui/src/translations/ko.json +++ b/ui/src/translations/ko.json @@ -522,7 +522,8 @@ "settings": { "label": "페이지 설정", "show_chart": "주요 차트 표시" - } + }, + "text_search": "이 텍스트 검색" }, "flow": "Flow", "flow already exists": "Flow가 이미 존재합니다", diff --git a/ui/src/translations/pl.json b/ui/src/translations/pl.json index efc4208800e..a8497a91e83 100644 --- a/ui/src/translations/pl.json +++ b/ui/src/translations/pl.json @@ -522,7 +522,8 @@ "settings": { "label": "Ustawienia strony", "show_chart": "Pokaż główny wykres" - } + }, + "text_search": "Wyszukaj ten tekst" }, "flow": "Flow", "flow already exists": "Flow już istnieje", diff --git a/ui/src/translations/pt.json b/ui/src/translations/pt.json index db10b096905..746174b2514 100644 --- a/ui/src/translations/pt.json +++ b/ui/src/translations/pt.json @@ -522,7 +522,8 @@ "settings": { "label": "Configurações da página", "show_chart": "Exibir gráfico principal" - } + }, + "text_search": "Pesquisar por este texto" }, "flow": "Flow", "flow already exists": "Flow já existe", diff --git a/ui/src/translations/ru.json b/ui/src/translations/ru.json index 6d048aa48f7..c6d202c3519 100644 --- a/ui/src/translations/ru.json +++ b/ui/src/translations/ru.json @@ -522,7 +522,8 @@ "settings": { "label": "Настройки страницы", "show_chart": "Показать основную диаграмму" - } + }, + "text_search": "Искать этот текст" }, "flow": "Flow", "flow already exists": "Flow уже существует", diff --git a/ui/src/translations/zh_CN.json b/ui/src/translations/zh_CN.json index 79e11d82686..92032994e8d 100644 --- a/ui/src/translations/zh_CN.json +++ b/ui/src/translations/zh_CN.json @@ -522,7 +522,8 @@ "settings": { "label": "页面设置", "show_chart": "显示主图表" - } + }, + "text_search": "搜索此文本" }, "flow": "流程", "flow already exists": "流程已存在", From 446a034d6b051584f1a93f5b5d9c5a1baa21ecb0 Mon Sep 17 00:00:00 2001 From: MilosPaunovic Date: Fri, 28 Feb 2025 12:48:38 +0100 Subject: [PATCH 2/7] fix(ui): additional check for text label of filters section --- ui/src/components/filter/KestraFilter.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/components/filter/KestraFilter.vue b/ui/src/components/filter/KestraFilter.vue index 60316b6d6f3..e56e74a6b82 100644 --- a/ui/src/components/filter/KestraFilter.vue +++ b/ui/src/components/filter/KestraFilter.vue @@ -672,8 +672,8 @@ (options) => { if (options.length || !dropdowns.value.first?.shown) return; - if (!getInputValue().startsWith(TEXT_PREFIX)) { - select.value!.states.inputValue = `${TEXT_PREFIX}${getInputValue()}`; + if (!getInputValue()?.startsWith(TEXT_PREFIX) && select.value?.states?.inputValue) { + select.value.states.inputValue = `${TEXT_PREFIX}${getInputValue()}`; } }, {immediate: true}, From 036a7cf4f7570a4497e5b9e4cbce339705ceeb4f Mon Sep 17 00:00:00 2001 From: MilosPaunovic Date: Fri, 28 Feb 2025 14:00:32 +0100 Subject: [PATCH 3/7] fix(ui): improve check for text label of filters section --- ui/src/components/filter/KestraFilter.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/components/filter/KestraFilter.vue b/ui/src/components/filter/KestraFilter.vue index e56e74a6b82..c082e171bc9 100644 --- a/ui/src/components/filter/KestraFilter.vue +++ b/ui/src/components/filter/KestraFilter.vue @@ -672,7 +672,7 @@ (options) => { if (options.length || !dropdowns.value.first?.shown) return; - if (!getInputValue()?.startsWith(TEXT_PREFIX) && select.value?.states?.inputValue) { + if (!getInputValue()?.startsWith(TEXT_PREFIX) && select.value) { select.value.states.inputValue = `${TEXT_PREFIX}${getInputValue()}`; } }, From 5f21eb57908332642b2432f01400a0b51132ebe5 Mon Sep 17 00:00:00 2001 From: "brian.mulier" Date: Fri, 28 Feb 2025 14:16:05 +0100 Subject: [PATCH 4/7] fix(ui): use watch with ref instead of accessing the value --- ui/src/components/filter/KestraFilter.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/components/filter/KestraFilter.vue b/ui/src/components/filter/KestraFilter.vue index c082e171bc9..0b0044b8487 100644 --- a/ui/src/components/filter/KestraFilter.vue +++ b/ui/src/components/filter/KestraFilter.vue @@ -184,7 +184,7 @@ import {computed, nextTick, onMounted, ref, shallowRef, watch} from "vue"; import {ElSelect} from "element-plus"; - import {Buttons, CurrentItem, Shown, Pair, Property} from "./utils/types"; + import {Buttons, CurrentItem, Pair, Property, Shown} from "./utils/types"; import Refresh from "../layout/RefreshButton.vue"; import Items from "./segments/Items.vue"; @@ -668,7 +668,7 @@ }); watch( - () => includedOptions.value, + includedOptions, (options) => { if (options.length || !dropdowns.value.first?.shown) return; From a098847c6590dc6ffd5c8b084995377afde4f6a6 Mon Sep 17 00:00:00 2001 From: Florian Hussonnois Date: Sun, 8 Dec 2024 22:45:35 +0100 Subject: [PATCH 5/7] feat(core): enhance plugin management Changes: * add new interface PluginManager * add new CLI for un-installing plugins * add new option --locally to CLI plugin install * refactor service for downloading plugins * refactor PluginController * move Version util class from EE to OSS * migrate aether lib to maven-resolver (#915) part-of: #915 --- cli/build.gradle | 12 - .../java/io/kestra/cli/AbstractCommand.java | 44 ++- .../cli/commands/plugins/PluginCommand.java | 18 +- .../commands/plugins/PluginDocCommand.java | 5 +- .../plugins/PluginInstallCommand.java | 105 ++--- .../commands/plugins/PluginListCommand.java | 18 +- .../plugins/PluginUninstallCommand.java | 69 ++++ .../kestra/cli/plugins/PluginDownloader.java | 153 -------- .../kestra/cli/plugins/RepositoryConfig.java | 30 -- .../commands/plugins/PluginCommandTest.java | 27 ++ .../plugins/PluginInstallCommandTest.java | 13 +- .../plugins/PluginListCommandTest.java | 3 +- core/build.gradle | 7 + .../contexts/MavenPluginRepositoryConfig.java | 26 ++ .../core/docs/AbstractClassDocumentation.java | 1 + .../core/docs/ClassPluginDocumentation.java | 29 +- .../core/docs/DocumentationGenerator.java | 17 +- .../io/kestra/core/docs/JsonSchemaCache.java | 65 ++++ .../main/java/io/kestra/core/docs/Plugin.java | 2 +- .../java/io/kestra/core/docs/SchemaType.java | 23 +- .../models/hierarchies/SubflowGraphTask.java | 5 + .../io/kestra/core/models/tasks/Task.java | 2 + .../core/models/tasks/TaskForExecution.java | 2 + .../core/models/tasks/TaskInterface.java | 6 + .../core/models/triggers/AbstractTrigger.java | 2 + .../triggers/AbstractTriggerForExecution.java | 5 +- .../models/triggers/TriggerInterface.java | 4 + .../core/plugins/DefaultPluginRegistry.java | 233 +++++++++--- .../core/plugins/LocalPluginManager.java | 240 ++++++++++++ .../core/plugins/MavenPluginDownloader.java | 226 +++++++++++ .../kestra/core/plugins/PluginArtifact.java | 166 ++++++++ .../core/plugins/PluginArtifactMetadata.java | 29 ++ .../core/plugins/PluginClassAndMetadata.java | 26 ++ .../io/kestra/core/plugins/PluginManager.java | 132 +++++++ .../kestra/core/plugins/PluginRegistry.java | 43 +++ .../core/plugins/PluginResolutionResult.java | 20 + .../kestra/core/plugins/RegisteredPlugin.java | 6 +- .../plugins/serdes/PluginDeserializer.java | 14 +- .../io/kestra/core/server/ClusterEvent.java | 2 +- .../java/io/kestra/core/utils/Version.java | 359 ++++++++++++++++++ .../docs/ClassPluginDocumentationTest.java | 16 +- .../core/docs/DocumentationGeneratorTest.java | 23 +- .../core/plugins/PluginArtifactTest.java | 76 ++++ .../serdes/PluginDeserializerTest.java | 1 - .../core/serializers/YamlParserTest.java | 2 +- .../io/kestra/core/utils/VersionTest.java | 148 ++++++++ .../java/io/kestra/core/models/Plugin.java | 1 + platform/build.gradle | 15 +- ui/src/components/plugins/Plugin.vue | 62 ++- ui/src/routes/routes.js | 2 +- ui/src/stores/plugins.js | 26 +- .../controllers/api/PluginController.java | 95 +++-- .../controllers/api/PluginControllerTest.java | 2 +- 53 files changed, 2195 insertions(+), 463 deletions(-) create mode 100644 cli/src/main/java/io/kestra/cli/commands/plugins/PluginUninstallCommand.java delete mode 100644 cli/src/main/java/io/kestra/cli/plugins/PluginDownloader.java delete mode 100644 cli/src/main/java/io/kestra/cli/plugins/RepositoryConfig.java create mode 100644 cli/src/test/java/io/kestra/cli/commands/plugins/PluginCommandTest.java create mode 100644 core/src/main/java/io/kestra/core/contexts/MavenPluginRepositoryConfig.java create mode 100644 core/src/main/java/io/kestra/core/docs/JsonSchemaCache.java create mode 100644 core/src/main/java/io/kestra/core/plugins/LocalPluginManager.java create mode 100644 core/src/main/java/io/kestra/core/plugins/MavenPluginDownloader.java create mode 100644 core/src/main/java/io/kestra/core/plugins/PluginArtifact.java create mode 100644 core/src/main/java/io/kestra/core/plugins/PluginArtifactMetadata.java create mode 100644 core/src/main/java/io/kestra/core/plugins/PluginClassAndMetadata.java create mode 100644 core/src/main/java/io/kestra/core/plugins/PluginManager.java create mode 100644 core/src/main/java/io/kestra/core/plugins/PluginResolutionResult.java create mode 100644 core/src/main/java/io/kestra/core/utils/Version.java create mode 100644 core/src/test/java/io/kestra/core/plugins/PluginArtifactTest.java create mode 100644 core/src/test/java/io/kestra/core/utils/VersionTest.java diff --git a/cli/build.gradle b/cli/build.gradle index 8f1d9f42b0e..80cec16069b 100644 --- a/cli/build.gradle +++ b/cli/build.gradle @@ -12,18 +12,6 @@ dependencies { implementation 'ch.qos.logback.contrib:logback-json-classic' implementation 'ch.qos.logback.contrib:logback-jackson' - // plugins - implementation 'org.eclipse.aether:aether-api' - implementation 'org.eclipse.aether:aether-spi' - implementation 'org.eclipse.aether:aether-util' - implementation 'org.eclipse.aether:aether-impl' - implementation 'org.eclipse.aether:aether-connector-basic' - implementation 'org.eclipse.aether:aether-transport-file' - implementation 'org.eclipse.aether:aether-transport-http' - implementation('org.apache.maven:maven-aether-provider') { - // sisu dependency injector is not used - exclude group: 'org.eclipse.sisu' - } // aether still use javax.inject compileOnly 'javax.inject:javax.inject:1' diff --git a/cli/src/main/java/io/kestra/cli/AbstractCommand.java b/cli/src/main/java/io/kestra/cli/AbstractCommand.java index 36df2560765..ac9872fa295 100644 --- a/cli/src/main/java/io/kestra/cli/AbstractCommand.java +++ b/cli/src/main/java/io/kestra/cli/AbstractCommand.java @@ -4,16 +4,17 @@ import com.google.common.collect.ImmutableMap; import io.kestra.cli.commands.servers.ServerCommandInterface; import io.kestra.cli.services.StartupHookInterface; -import io.kestra.core.contexts.KestraContext; +import io.kestra.core.plugins.PluginManager; import io.kestra.core.plugins.PluginRegistry; import io.kestra.webserver.services.FlowAutoLoaderService; import io.micronaut.context.ApplicationContext; import io.micronaut.context.env.yaml.YamlPropertySourceLoader; import io.micronaut.core.annotation.Introspected; +import io.micronaut.http.uri.UriBuilder; import io.micronaut.management.endpoint.EndpointDefaultConfiguration; import io.micronaut.runtime.server.EmbeddedServer; +import jakarta.inject.Provider; import lombok.extern.slf4j.Slf4j; -import org.apache.http.client.utils.URIBuilder; import io.kestra.core.utils.Rethrow; import picocli.CommandLine; @@ -26,10 +27,13 @@ import java.text.MessageFormat; import java.time.temporal.ChronoUnit; import java.util.Map; +import java.util.Optional; import java.util.concurrent.Callable; import jakarta.inject.Inject; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; -@CommandLine.Command( +@Command( versionProvider = VersionProvider.class, mixinStandardHelpOptions = true, showDefaultValues = true @@ -49,22 +53,28 @@ abstract public class AbstractCommand implements Callable { @Inject private io.kestra.core.utils.VersionProvider versionProvider; + @Inject + protected Provider pluginRegistryProvider; + + @Inject + protected Provider pluginManagerProvider; + private PluginRegistry pluginRegistry; - @CommandLine.Option(names = {"-v", "--verbose"}, description = "Change log level. Multiple -v options increase the verbosity.", showDefaultValue = CommandLine.Help.Visibility.NEVER) + @Option(names = {"-v", "--verbose"}, description = "Change log level. Multiple -v options increase the verbosity.", showDefaultValue = CommandLine.Help.Visibility.NEVER) private boolean[] verbose = new boolean[0]; - @CommandLine.Option(names = {"-l", "--log-level"}, description = "Change log level (values: ${COMPLETION-CANDIDATES})") + @Option(names = {"-l", "--log-level"}, description = "Change log level (values: ${COMPLETION-CANDIDATES})") private LogLevel logLevel = LogLevel.INFO; - @CommandLine.Option(names = {"--internal-log"}, description = "Change also log level for internal log") + @Option(names = {"--internal-log"}, description = "Change also log level for internal log") private boolean internalLog = false; - @CommandLine.Option(names = {"-c", "--config"}, description = "Path to a configuration file") + @Option(names = {"-c", "--config"}, description = "Path to a configuration file") private Path config = Paths.get(System.getProperty("user.home"), ".kestra/config.yml"); - @CommandLine.Option(names = {"-p", "--plugins"}, description = "Path to plugins directory") - protected Path pluginsPath = System.getenv("KESTRA_PLUGINS_PATH") != null ? Paths.get(System.getenv("KESTRA_PLUGINS_PATH")) : null; + @Option(names = {"-p", "--plugins"}, description = "Path to plugins directory") + protected Path pluginsPath = Optional.ofNullable(System.getenv("KESTRA_PLUGINS_PATH")).map(Paths::get).orElse(null); public enum LogLevel { TRACE, @@ -76,7 +86,7 @@ public enum LogLevel { @Override public Integer call() throws Exception { - Thread.currentThread().setName(this.getClass().getDeclaredAnnotation(CommandLine.Command.class).name()); + Thread.currentThread().setName(this.getClass().getDeclaredAnnotation(Command.class).name()); startLogger(); sendServerLog(); if (this.startupHook != null) { @@ -84,8 +94,10 @@ public Integer call() throws Exception { } if (this.pluginsPath != null && loadExternalPlugins()) { - pluginRegistry = pluginRegistry(); + pluginRegistry = pluginRegistryProvider.get(); pluginRegistry.registerIfAbsent(pluginsPath); + PluginManager manager = pluginManagerProvider.get(); + manager.start(); } startWebserver(); @@ -102,10 +114,6 @@ protected boolean loadExternalPlugins() { return true; } - protected PluginRegistry pluginRegistry() { - return KestraContext.getContext().getPluginRegistry(); // Lazy init - } - private static String message(String message, Object... format) { return CommandLine.Help.Ansi.AUTO.string( format.length == 0 ? message : MessageFormat.format(message, format) @@ -183,9 +191,9 @@ private void startWebserver() { if (this.endpointConfiguration.getPort().isPresent()) { URI endpoint = null; try { - endpoint = new URIBuilder(server.getURL().toURI()) - .setPort(this.endpointConfiguration.getPort().get()) - .setPath("/health") + endpoint = UriBuilder.of(server.getURL().toURI()) + .port(this.endpointConfiguration.getPort().get()) + .path("/health") .build(); } catch (URISyntaxException e) { e.printStackTrace(); diff --git a/cli/src/main/java/io/kestra/cli/commands/plugins/PluginCommand.java b/cli/src/main/java/io/kestra/cli/commands/plugins/PluginCommand.java index bae934205e1..720ca17a69e 100644 --- a/cli/src/main/java/io/kestra/cli/commands/plugins/PluginCommand.java +++ b/cli/src/main/java/io/kestra/cli/commands/plugins/PluginCommand.java @@ -1,13 +1,12 @@ package io.kestra.cli.commands.plugins; -import io.micronaut.configuration.picocli.PicocliRunner; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; import io.kestra.cli.AbstractCommand; import io.kestra.cli.App; -import picocli.CommandLine; +import io.micronaut.configuration.picocli.PicocliRunner; +import lombok.SneakyThrows; +import picocli.CommandLine.Command; -@CommandLine.Command( +@Command( name = "plugins", description = "Manage plugins", mixinStandardHelpOptions = true, @@ -18,15 +17,20 @@ PluginSearchCommand.class } ) -@Slf4j public class PluginCommand extends AbstractCommand { + @SneakyThrows @Override public Integer call() throws Exception { super.call(); - PicocliRunner.call(App.class, "plugins", "--help"); + PicocliRunner.call(App.class, "plugins", "--help"); return 0; } + + @Override + protected boolean loadExternalPlugins() { + return false; + } } diff --git a/cli/src/main/java/io/kestra/cli/commands/plugins/PluginDocCommand.java b/cli/src/main/java/io/kestra/cli/commands/plugins/PluginDocCommand.java index 04365908557..ab42a0cc887 100644 --- a/cli/src/main/java/io/kestra/cli/commands/plugins/PluginDocCommand.java +++ b/cli/src/main/java/io/kestra/cli/commands/plugins/PluginDocCommand.java @@ -3,6 +3,7 @@ import com.google.common.io.Files; import io.kestra.cli.AbstractCommand; import io.kestra.core.docs.DocumentationGenerator; +import io.kestra.core.plugins.PluginRegistry; import io.kestra.core.plugins.RegisteredPlugin; import io.kestra.core.serializers.JacksonMapper; import io.micronaut.context.ApplicationContext; @@ -42,8 +43,10 @@ public Integer call() throws Exception { super.call(); DocumentationGenerator documentationGenerator = applicationContext.getBean(DocumentationGenerator.class); - List plugins = core ? pluginRegistry().plugins() : pluginRegistry().externalPlugins(); + PluginRegistry registry = pluginRegistryProvider.get(); + List plugins = core ? registry.plugins() : registry.externalPlugins(); boolean hasFailures = false; + for (RegisteredPlugin registeredPlugin : plugins) { try { documentationGenerator diff --git a/cli/src/main/java/io/kestra/cli/commands/plugins/PluginInstallCommand.java b/cli/src/main/java/io/kestra/cli/commands/plugins/PluginInstallCommand.java index 1a1aea6523b..f6d3cb67c7a 100644 --- a/cli/src/main/java/io/kestra/cli/commands/plugins/PluginInstallCommand.java +++ b/cli/src/main/java/io/kestra/cli/commands/plugins/PluginInstallCommand.java @@ -1,98 +1,103 @@ package io.kestra.cli.commands.plugins; -import org.apache.commons.io.FilenameUtils; +import io.kestra.core.contexts.MavenPluginRepositoryConfig; +import io.kestra.core.plugins.LocalPluginManager; +import io.kestra.core.plugins.MavenPluginDownloader; +import io.kestra.core.plugins.PluginArtifact; +import io.kestra.core.plugins.PluginManager; +import io.micronaut.http.uri.UriBuilder; import io.kestra.cli.AbstractCommand; -import io.kestra.cli.plugins.PluginDownloader; -import io.kestra.cli.plugins.RepositoryConfig; import io.kestra.core.utils.IdUtils; -import org.apache.http.client.utils.URIBuilder; +import jakarta.inject.Provider; import picocli.CommandLine; import java.net.URI; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import jakarta.inject.Inject; +import picocli.CommandLine.Command; +import picocli.CommandLine.Parameters; +import picocli.CommandLine.Option; +import picocli.CommandLine.Spec; -import static io.kestra.core.utils.Rethrow.throwConsumer; - -@CommandLine.Command( +@Command( name = "install", description = "Install plugins" ) public class PluginInstallCommand extends AbstractCommand { - @CommandLine.Parameters(index = "0..*", description = "Plugins to install. Represented as Maven artifact coordinates.") + + @Option(names = {"--locally"}, description = "Specifies if plugins must be installed locally. If set to false the installation depends on your Kestra configuration.") + boolean locally = true; + + @Parameters(index = "0..*", description = "Plugins to install. Represented as Maven artifact coordinates.") List dependencies = new ArrayList<>(); - @CommandLine.Option(names = {"--repositories"}, description = "URL to additional Maven repositories") + @Option(names = {"--repositories"}, description = "URL to additional Maven repositories") private URI[] repositories; - @CommandLine.Spec + @Spec CommandLine.Model.CommandSpec spec; @Inject - private PluginDownloader pluginDownloader; + Provider mavenPluginRepositoryProvider; @Override public Integer call() throws Exception { super.call(); - if (this.pluginsPath == null) { + if (this.locally && this.pluginsPath == null) { throw new CommandLine.ParameterException(this.spec.commandLine(), "Missing required options '--plugins' " + "or environment variable 'KESTRA_PLUGINS_PATH" ); } - if (!pluginsPath.toFile().exists()) { - if (!pluginsPath.toFile().mkdir()) { - throw new RuntimeException("Cannot create directory: " + pluginsPath.toFile().getAbsolutePath()); - } - } - + List repositoryConfigs = List.of(); if (repositories != null) { - Arrays.stream(repositories) - .forEach(throwConsumer(s -> { - URIBuilder uriBuilder = new URIBuilder(s); - - RepositoryConfig.RepositoryConfigBuilder builder = RepositoryConfig.builder() + repositoryConfigs = Arrays.stream(repositories) + .map(uri -> { + MavenPluginRepositoryConfig.MavenPluginRepositoryConfigBuilder builder = MavenPluginRepositoryConfig + .builder() .id(IdUtils.create()); - if (uriBuilder.getUserInfo() != null) { - int index = uriBuilder.getUserInfo().indexOf(":"); - - builder.basicAuth(new RepositoryConfig.BasicAuth( - uriBuilder.getUserInfo().substring(0, index), - uriBuilder.getUserInfo().substring(index + 1) + String userInfo = uri.getUserInfo(); + if (userInfo != null) { + String[] userInfoParts = userInfo.split(":"); + builder = builder.basicAuth(new MavenPluginRepositoryConfig.BasicAuth( + userInfoParts[0], + userInfoParts[1] )); - - uriBuilder.setUserInfo(null); } - - builder.url(uriBuilder.build().toString()); - - pluginDownloader.addRepository(builder.build()); - })); + builder.url(UriBuilder.of(uri).userInfo(null).build().toString()); + return builder.build(); + }).toList(); } - List resolveUrl = pluginDownloader.resolve(dependencies); - stdOut("Resolved Plugin(s) with {0}", resolveUrl); + final List pluginArtifacts; + try { + pluginArtifacts = dependencies.stream().map(PluginArtifact::fromCoordinates).toList(); + } catch (IllegalArgumentException e) { + stdErr(e.getMessage()); + return CommandLine.ExitCode.USAGE; + } - for (URL url: resolveUrl) { - Files.copy( - Paths.get(url.toURI()), - Paths.get(pluginsPath.toString(), FilenameUtils.getName(url.toString())), - StandardCopyOption.REPLACE_EXISTING + try (final PluginManager pluginManager = getPluginManager()) { + List installed = pluginManager.install( + pluginArtifacts, + repositoryConfigs, + false, + pluginsPath ); - } - stdOut("Successfully installed plugins {0} into {1}", dependencies, pluginsPath); + List uris = installed.stream().map(PluginArtifact::uri).toList(); + stdOut("Successfully installed plugins {0} into {1}", dependencies, uris); + return CommandLine.ExitCode.OK; + } + } - return 0; + private PluginManager getPluginManager() { + return locally ? new LocalPluginManager(mavenPluginRepositoryProvider.get()) : this.pluginManagerProvider.get(); } @Override diff --git a/cli/src/main/java/io/kestra/cli/commands/plugins/PluginListCommand.java b/cli/src/main/java/io/kestra/cli/commands/plugins/PluginListCommand.java index 0445b69d628..dba717e5281 100644 --- a/cli/src/main/java/io/kestra/cli/commands/plugins/PluginListCommand.java +++ b/cli/src/main/java/io/kestra/cli/commands/plugins/PluginListCommand.java @@ -1,22 +1,31 @@ package io.kestra.cli.commands.plugins; import io.kestra.cli.AbstractCommand; +import io.kestra.core.plugins.PluginRegistry; import io.kestra.core.plugins.RegisteredPlugin; +import jakarta.inject.Inject; +import jakarta.inject.Provider; import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Spec; import java.util.List; -@CommandLine.Command( +@Command( name = "list", description = "List all plugins already installed" ) public class PluginListCommand extends AbstractCommand { - @CommandLine.Spec + @Spec CommandLine.Model.CommandSpec spec; - @CommandLine.Option(names = {"--core"}, description = "Also write core tasks plugins") + @Option(names = {"--core"}, description = "Also write core tasks plugins") private boolean core = false; + @Inject + private PluginRegistry registry; + @Override public Integer call() throws Exception { super.call(); @@ -27,7 +36,8 @@ public Integer call() throws Exception { ); } - List plugins = core ? pluginRegistry().plugins() : pluginRegistry().externalPlugins(); + List plugins = core ? registry.plugins() : registry.externalPlugins(); + plugins.forEach(registeredPlugin -> stdOut(registeredPlugin.toString())); return 0; diff --git a/cli/src/main/java/io/kestra/cli/commands/plugins/PluginUninstallCommand.java b/cli/src/main/java/io/kestra/cli/commands/plugins/PluginUninstallCommand.java new file mode 100644 index 00000000000..d9a8f85c055 --- /dev/null +++ b/cli/src/main/java/io/kestra/cli/commands/plugins/PluginUninstallCommand.java @@ -0,0 +1,69 @@ +package io.kestra.cli.commands.plugins; + +import io.kestra.cli.AbstractCommand; +import io.kestra.core.plugins.LocalPluginManager; +import io.kestra.core.plugins.MavenPluginDownloader; +import io.kestra.core.plugins.PluginArtifact; +import io.kestra.core.plugins.PluginManager; +import jakarta.inject.Inject; +import jakarta.inject.Provider; +import picocli.CommandLine; +import picocli.CommandLine.Parameters; +import picocli.CommandLine.Spec; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; + +@CommandLine.Command( + name = "uninstall", + description = "uninstall a plugin" +) +public class PluginUninstallCommand extends AbstractCommand { + @Parameters(index = "0..*", description = "the plugins to uninstall") + List dependencies = new ArrayList<>(); + + @Spec + CommandLine.Model.CommandSpec spec; + + @Inject + Provider mavenPluginRepositoryProvider; + + @Override + public Integer call() throws Exception { + super.call(); + + List pluginArtifacts; + try { + pluginArtifacts = dependencies.stream().map(PluginArtifact::fromCoordinates).toList(); + } catch (IllegalArgumentException e) { + stdErr(e.getMessage()); + return CommandLine.ExitCode.USAGE; + } + + final PluginManager pluginManager; + + // If a PLUGIN_PATH is provided, then use the LocalPluginManager + if (pluginsPath != null) { + pluginManager = new LocalPluginManager(mavenPluginRepositoryProvider.get()); + } else { + // Otherwise, we delegate to the configured plugin-manager. + pluginManager = this.pluginManagerProvider.get(); + } + + List uninstalled = pluginManager.uninstall( + pluginArtifacts, + false, + pluginsPath + ); + + List uris = uninstalled.stream().map(PluginArtifact::uri).toList(); + stdOut("Successfully uninstalled plugins {0} from {1}", dependencies, uris); + return CommandLine.ExitCode.OK; + } + + @Override + protected boolean loadExternalPlugins() { + return false; + } +} diff --git a/cli/src/main/java/io/kestra/cli/plugins/PluginDownloader.java b/cli/src/main/java/io/kestra/cli/plugins/PluginDownloader.java deleted file mode 100644 index 989cd66a90d..00000000000 --- a/cli/src/main/java/io/kestra/cli/plugins/PluginDownloader.java +++ /dev/null @@ -1,153 +0,0 @@ -package io.kestra.cli.plugins; - -import com.google.common.collect.ImmutableList; -import io.micronaut.context.annotation.Value; -import io.micronaut.core.annotation.Nullable; -import jakarta.inject.Inject; -import jakarta.inject.Singleton; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.apache.maven.repository.internal.MavenRepositorySystemUtils; -import org.eclipse.aether.DefaultRepositorySystemSession; -import org.eclipse.aether.RepositorySystem; -import org.eclipse.aether.RepositorySystemSession; -import org.eclipse.aether.artifact.Artifact; -import org.eclipse.aether.artifact.DefaultArtifact; -import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory; -import org.eclipse.aether.impl.DefaultServiceLocator; -import org.eclipse.aether.repository.LocalRepository; -import org.eclipse.aether.repository.RemoteRepository; -import org.eclipse.aether.resolution.*; -import org.eclipse.aether.spi.connector.RepositoryConnectorFactory; -import org.eclipse.aether.spi.connector.transport.TransporterFactory; -import org.eclipse.aether.transport.file.FileTransporterFactory; -import org.eclipse.aether.transport.http.HttpTransporterFactory; -import org.eclipse.aether.util.repository.AuthenticationBuilder; - -import java.io.File; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -@Singleton -@Slf4j -public class PluginDownloader { - private final List repositoryConfigs; - private final RepositorySystem system; - private final RepositorySystemSession session; - - @Inject - public PluginDownloader( - List repositoryConfigs, - @Nullable @Value("${kestra.plugins.local-repository-path}") String localRepositoryPath - ) { - this.repositoryConfigs = repositoryConfigs; - this.system = repositorySystem(); - this.session = repositorySystemSession(system, localRepositoryPath); - } - - public void addRepository(RepositoryConfig repositoryConfig) { - this.repositoryConfigs.add(repositoryConfig); - } - - public List resolve(List dependencies) throws MalformedURLException, ArtifactResolutionException, VersionRangeResolutionException { - List repositories = remoteRepositories(); - - List artifactResults = resolveArtifacts(repositories, dependencies); - List localUrls = resolveUrls(artifactResults); - log.debug("Resolved Plugin {} with {}", dependencies, localUrls); - - return localUrls; - } - - private List remoteRepositories() { - return repositoryConfigs - .stream() - .map(repositoryConfig -> { - var build = new RemoteRepository.Builder( - repositoryConfig.getId(), - "default", - repositoryConfig.getUrl() - ); - - if (repositoryConfig.getBasicAuth() != null) { - var authenticationBuilder = new AuthenticationBuilder(); - authenticationBuilder.addUsername(repositoryConfig.getBasicAuth().getUsername()); - authenticationBuilder.addPassword(repositoryConfig.getBasicAuth().getPassword()); - - build.setAuthentication(authenticationBuilder.build()); - } - - return build.build(); - }) - .toList(); - } - - private static RepositorySystem repositorySystem() { - DefaultServiceLocator locator = MavenRepositorySystemUtils.newServiceLocator(); - locator.addService(RepositoryConnectorFactory.class, BasicRepositoryConnectorFactory.class); - locator.addService(TransporterFactory.class, FileTransporterFactory.class); - locator.addService(TransporterFactory.class, HttpTransporterFactory.class); - - return locator.getService(RepositorySystem.class); - } - - private RepositorySystemSession repositorySystemSession(RepositorySystem system, String localRepositoryPath) { - DefaultRepositorySystemSession session = MavenRepositorySystemUtils.newSession(); - - if (localRepositoryPath == null) { - try { - final String tempDirectory = Files.createTempDirectory(this.getClass().getSimpleName().toLowerCase()) - .toAbsolutePath() - .toString(); - - localRepositoryPath = tempDirectory; - - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - FileUtils.deleteDirectory(new File(tempDirectory)); - } catch (IOException e) { - throw new RuntimeException(e); - } - })); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - LocalRepository localRepo = new LocalRepository(localRepositoryPath); - session.setLocalRepositoryManager(system.newLocalRepositoryManager(session, localRepo)); - - return session; - } - - private List resolveArtifacts(List repositories, List dependencies) throws ArtifactResolutionException, VersionRangeResolutionException { - List results = new ArrayList<>(dependencies.size()); - for (String dependency: dependencies) { - var artifact = new DefaultArtifact(dependency); - var version = system.resolveVersionRange(session, new VersionRangeRequest(artifact, repositories, null)); - var artifactRequest = new ArtifactRequest( - new DefaultArtifact(artifact.getGroupId(), artifact.getArtifactId(), "jar", version.getHighestVersion().toString()), - repositories, - null - ); - var artifactResult = system.resolveArtifact(session, artifactRequest); - results.add(artifactResult); - } - return results; - } - - private List resolveUrls(List artifactResults) throws MalformedURLException { - ImmutableList.Builder urls = ImmutableList.builder(); - for (ArtifactResult artifactResult : artifactResults) { - URL url; - url = artifactResult.getArtifact().getFile().toPath().toUri().toURL(); - urls.add(url); - } - return urls.build(); - } -} diff --git a/cli/src/main/java/io/kestra/cli/plugins/RepositoryConfig.java b/cli/src/main/java/io/kestra/cli/plugins/RepositoryConfig.java deleted file mode 100644 index 249acc230ab..00000000000 --- a/cli/src/main/java/io/kestra/cli/plugins/RepositoryConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -package io.kestra.cli.plugins; - -import io.micronaut.context.annotation.EachProperty; -import io.micronaut.context.annotation.Parameter; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Getter; - -@EachProperty("kestra.plugins.repositories") -@Getter -@AllArgsConstructor -@Builder -public class RepositoryConfig { - String id; - - String url; - - BasicAuth basicAuth; - - @Getter - @AllArgsConstructor - public static class BasicAuth { - private String username; - private String password; - } - - public RepositoryConfig(@Parameter String id) { - this.id = id; - } -} diff --git a/cli/src/test/java/io/kestra/cli/commands/plugins/PluginCommandTest.java b/cli/src/test/java/io/kestra/cli/commands/plugins/PluginCommandTest.java new file mode 100644 index 00000000000..9581f5df472 --- /dev/null +++ b/cli/src/test/java/io/kestra/cli/commands/plugins/PluginCommandTest.java @@ -0,0 +1,27 @@ +package io.kestra.cli.commands.plugins; + +import io.micronaut.configuration.picocli.PicocliRunner; +import io.micronaut.context.ApplicationContext; +import io.micronaut.context.env.Environment; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.StringContains.containsString; + +class PluginCommandTest { + + @Test + void shouldGetHelps() { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + System.setOut(new PrintStream(out)); + + try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) { + PicocliRunner.call(PluginCommand.class, ctx); + + assertThat(out.toString(), containsString("Usage: kestra plugins")); + } + } +} \ No newline at end of file diff --git a/cli/src/test/java/io/kestra/cli/commands/plugins/PluginInstallCommandTest.java b/cli/src/test/java/io/kestra/cli/commands/plugins/PluginInstallCommandTest.java index 3b4bb34b679..12668cdcd80 100644 --- a/cli/src/test/java/io/kestra/cli/commands/plugins/PluginInstallCommandTest.java +++ b/cli/src/test/java/io/kestra/cli/commands/plugins/PluginInstallCommandTest.java @@ -9,7 +9,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; -import java.util.stream.Collectors; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.*; @@ -17,7 +16,7 @@ class PluginInstallCommandTest { @Test - void fixedVersion() throws IOException { + void shouldInstallPluginLocallyGivenFixedVersion() throws IOException { Path pluginsPath = Files.createTempDirectory(PluginInstallCommandTest.class.getSimpleName()); pluginsPath.toFile().deleteOnExit(); @@ -28,12 +27,12 @@ void fixedVersion() throws IOException { List files = Files.list(pluginsPath).toList(); assertThat(files.size(), is(1)); - assertThat(files.getFirst().getFileName().toString(), is("plugin-notifications-0.6.0.jar")); + assertThat(files.getFirst().getFileName().toString(), is("io_kestra_plugin__plugin-notifications__0_6_0.jar")); } } @Test - void latestVersion() throws IOException { + void shouldInstallPluginLocallyGivenLatestVersion() throws IOException { Path pluginsPath = Files.createTempDirectory(PluginInstallCommandTest.class.getSimpleName()); pluginsPath.toFile().deleteOnExit(); @@ -44,13 +43,13 @@ void latestVersion() throws IOException { List files = Files.list(pluginsPath).toList(); assertThat(files.size(), is(1)); - assertThat(files.getFirst().getFileName().toString(), startsWith("plugin-notifications")); + assertThat(files.getFirst().getFileName().toString(), startsWith("io_kestra_plugin__plugin-notifications__")); assertThat(files.getFirst().getFileName().toString(), not(containsString("LATEST"))); } } @Test - void rangeVersion() throws IOException { + void shouldInstallPluginLocallyGivenRangeVersion() throws IOException { Path pluginsPath = Files.createTempDirectory(PluginInstallCommandTest.class.getSimpleName()); pluginsPath.toFile().deleteOnExit(); @@ -62,7 +61,7 @@ void rangeVersion() throws IOException { List files = Files.list(pluginsPath).toList(); assertThat(files.size(), is(1)); - assertThat(files.getFirst().getFileName().toString(), is("storage-s3-0.12.1.jar")); + assertThat(files.getFirst().getFileName().toString(), is("io_kestra_storage__storage-s3__0_12_1.jar")); } } } diff --git a/cli/src/test/java/io/kestra/cli/commands/plugins/PluginListCommandTest.java b/cli/src/test/java/io/kestra/cli/commands/plugins/PluginListCommandTest.java index 6cdd17a5f65..c7a7485630e 100644 --- a/cli/src/test/java/io/kestra/cli/commands/plugins/PluginListCommandTest.java +++ b/cli/src/test/java/io/kestra/cli/commands/plugins/PluginListCommandTest.java @@ -4,7 +4,6 @@ import io.micronaut.context.ApplicationContext; import io.micronaut.context.env.Environment; import org.apache.commons.io.FileUtils; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import java.io.ByteArrayOutputStream; @@ -25,7 +24,7 @@ class PluginListCommandTest { private static final String PLUGIN_TEMPLATE_TEST = "plugin-template-test-0.18.0-SNAPSHOT.jar"; @Test - void run() throws IOException, URISyntaxException { + void shouldListPluginsInstalledLocally() throws IOException, URISyntaxException { Path pluginsPath = Files.createTempDirectory(PluginListCommandTest.class.getSimpleName()); pluginsPath.toFile().deleteOnExit(); diff --git a/core/build.gradle b/core/build.gradle index 37b687d9020..74ac7ad934e 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -38,6 +38,13 @@ dependencies { implementation group: 'dev.failsafe', name: 'failsafe' api 'org.apache.httpcomponents.client5:httpclient5' + // plugins + implementation 'org.apache.maven.resolver:maven-resolver-impl' + implementation 'org.apache.maven.resolver:maven-resolver-supplier' + implementation 'org.apache.maven.resolver:maven-resolver-connector-basic' + implementation 'org.apache.maven.resolver:maven-resolver-transport-file' + implementation 'org.apache.maven.resolver:maven-resolver-transport-http' + // scheduler implementation group: 'com.cronutils', name: 'cron-utils' diff --git a/core/src/main/java/io/kestra/core/contexts/MavenPluginRepositoryConfig.java b/core/src/main/java/io/kestra/core/contexts/MavenPluginRepositoryConfig.java new file mode 100644 index 00000000000..fa402ef96d2 --- /dev/null +++ b/core/src/main/java/io/kestra/core/contexts/MavenPluginRepositoryConfig.java @@ -0,0 +1,26 @@ +package io.kestra.core.contexts; + +import io.micronaut.context.annotation.EachProperty; +import io.micronaut.context.annotation.Parameter; +import io.micronaut.core.annotation.Nullable; +import lombok.Builder; + +@EachProperty("kestra.plugins.repositories") +@Builder +public record MavenPluginRepositoryConfig( + @Parameter + String id, + String url, + + @Nullable + BasicAuth basicAuth +) { + + + public record BasicAuth( + String username, + String password + ) { + + } +} diff --git a/core/src/main/java/io/kestra/core/docs/AbstractClassDocumentation.java b/core/src/main/java/io/kestra/core/docs/AbstractClassDocumentation.java index febdf4c8f41..b4763d41398 100644 --- a/core/src/main/java/io/kestra/core/docs/AbstractClassDocumentation.java +++ b/core/src/main/java/io/kestra/core/docs/AbstractClassDocumentation.java @@ -1,6 +1,7 @@ package io.kestra.core.docs; import com.google.common.base.CaseFormat; +import io.kestra.core.models.Plugin; import io.kestra.core.models.tasks.retrys.AbstractRetry; import io.kestra.core.models.tasks.runners.TaskRunner; import lombok.AllArgsConstructor; diff --git a/core/src/main/java/io/kestra/core/docs/ClassPluginDocumentation.java b/core/src/main/java/io/kestra/core/docs/ClassPluginDocumentation.java index d4bef2e3927..f0d37e7ebca 100644 --- a/core/src/main/java/io/kestra/core/docs/ClassPluginDocumentation.java +++ b/core/src/main/java/io/kestra/core/docs/ClassPluginDocumentation.java @@ -1,10 +1,9 @@ package io.kestra.core.docs; -import io.kestra.core.plugins.RegisteredPlugin; +import io.kestra.core.plugins.PluginClassAndMetadata; import lombok.*; import java.util.*; -import java.util.stream.Collectors; @Getter @EqualsAndHashCode @@ -21,16 +20,18 @@ public class ClassPluginDocumentation extends AbstractClassDocumentation { private Map outputsSchema; @SuppressWarnings("unchecked") - private ClassPluginDocumentation(JsonSchemaGenerator jsonSchemaGenerator, RegisteredPlugin plugin, Class cls, Class baseCls, String alias) { - super(jsonSchemaGenerator, cls, baseCls); + private ClassPluginDocumentation(JsonSchemaGenerator jsonSchemaGenerator, PluginClassAndMetadata plugin, boolean allProperties) { + super(jsonSchemaGenerator, plugin.type(), allProperties ? null : plugin.baseClass()); // plugins metadata - this.cls = alias == null ? cls.getName() : alias; + Class cls = plugin.type(); + + this.cls = plugin.alias() == null ? cls.getName() : plugin.alias(); this.group = plugin.group(); this.docLicense = plugin.license(); this.pluginTitle = plugin.title(); - this.icon = plugin.icon(cls); - if (alias != null) { + this.icon = plugin.icon(); + if (plugin.alias() != null) { replacement = cls.getName(); } @@ -38,10 +39,10 @@ private ClassPluginDocumentation(JsonSchemaGenerator jsonSchemaGenerator, Regist this.subGroup = cls.getPackageName().substring(this.group.length() + 1); } - this.shortName = alias == null ? cls.getSimpleName() : alias.substring(alias.lastIndexOf('.') + 1); + this.shortName = plugin.alias() == null ? cls.getSimpleName() : plugin.alias().substring(plugin.alias().lastIndexOf('.') + 1); // outputs - this.outputsSchema = jsonSchemaGenerator.outputs(baseCls, cls); + this.outputsSchema = jsonSchemaGenerator.outputs(allProperties ? null : plugin.baseClass(), cls); if (this.outputsSchema.containsKey("$defs")) { this.defs.putAll((Map) this.outputsSchema.get("$defs")); @@ -67,17 +68,13 @@ private ClassPluginDocumentation(JsonSchemaGenerator jsonSchemaGenerator, Regist .toList(); } - if (alias != null) { + if (plugin.alias() != null) { this.deprecated = true; } } - public static ClassPluginDocumentation of(JsonSchemaGenerator jsonSchemaGenerator, RegisteredPlugin plugin, Class cls, Class baseCls) { - return new ClassPluginDocumentation<>(jsonSchemaGenerator, plugin, cls, baseCls, null); - } - - public static ClassPluginDocumentation of(JsonSchemaGenerator jsonSchemaGenerator, RegisteredPlugin plugin, Class cls, Class baseCls, String alias) { - return new ClassPluginDocumentation<>(jsonSchemaGenerator, plugin, cls, baseCls, alias); + public static ClassPluginDocumentation of(JsonSchemaGenerator jsonSchemaGenerator, PluginClassAndMetadata plugin, boolean allProperties) { + return new ClassPluginDocumentation<>(jsonSchemaGenerator, plugin, allProperties); } @AllArgsConstructor diff --git a/core/src/main/java/io/kestra/core/docs/DocumentationGenerator.java b/core/src/main/java/io/kestra/core/docs/DocumentationGenerator.java index 7cb10b834b0..11f1e7cd6c1 100644 --- a/core/src/main/java/io/kestra/core/docs/DocumentationGenerator.java +++ b/core/src/main/java/io/kestra/core/docs/DocumentationGenerator.java @@ -7,6 +7,7 @@ import io.kestra.core.models.tasks.runners.TaskRunner; import io.kestra.core.models.tasks.Task; import io.kestra.core.models.triggers.AbstractTrigger; +import io.kestra.core.plugins.PluginClassAndMetadata; import io.kestra.core.plugins.RegisteredPlugin; import io.kestra.core.runners.pebble.Extension; import io.kestra.core.runners.pebble.JsonWriter; @@ -217,7 +218,15 @@ private static List guides(RegisteredPlugin plugin) throws Exception { private List generate(RegisteredPlugin registeredPlugin, List> cls, Class baseCls, String type) { return cls .stream() - .map(r -> ClassPluginDocumentation.of(jsonSchemaGenerator, registeredPlugin, r, baseCls)) + .map(pluginClass -> { + PluginClassAndMetadata metadata = PluginClassAndMetadata.create( + registeredPlugin, + pluginClass, + baseCls, + null + ); + return ClassPluginDocumentation.of(jsonSchemaGenerator, metadata, true); + }) .map(pluginDocumentation -> { try { return new Document( @@ -247,15 +256,15 @@ private static String docPath(RegisteredPlugin registeredPlugin, String type classPluginDocumentation.getCls() + ".md"; } - public static String render(ClassPluginDocumentation classPluginDocumentation) throws IOException { + public static String render(ClassPluginDocumentation classPluginDocumentation) throws IOException { return render("task", JacksonMapper.toMap(classPluginDocumentation)); } - public static String render(AbstractClassDocumentation classInputDocumentation) throws IOException { + public static String render(AbstractClassDocumentation classInputDocumentation) throws IOException { return render("task", JacksonMapper.toMap(classInputDocumentation)); } - public static String render(String templateName, Map vars) throws IOException { + public static String render(String templateName, Map vars) throws IOException { String pebbleTemplate = IOUtils.toString( Objects.requireNonNull(DocumentationGenerator.class.getClassLoader().getResourceAsStream("docs/" + templateName + ".peb")), StandardCharsets.UTF_8 diff --git a/core/src/main/java/io/kestra/core/docs/JsonSchemaCache.java b/core/src/main/java/io/kestra/core/docs/JsonSchemaCache.java new file mode 100644 index 00000000000..2980675766e --- /dev/null +++ b/core/src/main/java/io/kestra/core/docs/JsonSchemaCache.java @@ -0,0 +1,65 @@ +package io.kestra.core.docs; + +import io.kestra.core.models.dashboards.Dashboard; +import io.kestra.core.models.flows.Flow; +import io.kestra.core.models.flows.PluginDefault; +import io.kestra.core.models.tasks.Task; +import io.kestra.core.models.templates.Template; +import io.kestra.core.models.triggers.AbstractTrigger; +import jakarta.inject.Singleton; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * Service for getting schemas. + */ +@Singleton +public class JsonSchemaCache { + + private final JsonSchemaGenerator jsonSchemaGenerator; + + private final ConcurrentMap> schemaCache = new ConcurrentHashMap<>(); + + private final Map> classesBySchemaType = new HashMap<>(); + + /** + * Creates a new {@link JsonSchemaCache} instance. + * + * @param jsonSchemaGenerator The {@link JsonSchemaGenerator}. + */ + public JsonSchemaCache(final JsonSchemaGenerator jsonSchemaGenerator) { + this.jsonSchemaGenerator = Objects.requireNonNull(jsonSchemaGenerator, "JsonSchemaGenerator cannot be null"); + registerClassForType(SchemaType.FLOW, Flow.class); + registerClassForType(SchemaType.TEMPLATE, Template.class); + registerClassForType(SchemaType.TASK, Task.class); + registerClassForType(SchemaType.TRIGGER, AbstractTrigger.class); + registerClassForType(SchemaType.PLUGINDEFAULT, PluginDefault.class); + registerClassForType(SchemaType.DASHBOARD, Dashboard.class); + } + + public Map getSchemaForType(final SchemaType type, + final boolean arrayOf) { + return schemaCache.computeIfAbsent(new CacheKey(type, arrayOf), (key) -> { + + Class cls = Optional.ofNullable(classesBySchemaType.get(type)) + .orElseThrow(() -> new IllegalArgumentException("Cannot found schema for type '" + type + "'")); + return jsonSchemaGenerator.schemas(cls, arrayOf); + }); + } + + public void registerClassForType(final SchemaType type, final Class clazz) { + classesBySchemaType.put(type, clazz); + } + + public void clear() { + schemaCache.clear(); + } + + private record CacheKey(SchemaType type, boolean arrayOf) { + } +} diff --git a/core/src/main/java/io/kestra/core/docs/Plugin.java b/core/src/main/java/io/kestra/core/docs/Plugin.java index bd55555b7ef..1c70c4fa74e 100644 --- a/core/src/main/java/io/kestra/core/docs/Plugin.java +++ b/core/src/main/java/io/kestra/core/docs/Plugin.java @@ -51,7 +51,7 @@ public static Plugin of(RegisteredPlugin registeredPlugin, @Nullable String subg plugin.title = registeredPlugin.title(); } else { subGroupInfos = registeredPlugin.allClass().stream().filter(c -> c.getName().contains(subgroup)).map(clazz -> clazz.getPackage().getDeclaredAnnotation(PluginSubGroup.class)).toList().getFirst(); - plugin.title = !subGroupInfos.title().isEmpty() ? subGroupInfos.title() : subgroup.substring(subgroup.lastIndexOf('.') + 1);; + plugin.title = !subGroupInfos.title().isEmpty() ? subGroupInfos.title() : subgroup.substring(subgroup.lastIndexOf('.') + 1); } plugin.group = registeredPlugin.group(); diff --git a/core/src/main/java/io/kestra/core/docs/SchemaType.java b/core/src/main/java/io/kestra/core/docs/SchemaType.java index 5ab597a7a1f..d22c773e1a1 100644 --- a/core/src/main/java/io/kestra/core/docs/SchemaType.java +++ b/core/src/main/java/io/kestra/core/docs/SchemaType.java @@ -1,11 +1,20 @@ package io.kestra.core.docs; +import com.fasterxml.jackson.annotation.JsonCreator; +import io.kestra.core.utils.Enums; + + public enum SchemaType { - flow, - template, - task, - trigger, - plugindefault, - apps, - dashboard + FLOW, + TEMPLATE, + TASK, + TRIGGER, + PLUGINDEFAULT, + APPS, + DASHBOARD; + + @JsonCreator + public static SchemaType fromString(final String value) { + return Enums.getForNameIgnoreCase(value, SchemaType.class); + } } diff --git a/core/src/main/java/io/kestra/core/models/hierarchies/SubflowGraphTask.java b/core/src/main/java/io/kestra/core/models/hierarchies/SubflowGraphTask.java index 69b970eb2f4..8aa1c9524d0 100644 --- a/core/src/main/java/io/kestra/core/models/hierarchies/SubflowGraphTask.java +++ b/core/src/main/java/io/kestra/core/models/hierarchies/SubflowGraphTask.java @@ -90,5 +90,10 @@ public String getId() { public String getType() { return ((TaskInterface) subflowTask).getType(); } + + @Override + public String getVersion() { + return ((TaskInterface) subflowTask).getVersion(); + } } } diff --git a/core/src/main/java/io/kestra/core/models/tasks/Task.java b/core/src/main/java/io/kestra/core/models/tasks/Task.java index 3ee2f993b99..88364bc1a19 100644 --- a/core/src/main/java/io/kestra/core/models/tasks/Task.java +++ b/core/src/main/java/io/kestra/core/models/tasks/Task.java @@ -31,6 +31,8 @@ abstract public class Task implements TaskInterface { protected String type; + protected String version; + private String description; @Valid diff --git a/core/src/main/java/io/kestra/core/models/tasks/TaskForExecution.java b/core/src/main/java/io/kestra/core/models/tasks/TaskForExecution.java index 20cbe23d351..29b1771f5ac 100644 --- a/core/src/main/java/io/kestra/core/models/tasks/TaskForExecution.java +++ b/core/src/main/java/io/kestra/core/models/tasks/TaskForExecution.java @@ -16,6 +16,8 @@ public class TaskForExecution implements TaskInterface { protected String type; + protected String version; + protected List tasks; protected List> inputs; diff --git a/core/src/main/java/io/kestra/core/models/tasks/TaskInterface.java b/core/src/main/java/io/kestra/core/models/tasks/TaskInterface.java index 9b31f0e0499..0bbdcc83211 100644 --- a/core/src/main/java/io/kestra/core/models/tasks/TaskInterface.java +++ b/core/src/main/java/io/kestra/core/models/tasks/TaskInterface.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import io.kestra.core.models.Plugin; +import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.Pattern; @@ -16,5 +17,10 @@ public interface TaskInterface extends Plugin { @NotNull @NotBlank @Pattern(regexp="\\p{javaJavaIdentifierStart}\\p{javaJavaIdentifierPart}*(\\.\\p{javaJavaIdentifierStart}\\p{javaJavaIdentifierPart}*)*") + @Schema(title = "The class name of this task.") String getType(); + + @Pattern(regexp="\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9]+)?|([a-zA-Z0-9]+)") + @Schema(title = "The class version of this task.") + String getVersion(); } diff --git a/core/src/main/java/io/kestra/core/models/triggers/AbstractTrigger.java b/core/src/main/java/io/kestra/core/models/triggers/AbstractTrigger.java index 43f8647759d..ae5134d8385 100644 --- a/core/src/main/java/io/kestra/core/models/triggers/AbstractTrigger.java +++ b/core/src/main/java/io/kestra/core/models/triggers/AbstractTrigger.java @@ -34,6 +34,8 @@ abstract public class AbstractTrigger implements TriggerInterface { protected String type; + protected String version; + private String description; @Valid diff --git a/core/src/main/java/io/kestra/core/models/triggers/AbstractTriggerForExecution.java b/core/src/main/java/io/kestra/core/models/triggers/AbstractTriggerForExecution.java index 9b0bb949317..7f3a8ac13d1 100644 --- a/core/src/main/java/io/kestra/core/models/triggers/AbstractTriggerForExecution.java +++ b/core/src/main/java/io/kestra/core/models/triggers/AbstractTriggerForExecution.java @@ -1,8 +1,5 @@ package io.kestra.core.models.triggers; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.micronaut.core.annotation.Introspected; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.experimental.SuperBuilder; @@ -16,6 +13,8 @@ public class AbstractTriggerForExecution implements TriggerInterface { protected String type; + protected String version; + public static AbstractTriggerForExecution of(AbstractTrigger abstractTrigger) { return AbstractTriggerForExecution.builder() .id(abstractTrigger.getId()) diff --git a/core/src/main/java/io/kestra/core/models/triggers/TriggerInterface.java b/core/src/main/java/io/kestra/core/models/triggers/TriggerInterface.java index 60e331a83f2..57514e96900 100644 --- a/core/src/main/java/io/kestra/core/models/triggers/TriggerInterface.java +++ b/core/src/main/java/io/kestra/core/models/triggers/TriggerInterface.java @@ -20,4 +20,8 @@ public interface TriggerInterface extends Plugin { @Schema(title = "The class name for this current trigger.") String getType(); + @Pattern(regexp="\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9]+)|([a-zA-Z0-9]+)") + @Schema(title = "The class version for this trigger.") + String getVersion(); + } diff --git a/core/src/main/java/io/kestra/core/plugins/DefaultPluginRegistry.java b/core/src/main/java/io/kestra/core/plugins/DefaultPluginRegistry.java index 63e0c1dff47..7bf74afb011 100644 --- a/core/src/main/java/io/kestra/core/plugins/DefaultPluginRegistry.java +++ b/core/src/main/java/io/kestra/core/plugins/DefaultPluginRegistry.java @@ -3,14 +3,31 @@ import io.kestra.core.models.Plugin; import jakarta.annotation.Nullable; import jakarta.validation.constraints.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.net.MalformedURLException; +import java.io.Closeable; +import java.io.IOException; import java.net.URL; import java.nio.file.Path; -import java.util.*; +import java.util.AbstractMap.SimpleEntry; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.ListIterator; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static java.util.Objects.requireNonNull; /** * Registry for managing all Kestra's {@link Plugin}. @@ -20,16 +37,20 @@ */ public class DefaultPluginRegistry implements PluginRegistry { + private static final Logger log = LoggerFactory.getLogger(DefaultPluginRegistry.class); + private static class LazyHolder { static final DefaultPluginRegistry INSTANCE = new DefaultPluginRegistry(); } - private final Map> pluginClassByIdentifier = new ConcurrentHashMap<>(); + private final Map> pluginClassByIdentifier = new ConcurrentHashMap<>(); private final Map plugins = new ConcurrentHashMap<>(); private final PluginScanner scanner = new PluginScanner(DefaultPluginRegistry.class.getClassLoader()); private final AtomicBoolean initialized = new AtomicBoolean(false); private final Set scannedPluginPaths = new HashSet<>(); + private final ReentrantLock lock = new ReentrantLock(); + /** * Gets or instantiates a {@link DefaultPluginRegistry} and register it as singleton object. * @@ -59,6 +80,24 @@ protected void init() { } } + /** + * {@inheritDoc} + */ + @Override + public List getAllVersionsForType(final String type) { + return plugins.values() + .stream().filter( + registered -> registered.allClass() + .stream() + .map(Class::getName) + .anyMatch(cls -> cls.equalsIgnoreCase(type)) + ).findFirst() + .map(RegisteredPlugin::version) + .filter(Objects::nonNull) + .map(List::of) + .orElse(List.of()); + } + /** * {@inheritDoc} */ @@ -86,6 +125,59 @@ public void register(final Path pluginPath) { } } + /** + * {@inheritDoc} + */ + @Override + public void unregister(final List pluginsToUnregister) { + if (pluginsToUnregister == null || pluginsToUnregister.isEmpty()) { + return; + } + + lock.lock(); + try { + ListIterator iter = pluginsToUnregister.listIterator(); + while (iter.hasNext()) { + final RegisteredPlugin current = iter.next(); + final PluginBundleIdentifier identifier = PluginBundleIdentifier.of(current); + + if (identifier.equals(PluginBundleIdentifier.CORE)) { + continue; // Skip the core plugin + } + + // Remove the plugin from the registry + this.plugins.remove(identifier); + + // Remove all classes to this plugin from the registry + this.pluginClassByIdentifier.entrySet().removeIf(entry -> { + PluginClassAndMetadata metadata = entry.getValue(); + return metadata.type().getClassLoader().equals(current.getClassLoader()); + }); + + // Close ClassLoader resources if applicable + if (current.getClassLoader() instanceof Closeable closeable) { + try { + closeable.close(); + } catch (IOException e) { + log.warn("Unexpected error while closing ClassLoader for plugins under {}", identifier.location(), e); + } + } + // Remove the plugin from the input list + iter.remove(); + } + } finally { + lock.unlock(); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void registerClassForIdentifier(PluginIdentifier identifier, PluginClassAndMetadata plugin) { + this.pluginClassByIdentifier.put(identifier, plugin); + } + private static boolean isPluginPathValid(final Path pluginPath) { return pluginPath != null && pluginPath.toFile().exists(); } @@ -96,20 +188,50 @@ private static boolean isPluginPathValid(final Path pluginPath) { * @param plugin the plugin to be registered. */ public void register(final RegisteredPlugin plugin) { - if (containsPluginBundle(PluginBundleIdentifier.of(plugin))) { - unregister(plugin); + final PluginBundleIdentifier identifier = PluginBundleIdentifier.of(plugin); + + // Skip registration if plugin-bundle already exists in the registry. + if (containsPluginBundle(identifier)) { + return; + } + + lock.lock(); + try { + plugins.put(PluginBundleIdentifier.of(plugin), plugin); + pluginClassByIdentifier.putAll(getPluginClassesByIdentifier(plugin)); + } finally { + lock.unlock(); } - plugins.put(PluginBundleIdentifier.of(plugin), plugin); - plugin.allClass().forEach(clazz -> { - @SuppressWarnings("unchecked") - Class pluginClass = (Class) clazz; - pluginClassByIdentifier.put(ClassTypeIdentifier.create(clazz), pluginClass); - }); - plugin.getAliases().values().forEach(e -> { - @SuppressWarnings("unchecked") - Class pluginClass = (Class) e.getValue(); - pluginClassByIdentifier.put(ClassTypeIdentifier.create(e.getKey()), pluginClass); - }); + } + + @SuppressWarnings("unchecked") + protected Map> getPluginClassesByIdentifier(final RegisteredPlugin plugin) { + Map> classes = new HashMap<>(); + classes.putAll(plugin.allClass() + .stream() + .map(cls -> { + + Class pluginClass = (Class) cls; + Class pluginBaseClass = plugin.baseClass(pluginClass.getName()); + + return new SimpleEntry<>( + ClassTypeIdentifier.create(cls.getName()), + PluginClassAndMetadata.create(plugin, pluginClass, pluginBaseClass, null) + ); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + + classes.putAll(plugin.getAliases().values().stream().map(e -> { + Class pluginClass = (Class) e.getValue(); + Class pluginBaseClass = plugin.baseClass(pluginClass.getName()); + + return new SimpleEntry<>( + ClassTypeIdentifier.create(e.getKey()), + PluginClassAndMetadata.create(plugin, pluginClass, pluginBaseClass, e.getKey()) + ); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + return classes; } private boolean containsPluginBundle(PluginBundleIdentifier identifier) { @@ -117,33 +239,24 @@ private boolean containsPluginBundle(PluginBundleIdentifier identifier) { } /** - * Unregisters a given plugin. - * - * @param plugin the plugin to be registered. - */ - public void unregister(final RegisteredPlugin plugin) { - if (plugins.remove(PluginBundleIdentifier.of(plugin)) != null) { - plugin.allClass().forEach(clazz -> { - pluginClassByIdentifier.remove(ClassTypeIdentifier.create(clazz)); - }); - } - } - - - /** {@inheritDoc} **/ + * {@inheritDoc} + **/ @Override public List plugins() { return plugins(null); } - /** {@inheritDoc} **/ + /** + * {@inheritDoc} + **/ @Override public List externalPlugins() { return plugins(plugin -> plugin.getExternalPlugin() != null); } - - /** {@inheritDoc} **/ + /** + * {@inheritDoc} + **/ @Override public List plugins(final Predicate predicate) { if (predicate == null) { @@ -161,8 +274,13 @@ public List plugins(final Predicate predicat **/ @Override public Class findClassByIdentifier(final PluginIdentifier identifier) { - Objects.requireNonNull(identifier, "Cannot found plugin for null identifier"); - return pluginClassByIdentifier.get(identifier); + requireNonNull(identifier, "Cannot found plugin for null identifier"); + lock.lock(); + try { + return findMetadataByIdentifier(identifier).map(PluginClassAndMetadata::type).orElse(null); + } finally { + lock.unlock(); + } } /** @@ -170,8 +288,35 @@ public Class findClassByIdentifier(final PluginIdentifier iden **/ @Override public Class findClassByIdentifier(final String identifier) { - Objects.requireNonNull(identifier, "Cannot found plugin for null identifier"); - return findClassByIdentifier(ClassTypeIdentifier.create(identifier)); + requireNonNull(identifier, "Cannot found plugin for null identifier"); + lock.lock(); + try { + return findClassByIdentifier(ClassTypeIdentifier.create(identifier)); + } finally { + lock.unlock(); + } + } + + /** + * {@inheritDoc} + **/ + @Override + public Optional> findMetadataByIdentifier(final String identifier) { + return findMetadataByIdentifier(ClassTypeIdentifier.create(identifier)); + } + + /** + * {@inheritDoc} + **/ + @Override + public Optional> findMetadataByIdentifier(final PluginIdentifier identifier) { + requireNonNull(identifier, "Cannot found plugin for null identifier"); + lock.lock(); + try { + return Optional.ofNullable(pluginClassByIdentifier.get(identifier)); + } finally { + lock.unlock(); + } } /** @@ -182,18 +327,10 @@ public void clear() { pluginClassByIdentifier.clear(); } - private record PluginBundleIdentifier(@Nullable URL location) { + public record PluginBundleIdentifier(@Nullable URL location) { public static PluginBundleIdentifier CORE = new PluginBundleIdentifier(null); - public static Optional of(final Path path) { - try { - return Optional.of(new PluginBundleIdentifier(path.toUri().toURL())); - } catch (MalformedURLException e) { - return Optional.empty(); - } - } - public static PluginBundleIdentifier of(final RegisteredPlugin plugin) { return Optional.ofNullable(plugin.getExternalPlugin()) .map(ExternalPlugin::getLocation) @@ -209,10 +346,6 @@ public static PluginBundleIdentifier of(final RegisteredPlugin plugin) { */ public record ClassTypeIdentifier(@NotNull String type) implements PluginIdentifier { - public static ClassTypeIdentifier create(final Class identifier) { - return create(identifier.getName()); - } - public static ClassTypeIdentifier create(final String identifier) { if (identifier == null || identifier.isBlank()) { throw new IllegalArgumentException("Cannot create plugin identifier from null or empty string"); diff --git a/core/src/main/java/io/kestra/core/plugins/LocalPluginManager.java b/core/src/main/java/io/kestra/core/plugins/LocalPluginManager.java new file mode 100644 index 00000000000..188d56556bf --- /dev/null +++ b/core/src/main/java/io/kestra/core/plugins/LocalPluginManager.java @@ -0,0 +1,240 @@ +package io.kestra.core.plugins; + +import io.kestra.core.contexts.MavenPluginRepositoryConfig; +import io.kestra.core.exceptions.KestraRuntimeException; +import io.micronaut.context.annotation.Value; +import jakarta.annotation.Nullable; +import jakarta.inject.Inject; +import jakarta.inject.Provider; +import jakarta.inject.Singleton; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Stream; + +import static io.kestra.core.plugins.PluginManager.createLocalRepositoryIfNotExist; + +/** + * A {@link PluginManager} implementation managing plugin artifacts on local storage. + */ +@Singleton +public class LocalPluginManager implements PluginManager { + + private static final Logger log = LoggerFactory.getLogger(LocalPluginManager.class); + + private final Provider pluginRegistryProvider; + + private final MavenPluginDownloader mavenPluginDownloader; + + private final Path localRepositoryPath; + + /** + * Creates a new {@link LocalPluginManager} instance. + * + * @param mavenPluginDownloader The {@link MavenPluginDownloader}. + */ + public LocalPluginManager(final MavenPluginDownloader mavenPluginDownloader) { + this(null, mavenPluginDownloader, null); + } + + /** + * Creates a new {@link LocalPluginManager} instance. + * + * @param pluginRegistryProvider The {@link PluginRegistry}. + * @param mavenPluginDownloader The {@link MavenPluginDownloader}. + * @param localRepositoryPath The local repository path used to stored plugins. + */ + @Inject + public LocalPluginManager(final Provider pluginRegistryProvider, + final MavenPluginDownloader mavenPluginDownloader, + @Nullable @Value("${kestra.plugins.management.localRepositoryPath}") final String localRepositoryPath) { + this.pluginRegistryProvider = pluginRegistryProvider; + this.mavenPluginDownloader = mavenPluginDownloader; + this.localRepositoryPath = PluginManager.getLocalManagedRepositoryPathOrDefault(localRepositoryPath); + } + + /** + * {@inheritDoc} + **/ + @Override + public void start() { + // no-op + } + + /** + * {@inheritDoc} + **/ + @Override + public boolean isReady() { + return true; + } + + /** + * {@inheritDoc} + **/ + @Override + public List list() { + try (Stream files = Files.list(localRepositoryPath)) { + return files + .filter(file -> Files.isRegularFile(file) && Files.isReadable(file) && file.toString().endsWith(".jar")) + .map(file -> { + try { + BasicFileAttributes attrs = Files.readAttributes(file, BasicFileAttributes.class); + return new PluginArtifactMetadata( + file.toUri(), + file.getFileName().toString(), + attrs.size(), + attrs.creationTime().toMillis(), + attrs.lastModifiedTime().toMillis() + ); + } catch (IOException e) { + log.warn("Failed to get file attribute from file {}", file.getFileName()); + return null; + } + }) + .filter(Objects::nonNull) + .toList(); + } catch (IOException e) { + throw new KestraRuntimeException(e); + } + } + + /** + * {@inheritDoc} + **/ + @Override + public PluginArtifact install(PluginArtifact artifact, + List repositoryConfigs, + boolean installForRegistration, + @Nullable Path localRepositoryPath) { + Objects.requireNonNull(artifact, "cannot install null artifact"); + + final PluginArtifact resolvedPluginArtifact = mavenPluginDownloader.resolve(artifact.toString(), repositoryConfigs); + + return install(resolvedPluginArtifact, installForRegistration, localRepositoryPath); + } + + private PluginArtifact install(final PluginArtifact artifact, + final boolean installForRegistration, + Path localRepositoryPath) { + + log.info("Installing managed plugin artifact '{}'", artifact); + localRepositoryPath = createLocalRepositoryIfNotExist(Optional.ofNullable(localRepositoryPath).orElse(this.localRepositoryPath)); + Path localPluginPath = getLocalPluginPath(localRepositoryPath, artifact); + + try { + Files.createDirectories(localPluginPath.getParent()); + Files.copy(Path.of(artifact.uri()), localPluginPath, StandardCopyOption.REPLACE_EXISTING); + + if (installForRegistration && pluginRegistryProvider != null) { + pluginRegistryProvider.get().register(localRepositoryPath); + } + log.info("Plugin '{}' installed successfully in local repository: {}", artifact, localRepositoryPath); + return artifact.relocateTo(localPluginPath.toUri()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * {@inheritDoc} + **/ + @Override + public PluginArtifact install(File file, boolean installForRegistration, @Nullable Path localRepositoryPath) { + try { + return install(PluginArtifact.fromFile(file), installForRegistration, localRepositoryPath); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * {@inheritDoc} + **/ + @Override + public List install(List artifacts, + List repositoryConfigs, + boolean refreshPluginRegistry, + @Nullable Path localRepositoryPath) { + return artifacts.stream() + .map(artifact -> install(artifact, repositoryConfigs, refreshPluginRegistry, localRepositoryPath)) + .toList(); + } + + private Path getLocalPluginPath(final Path localRepositoryPath, final PluginArtifact artifact) { + return localRepositoryPath.resolve(artifact.toFileName()); + } + + /** + * {@inheritDoc} + **/ + @Override + public List uninstall(List artifacts, boolean refreshPluginRegistry, @Nullable Path localRepositoryPath) { + + final Path repositoryPath = Optional.ofNullable(localRepositoryPath).orElse(this.localRepositoryPath); + + final List uninstalled = artifacts.stream() + .map(artifact -> doUninstall(artifact, repositoryPath) ? artifact : null) + .filter(Objects::nonNull) + .toList(); + + if (refreshPluginRegistry && pluginRegistryProvider != null) { + pluginRegistryProvider.get().register(localRepositoryPath); + } + return uninstalled; + } + + /** + * {@inheritDoc} + **/ + @Override + public List resolveVersions(List artifacts) { + return mavenPluginDownloader.resolveVersions(artifacts); + } + + private boolean doUninstall(final PluginArtifact artifact, final Path localRepositoryPath) { + + final Path localPluginPath = getLocalPluginPath(localRepositoryPath, artifact); + + if (Files.exists(localPluginPath)) { + log.info("Removing plugin artifact from local repository: {}", localPluginPath); + if (pluginRegistryProvider != null) { + final PluginRegistry registry = pluginRegistryProvider.get(); + // Unregister all plugins from registry + registry.unregister(new ArrayList<>(registry.plugins((plugin) -> { + if (plugin.getClassLoader() instanceof PluginClassLoader pluginClassLoader) { + URI location = URI.create(pluginClassLoader.location()); + return localPluginPath.equals(Path.of(location)); + } + return false; + }))); + } + + try { + if (Files.deleteIfExists(localPluginPath)) { + log.info("Removed plugin artifact from local repository: {}", localPluginPath); + } + return true; + } catch (IOException e) { + log.error( + "Unexpected error while removing plugin artifact from plugin repository: {}", + localPluginPath, + e + ); + return false; + } + } + return false; + } +} diff --git a/core/src/main/java/io/kestra/core/plugins/MavenPluginDownloader.java b/core/src/main/java/io/kestra/core/plugins/MavenPluginDownloader.java new file mode 100644 index 00000000000..0353c7a5516 --- /dev/null +++ b/core/src/main/java/io/kestra/core/plugins/MavenPluginDownloader.java @@ -0,0 +1,226 @@ +package io.kestra.core.plugins; + +import io.kestra.core.contexts.MavenPluginRepositoryConfig; +import io.kestra.core.utils.Version; +import io.micronaut.context.annotation.Value; +import io.micronaut.core.annotation.Nullable; +import jakarta.annotation.PreDestroy; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +import org.apache.commons.io.FileUtils; +import org.apache.maven.repository.internal.MavenRepositorySystemUtils; +import org.eclipse.aether.DefaultRepositorySystemSession; +import org.eclipse.aether.MultiRuntimeException; +import org.eclipse.aether.RepositorySystem; +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.artifact.DefaultArtifact; +import org.eclipse.aether.repository.LocalRepository; +import org.eclipse.aether.repository.RemoteRepository; +import org.eclipse.aether.resolution.ArtifactRequest; +import org.eclipse.aether.resolution.ArtifactResolutionException; +import org.eclipse.aether.resolution.ArtifactResult; +import org.eclipse.aether.resolution.VersionRangeRequest; +import org.eclipse.aether.resolution.VersionRangeResolutionException; +import org.eclipse.aether.resolution.VersionRangeResult; +import org.eclipse.aether.supplier.RepositorySystemSupplier; +import org.eclipse.aether.util.repository.AuthenticationBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +/** + * Service for resolving plugins from a Maven repository. + */ +@Singleton +public class MavenPluginDownloader implements Closeable { + + private static final Logger log = LoggerFactory.getLogger(MavenPluginDownloader.class); + private static final String DEFAULT_LOCAL_REPOSITORY_PREFIX = "kestra-plugins-m2-repository"; + private static final String DEFAULT_REPOSITORY_TYPE = "default"; + public static final String LATEST = "latest"; + + private final List repositoryConfigs; + private final RepositorySystem system; + private final RepositorySystemSession session; + + @Inject + public MavenPluginDownloader(List repositoryConfigs, + @Nullable @Value("${kestra.plugins.local-repository-path}") String localRepositoryPath) { + this.repositoryConfigs = repositoryConfigs; + this.system = new RepositorySystemSupplier().get(); + this.session = repositorySystemSession(system, localRepositoryPath); + } + + /** + * Resolves the given dependencies. + * + * @param dependency The dependency to resolve. + * @return the local {@link Path} of the resolved dependency. + */ + public PluginArtifact resolve(String dependency) { + return doResolve(buildRemoteRepositories(repositoryConfigs), dependency); + } + + /** + * Resolves the version of the given dependencies. + * + * @param dependency The dependency to resolve. + * @return the local {@link Path} of the resolved dependency. + */ + public List listAllVersions(final String dependency) { + try { + DefaultArtifact artifact = new DefaultArtifact(dependency); + + VersionRangeRequest request = new VersionRangeRequest(); + request.setArtifact(artifact.setVersion("[0,)")); // use a wide version range + request.setRepositories(buildRemoteRepositories(this.repositoryConfigs)); + + VersionRangeResult result = system.resolveVersionRange(session, request); + return result.getVersions().stream().map(Object::toString).toList(); + } catch (VersionRangeResolutionException e) { + log.debug("Failed to resolve all versions for '{}'", dependency); + return List.of(); + } + } + + + /** + * Resolves the given dependencies given the additional repositories. + * + * @param dependency The dependency to resolve. + * @param repositories The Maven repositories. + * @return the local {@link Path} of the resolved dependency. + */ + public PluginArtifact resolve(String dependency, List repositories) { + List allRepositories = new ArrayList<>(); + allRepositories.addAll(buildRemoteRepositories(this.repositoryConfigs)); + allRepositories.addAll(buildRemoteRepositories(repositories)); + + return doResolve(allRepositories, dependency); + } + + private PluginArtifact doResolve(List repositories, String dependency) { + PluginArtifact result = resolveArtifact(repositories, dependency); + log.debug("Resolved Plugin '{}' with '{}'", dependency, result.uri()); + return result; + } + + public List resolveVersions(final List artifacts) { + return artifacts.stream() + .map(artifact -> { + List versions = listAllVersions(artifact.toCoordinates()); + + final List parsedVersions = versions.stream().map(Version::of).sorted().toList(); + + if (versions.isEmpty()) { + return new PluginResolutionResult(artifact, null, List.of(), false); + } + + final List sortedVersions = parsedVersions.stream().map(Version::toString).toList(); + if (artifact.version().equalsIgnoreCase(LATEST)) { + return new PluginResolutionResult(artifact, Version.getLatest(parsedVersions).toString(), sortedVersions, true); + } + + return versions.contains(artifact.version()) ? + new PluginResolutionResult(artifact, artifact.version(), versions, true) : + new PluginResolutionResult(artifact, null, sortedVersions, false); + }) + .toList(); + } + + private static List buildRemoteRepositories(List repositoryConfigs) { + return repositoryConfigs + .stream() + .map(repositoryConfig -> { + var build = new RemoteRepository.Builder( + repositoryConfig.id(), + DEFAULT_REPOSITORY_TYPE, + repositoryConfig.url() + ); + + if (repositoryConfig.basicAuth() != null) { + var authenticationBuilder = new AuthenticationBuilder(); + authenticationBuilder.addUsername(repositoryConfig.basicAuth().username()); + authenticationBuilder.addPassword(repositoryConfig.basicAuth().password()); + build.setAuthentication(authenticationBuilder.build()); + } + + return build.build(); + }) + .toList(); + } + + private RepositorySystemSession repositorySystemSession(RepositorySystem system, String localRepositoryPath) { + DefaultRepositorySystemSession session = MavenRepositorySystemUtils.newSession(); + + if (localRepositoryPath == null) { + try { + final String tmpDir = Files.createTempDirectory(DEFAULT_LOCAL_REPOSITORY_PREFIX).toAbsolutePath().toString(); + + localRepositoryPath = tmpDir; + + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + FileUtils.deleteDirectory(new File(tmpDir)); + } catch (IOException e) { + throw new RuntimeException(e); + } + })); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + LocalRepository localRepo = new LocalRepository(localRepositoryPath); + session.setLocalRepositoryManager(system.newLocalRepositoryManager(session, localRepo)); + + return session; + } + + private PluginArtifact resolveArtifact(List repositories, String dependency) { + try { + DefaultArtifact artifact = new DefaultArtifact(dependency); + VersionRangeResult version = system.resolveVersionRange(session, new VersionRangeRequest(artifact, repositories, null)); + + final String highestVersion = version.getHighestVersion().toString(); + ArtifactRequest artifactRequest = new ArtifactRequest( + new DefaultArtifact(artifact.getGroupId(), artifact.getArtifactId(), "jar", highestVersion), + repositories, + null + ); + ArtifactResult result = system.resolveArtifact(session, artifactRequest); + return new PluginArtifact( + result.getArtifact().getGroupId(), + result.getArtifact().getArtifactId(), + result.getArtifact().getExtension(), + result.getArtifact().getClassifier(), + // Use the version from ArtifactRequest and not the one from the ArtifactResult. + // Otherwise, SNAPSHOT version will result in a timestamped version string. + highestVersion.endsWith("-SNAPSHOT") ? highestVersion : result.getArtifact().getVersion(), + result.getArtifact().getFile().toPath().toUri() + ); + } catch (VersionRangeResolutionException | ArtifactResolutionException e) { + throw new RuntimeException("Failed to resolve dependency: '" + dependency + "'", e); + } + } + + /** + * {@inheritDoc} + */ + @PreDestroy + @Override + public void close() throws IOException { + try { + system.shutdown(); + } catch (MultiRuntimeException e) { + log.warn("Error while shutting down Maven repository", e); + } + } +} diff --git a/core/src/main/java/io/kestra/core/plugins/PluginArtifact.java b/core/src/main/java/io/kestra/core/plugins/PluginArtifact.java new file mode 100644 index 00000000000..6ce56958a0c --- /dev/null +++ b/core/src/main/java/io/kestra/core/plugins/PluginArtifact.java @@ -0,0 +1,166 @@ +package io.kestra.core.plugins; + +import lombok.Builder; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.Objects; +import java.util.Optional; +import java.util.jar.JarFile; +import java.util.jar.Manifest; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.util.function.Predicate.not; + +/** + * A specific plugin artifact. + * + * @param groupId the group identifier of this plugin artifact + * @param artifactId the artifact identifier of this plugin artifact + * @param version the version of this plugin artifact + * @param uri the location of this plugin artifact. + */ +@Builder(toBuilder = true) +public record PluginArtifact( + String groupId, + String artifactId, + String extension, + String classifier, + String version, + URI uri +) implements Comparable { + + private static final Pattern ARTIFACT_PATTERN = Pattern.compile( + "([^: ]+):([^: ]+)(:([^: ]*)(:([^: ]+))?)?:([^: ]+)" + ); + private static final Pattern FILENAME_PATTERN = Pattern.compile( + "^(?[\\w_]+)__(?[\\w-_]+)(?:__(?[\\w-_]+))?__(?\\d+_\\d+_\\d+(-[a-zA-Z0-9]+)?|([a-zA-Z0-9]+))\\.jar$" + ); + + public static final String JAR_EXTENSION = "jar"; + + /** + * Static helper method for constructing a new {@link PluginArtifact} from a JAR file. + * + * @param file The JAR file. + * @return a new {@link PluginArtifact}. + * @throws IOException if the file cannot be read. + */ + public static PluginArtifact fromFile(final File file) throws IOException { + try (JarFile jarFile = new JarFile(file)) { + Manifest manifest = jarFile.getManifest(); + final String version = manifest.getMainAttributes().getValue("X-Kestra-Version"); + final String artifactId = manifest.getMainAttributes().getValue("X-Kestra-Name"); + final String groupId = manifest.getMainAttributes().getValue("X-Kestra-Group"); + + return new PluginArtifact( + groupId, + artifactId, + "jar", + null, + version, + file.toURI() + ); + } + } + + /** + * Static helper method for constructing a new {@link PluginArtifact} from an artifact string coordinates. + * + * @param coordinates The artifact's coordinates + * @return a new {@link PluginArtifact}. + */ + public static PluginArtifact fromCoordinates(final String coordinates) { + Matcher m = ARTIFACT_PATTERN.matcher(coordinates); + if (!m.matches()) { + throw new IllegalArgumentException("Bad artifact coordinates " + coordinates + + ", expected format is :[:[:]]:"); + } + return new PluginArtifact( + m.group(1), + m.group(2), + Optional.ofNullable(m.group(4)).filter(not(String::isEmpty)).orElse(JAR_EXTENSION), + Optional.ofNullable(m.group(6)).filter(not(String::isEmpty)).orElse(null), + "LATEST".equalsIgnoreCase(m.group(7)) ? "LATEST": m.group(7), + null + ); + } + + /** + * Static helper method for constructing a new {@link PluginArtifact} from an artifact a file name. + * + * @param fileName The artifact's file name + * @return a new {@link PluginArtifact}. + */ + public static PluginArtifact fromFileName(final String fileName) { + Matcher matcher = FILENAME_PATTERN.matcher(fileName); + if (!matcher.matches()) { + throw new IllegalArgumentException("Invalid artifact filename '" + fileName + "', expected format is __[__]__.jar"); + } + + String[] parts = fileName.substring(0, fileName.lastIndexOf(".")).split("__"); + + String groupId = parts[0].replace("_", "."); + String artifactId = parts[1]; + String version; + String classifier = null; // optional + if (parts.length == 4) { + classifier = parts[2]; + version = parts[3].replace("_", "."); + } else { + version = parts[2].replace("_", "."); + } + return new PluginArtifact(groupId, artifactId, "jar", classifier, version, null); + } + + public PluginArtifact relocateTo(URI uri) { + return new PluginArtifact( + groupId, + artifactId, + extension, + classifier, + version, + uri + ); + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() { + return toCoordinates(); + } + + public String toCoordinates() { + return Stream.of(groupId, artifactId, extension, classifier, version) + .filter(Objects::nonNull) + .filter(it -> !it.isEmpty()) + .collect(Collectors.joining(":")); + } + + public String toFileName() { + String name = Stream.of( + groupId.replace(".", "_"), + artifactId, + classifier, + version.replace(".", "_") + ) + .filter(Objects::nonNull) + .filter(it -> !it.isEmpty()) + .collect(Collectors.joining("__")); + return name + "." + extension; + } + + /** + * {@inheritDoc} + */ + @Override + public int compareTo(PluginArtifact that) { + return this.toCoordinates().compareTo(that.toCoordinates()); + } +} diff --git a/core/src/main/java/io/kestra/core/plugins/PluginArtifactMetadata.java b/core/src/main/java/io/kestra/core/plugins/PluginArtifactMetadata.java new file mode 100644 index 00000000000..553609ae71e --- /dev/null +++ b/core/src/main/java/io/kestra/core/plugins/PluginArtifactMetadata.java @@ -0,0 +1,29 @@ +package io.kestra.core.plugins; + +import java.net.URI; + +/** + * Metadata for a specific plugin artifact file. + * + * @param uri The URI of the plugin artifact. + * @param name The name of the plugin artifact. + * @param size The size of the plugin artifact. + * @param lastModifiedTime The last modified time of the plugin artifact. + * @param creationTime The creation time of the plugin artifact. + */ +public record PluginArtifactMetadata( + URI uri, + String name, + long size, + long lastModifiedTime, + long creationTime) { + + /** + * Gets a new {@link PluginArtifact} from this. + * + * @return a new {@link PluginArtifact}. + */ + public PluginArtifact toPluginArtifact() { + return PluginArtifact.fromFileName(this.name).relocateTo(uri); + } +} diff --git a/core/src/main/java/io/kestra/core/plugins/PluginClassAndMetadata.java b/core/src/main/java/io/kestra/core/plugins/PluginClassAndMetadata.java new file mode 100644 index 00000000000..60e261f35e0 --- /dev/null +++ b/core/src/main/java/io/kestra/core/plugins/PluginClassAndMetadata.java @@ -0,0 +1,26 @@ +package io.kestra.core.plugins; + +public record PluginClassAndMetadata( + Class type, + Class baseClass, + String group, + String license, + String title, + String icon, + String alias +) { + public static PluginClassAndMetadata create(RegisteredPlugin registered, + Class pluginClass, + Class pluginBaseClass, + String alias) { + return new PluginClassAndMetadata<>( + pluginClass, + pluginBaseClass, + registered.group(), + registered.license(), + registered.title(), + registered.icon(pluginClass), + alias + ); + } +} diff --git a/core/src/main/java/io/kestra/core/plugins/PluginManager.java b/core/src/main/java/io/kestra/core/plugins/PluginManager.java new file mode 100644 index 00000000000..e207dc77cd5 --- /dev/null +++ b/core/src/main/java/io/kestra/core/plugins/PluginManager.java @@ -0,0 +1,132 @@ +package io.kestra.core.plugins; + +import io.kestra.core.contexts.MavenPluginRepositoryConfig; +import jakarta.annotation.Nullable; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; + +/** + * Service interface for managing Kestra's plugins. + */ +public interface PluginManager extends AutoCloseable { + + /** + * Starts this manager. + */ + void start(); + + /** + * Checks whether this manager is ready. + *

+ * This method should return {@code true} when this manager is fully started. + * @see #start() + * + * @return {@code true} if the manager is ready. + */ + boolean isReady(); + + /** + * Gets the list of plugin artifact managed this by class. + * + * @return The list of {@link PluginArtifact}. + */ + List list(); + + /** + * Installs the given plugin artifact. + * + * @param artifact the plugin artifact. + * @param repositoryConfigs the addition repository configs. + * @param installForRegistration specify whether plugin artifacts should be scanned and registered. + * @param localRepositoryPath the optional local repository path to install artifact. + * @return The URI of the installed plugin. + */ + PluginArtifact install(PluginArtifact artifact, + List repositoryConfigs, + boolean installForRegistration, + @Nullable Path localRepositoryPath); + + + /** + * Installs the given plugin artifact. + * + * @param file the plugin JAR file. + * @param installForRegistration specify whether plugin artifacts should be scanned and registered. + * @param localRepositoryPath the optional local repository path to install artifact. + * @return The URI of the installed plugin. + */ + PluginArtifact install(final File file, + boolean installForRegistration, + @Nullable Path localRepositoryPath); + + /** + * Installs the given plugin artifact. + * + * @param artifacts the list of plugin artifacts. + * @param repositoryConfigs the addition repository configs. + * @param installForRegistration specify whether the plugin registry should be refreshed. + * @param localRepositoryPath the optional local repository path to install artifact. + * @return The URIs of the installed plugins. + */ + List install(List artifacts, + List repositoryConfigs, + boolean installForRegistration, + @Nullable Path localRepositoryPath); + + /** + * Uninstall the given plugin artifact. + * + * @param artifacts the plugin artifacts to be uninstalled. + * @param refreshPluginRegistry specify whether the plugin registry should be refreshed. + * @param localRepositoryPath the optional local repository path to install artifact. + */ + List uninstall(List artifacts, + boolean refreshPluginRegistry, + @Nullable Path localRepositoryPath); + + /** + * Resolves the version for the given artifacts. + * + * @param artifacts The list of artifacts to resolve. + * @return The list of results. + */ + List resolveVersions(List artifacts); + + @Override + default void close() throws Exception { + + } + + /** + * Static helper method to resolve the given local repository path. + * + * @param path the local repository path. + * @return the repository path or the default one. + */ + static Path getLocalManagedRepositoryPathOrDefault(final @Nullable String path) { + Path resolved = Optional.ofNullable(path) + .map(Path::of) + .orElseGet(() -> Path + .of(System.getProperty("java.io.tmpdir")) + .resolve("kestra/plugins-repository") + ); + return createLocalRepositoryIfNotExist(resolved); + } + + static Path createLocalRepositoryIfNotExist(final Path resolved) { + if (!Files.exists(resolved)) { + try { + Files.createDirectories(resolved); + } catch (IOException e) { + throw new RuntimeException("Cannot create local repository for plugins", e); + } + } + return resolved; + } + +} diff --git a/core/src/main/java/io/kestra/core/plugins/PluginRegistry.java b/core/src/main/java/io/kestra/core/plugins/PluginRegistry.java index 3c3956780ef..1fda078d86c 100644 --- a/core/src/main/java/io/kestra/core/plugins/PluginRegistry.java +++ b/core/src/main/java/io/kestra/core/plugins/PluginRegistry.java @@ -4,6 +4,7 @@ import java.nio.file.Path; import java.util.List; +import java.util.Optional; import java.util.function.Predicate; /** @@ -11,6 +12,14 @@ */ public interface PluginRegistry { + /** + * Gets all versions for a given plugin type. + * + * @param type The plugin type. + * @return The list of supported versions,or an empty list if the type is unknown. + */ + List getAllVersionsForType(final String type); + /** * Scans and registers the given plugin path, if the path is not already registered. * This method should be a no-op if the given path is {@code null} or does not exist. @@ -27,6 +36,24 @@ public interface PluginRegistry { */ void register(final Path pluginPath); + /** + * Unregisters the given plugin bundle. + * + * @param plugin the plugin bundle to un-register. + */ + void unregister(List plugin); + + /** + * Registers a plugin class with the given identifier. + *

+ * Any plugin class registered through this method will be then accessible from + * the method {@link #findClassByIdentifier(PluginIdentifier)}. + * + * @param identifier The plugin identifier. + * @param plugin The class for the register. + */ + void registerClassForIdentifier(PluginIdentifier identifier, PluginClassAndMetadata plugin); + /** * Finds the Java class corresponding to the given plugin identifier. * @@ -43,6 +70,22 @@ public interface PluginRegistry { */ Class findClassByIdentifier(String identifier); + /** + * Finds the Java class and metadata corresponding to the given identifier. + * + * @param identifier The raw plugin identifier - must not be {@code null}. + * @return the {@link PluginClassAndMetadata} of the plugin or {@link Optional#empty()} if no plugin can be found. + */ + Optional> findMetadataByIdentifier(String identifier); + + /** + * Finds the Java class and metadata corresponding to the given identifier. + * + * @param identifier The raw plugin identifier - must not be {@code null}. + * @return the {@link PluginClassAndMetadata} of the plugin or {@link Optional#empty()} if no plugin can be found. + */ + Optional> findMetadataByIdentifier(PluginIdentifier identifier); + /** * Gets the list of all registered plugins. * diff --git a/core/src/main/java/io/kestra/core/plugins/PluginResolutionResult.java b/core/src/main/java/io/kestra/core/plugins/PluginResolutionResult.java new file mode 100644 index 00000000000..e108b4cfb7f --- /dev/null +++ b/core/src/main/java/io/kestra/core/plugins/PluginResolutionResult.java @@ -0,0 +1,20 @@ +package io.kestra.core.plugins; + +import java.util.List; + +/** + * Represents the result of a version resolution for an artifact. + * + * @param artifact The artifact that was resolved. + * @param version The resolved version. + * @param versions The list of all available versions. + * @param resolved {@code true} if version was resolved. Otherwise {@code false}. + */ +public record PluginResolutionResult( + PluginArtifact artifact, + String version, + List versions, + boolean resolved +) { + +} \ No newline at end of file diff --git a/core/src/main/java/io/kestra/core/plugins/RegisteredPlugin.java b/core/src/main/java/io/kestra/core/plugins/RegisteredPlugin.java index e233dd24458..c82ab68f1e3 100644 --- a/core/src/main/java/io/kestra/core/plugins/RegisteredPlugin.java +++ b/core/src/main/java/io/kestra/core/plugins/RegisteredPlugin.java @@ -266,10 +266,13 @@ public String icon(Class cls) { IOUtils.toString(resourceAsStream, StandardCharsets.UTF_8).getBytes(StandardCharsets.UTF_8) ); } - return null; } + public String icon() { + return icon("plugin-icon"); + } + @SneakyThrows public String icon(String iconName) { InputStream resourceAsStream = this.getClassLoader().getResourceAsStream("icons/" + iconName + ".svg"); @@ -278,7 +281,6 @@ public String icon(String iconName) { IOUtils.toString(resourceAsStream, StandardCharsets.UTF_8).getBytes(StandardCharsets.UTF_8) ); } - return null; } diff --git a/core/src/main/java/io/kestra/core/plugins/serdes/PluginDeserializer.java b/core/src/main/java/io/kestra/core/plugins/serdes/PluginDeserializer.java index be9cad6205d..3262d0ae3e3 100644 --- a/core/src/main/java/io/kestra/core/plugins/serdes/PluginDeserializer.java +++ b/core/src/main/java/io/kestra/core/plugins/serdes/PluginDeserializer.java @@ -33,6 +33,7 @@ public final class PluginDeserializer extends JsonDeserializer private static final Logger log = LoggerFactory.getLogger(PluginDeserializer.class); private static final String TYPE = "type"; + private static final String VERSION = "version"; private volatile PluginRegistry pluginRegistry; @@ -102,8 +103,8 @@ private T fromObjectNode(JsonParser jp, ); if (DataChart.class.isAssignableFrom(pluginType)) { - Class dataFilterClass = pluginRegistry.findClassByIdentifier(extractPluginRawIdentifier(node.get("data"))); - ParameterizedType genericDataFilterClass = (ParameterizedType) pluginRegistry.findClassByIdentifier(extractPluginRawIdentifier(node.get("data"))).getGenericSuperclass(); + final Class dataFilterClass = pluginRegistry.findClassByIdentifier(extractPluginRawIdentifier(node.get("data"))); + ParameterizedType genericDataFilterClass = (ParameterizedType) dataFilterClass.getGenericSuperclass(); Type dataFieldsEnum = genericDataFilterClass.getActualTypeArguments()[0]; TypeFactory typeFactory = JacksonMapper.ofJson().getTypeFactory(); Type chartAwareColumnDescriptorClass = ((ParameterizedType) ((WildcardType) ((ParameterizedType) ((TypeVariable) @@ -142,10 +143,13 @@ private static void throwInvalidTypeException(final DeserializationContext conte } static String extractPluginRawIdentifier(final JsonNode node) { - JsonNode type = node.get(TYPE); - if (type == null || type.textValue().isEmpty()) { + String type = Optional.ofNullable(node.get(TYPE)).map(JsonNode::textValue).orElse(null); + String version = Optional.ofNullable(node.get(VERSION)).map(JsonNode::textValue).orElse(null); + + if (type == null || type.isEmpty()) { return null; } - return type.textValue(); + + return version != null && !version.isEmpty() ? type + ":" + version : type; } } diff --git a/core/src/main/java/io/kestra/core/server/ClusterEvent.java b/core/src/main/java/io/kestra/core/server/ClusterEvent.java index f732e7b80db..af621d87f86 100644 --- a/core/src/main/java/io/kestra/core/server/ClusterEvent.java +++ b/core/src/main/java/io/kestra/core/server/ClusterEvent.java @@ -11,5 +11,5 @@ public ClusterEvent(EventType eventType, LocalDateTime eventDate, String message this(IdUtils.create(), eventType, eventDate, message); } - public enum EventType { MAINTENANCE_ENTER, MAINTENANCE_EXIT } + public enum EventType { MAINTENANCE_ENTER, MAINTENANCE_EXIT, PLUGINS_SYNC_REQUESTED } } diff --git a/core/src/main/java/io/kestra/core/utils/Version.java b/core/src/main/java/io/kestra/core/utils/Version.java new file mode 100644 index 00000000000..a90a67d831b --- /dev/null +++ b/core/src/main/java/io/kestra/core/utils/Version.java @@ -0,0 +1,359 @@ +package io.kestra.core.utils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; + +/** + * A version class which supports the following pattern : + *

+ * ..- + *

+ * Supported qualifier are : alpha, beta, snapshot, rc, release. + */ +public class Version implements Comparable { + + public static final Version ZERO = new Version(0, 0, 0, null); + + public static boolean isEqual(final String v1, final String v2) { + return isEqual(Version.of(v1), v2); + } + + public static boolean isEqual(final Version v1, final String v2) { + return v1.equals(Version.of(v2)); + } + + /** + * Static helper for creating a new version based on the specified string. + * + * @param version the version. + * @return a new {@link Version} instance. + */ + public static Version of(String version) { + + if (version.startsWith("v")) { + version = version.substring(1); + } + + int qualifier = version.indexOf("-"); + + final String[] versions = qualifier > 0 ? + version.substring(0, qualifier).split("\\.") : + version.split("\\."); + try { + final int majorVersion = Integer.parseInt(versions[0]); + final int minorVersion = versions.length > 1 ? Integer.parseInt(versions[1]) : 0; + final int incrementalVersion = versions.length > 2 ? Integer.parseInt(versions[2]) : 0; + + return new Version( + majorVersion, + minorVersion, + incrementalVersion, + qualifier > 0 ? version.substring(qualifier + 1) : null, + version + ); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid version, cannot parse '" + version + "'"); + } + } + + /** + * Static helper method for returning the most recent stable version for a current {@link Version}. + * + * @param from the current version. + * @param versions the list of version. + * + * @return the last stable version. + */ + public static Version getStable(final Version from, final Collection versions) { + List compatibleVersions = versions.stream() + .filter(v -> v.majorVersion() == from.majorVersion() && v.minorVersion() == from.minorVersion()) + .toList(); + if (compatibleVersions.isEmpty()) return null; + return Version.getLatest(compatibleVersions); + } + + /** + * Static helper method for returning the latest version from a list of {@link Version}. + * + * @param versions the list of version. + * @return the latest version. + */ + public static Version getLatest(final Version...versions) { + return getLatest(Stream.of(versions).toList()); + } + + /** + * Static helper method for returning the latest version from a list of {@link Version}. + * + * @param versions the list of version. + * @return the latest version. + */ + public static Version getLatest(final Collection versions) { + return versions.stream() + .filter(Objects::nonNull) + .min(Comparator.naturalOrder()) + .orElseThrow(() -> new IllegalArgumentException("empty list")); + } + + /** + * Static helper for returning the latest version from a list of {@link Version}. + * + * @param versions the list of version. + * @return the latest version. + */ + public static Version getOldest(final Version...versions) { + return getOldest(Stream.of(versions).toList()); + } + + /** + * Static helper for returning the latest version from a list of {@link Version}. + * + * @param versions the list of version. + * @return the latest version. + */ + public static Version getOldest(final Collection versions) { + return versions.stream() + .filter(Objects::nonNull) + .max(Comparator.naturalOrder()) + .orElseThrow(() -> new IllegalArgumentException("empty list")); + } + + private final int majorVersion; + private final int minorVersion; + private final int incrementalVersion; + private final Qualifier qualifier; + + private final String originalVersion; + + /** + * Creates a new {@link Version} instance. + * + * @param majorVersion the major version (must be superior or equal to 0). + * @param minorVersion the minor version (must be superior or equal to 0). + * @param incrementalVersion the incremental version (must be superior or equal to 0). + * @param qualifier the qualifier. + */ + public Version(final int majorVersion, + final int minorVersion, + final int incrementalVersion, + final String qualifier) { + this(majorVersion, minorVersion, incrementalVersion, qualifier, null); + } + + /** + * Creates a new {@link Version} instance. + * + * @param majorVersion the major version (must be superior or equal to 0). + * @param minorVersion the minor version (must be superior or equal to 0). + * @param incrementalVersion the incremental version (must be superior or equal to 0). + * @param qualifier the qualifier. + * @param originalVersion the original string version. + */ + private Version(final int majorVersion, + final int minorVersion, + final int incrementalVersion, + final String qualifier, + final String originalVersion) { + this.majorVersion = requirePositive(majorVersion, "major"); + this.minorVersion = requirePositive(minorVersion, "minor"); + this.incrementalVersion = requirePositive(incrementalVersion, "incremental"); + this.qualifier = qualifier != null ? new Qualifier(qualifier) : null; + this.originalVersion = originalVersion; + } + + + private static int requirePositive(int version, final String message) { + if (version < 0) { + throw new IllegalArgumentException(String.format("The '%s' version must super or equal to 0", message)); + } + return version; + } + + public int majorVersion() { + return majorVersion; + } + + public int minorVersion() { + return minorVersion; + } + + public int incrementalVersion() { + return incrementalVersion; + } + + public Qualifier qualifier() { + return qualifier; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Version)) return false; + Version version = (Version) o; + return majorVersion == version.majorVersion && + minorVersion == version.minorVersion && + incrementalVersion == version.incrementalVersion && + Objects.equals(qualifier, version.qualifier); + } + + /** + * {@inheritDoc} + */ + @Override + public int hashCode() { + return Objects.hash(majorVersion, minorVersion, incrementalVersion, qualifier); + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() { + if (originalVersion != null) return originalVersion; + + String version = majorVersion + "." + minorVersion + "." + incrementalVersion; + return (qualifier != null) ? version +"-" + qualifier : version; + } + + /** + * {@inheritDoc} + */ + @Override + public int compareTo(final Version that) { + + int compareMajor = Integer.compare(that.majorVersion, this.majorVersion); + if (compareMajor != 0) { + return compareMajor; + } + + int compareMinor = Integer.compare(that.minorVersion, this.minorVersion); + if (compareMinor != 0) { + return compareMinor; + } + + int compareIncremental = Integer.compare(that.incrementalVersion, this.incrementalVersion); + if (compareIncremental != 0) { + return compareIncremental; + } + + if (that.qualifier == null && this.qualifier == null) { + return 0; + } else if (that.qualifier == null) { + return 1; + } else if (this.qualifier == null) { + return -1; + } + + return this.qualifier.compareTo(that.qualifier); + } + + /** + * Checks whether this version is before the given one. + * + * @param version The version to compare. + * @return {@code true} if this version is before.Otherwise {@code false}. + */ + public boolean isBefore(final Version version) { + return this.compareTo(version) > 0; + } + + public static final class Qualifier implements Comparable { + + private static final List DEFAULT_QUALIFIER_NAME; + + static { + // order is important + DEFAULT_QUALIFIER_NAME = new ArrayList<>(); + DEFAULT_QUALIFIER_NAME.add("ALPHA"); + DEFAULT_QUALIFIER_NAME.add("BETA"); + DEFAULT_QUALIFIER_NAME.add("SNAPSHOT"); + DEFAULT_QUALIFIER_NAME.add("RC"); + DEFAULT_QUALIFIER_NAME.add("RELEASE"); + } + + private final String qualifier; + private final String label; + private final int priority; + private final int number; + + /** + * Creates a new {@link Qualifier} instance. + * @param qualifier the qualifier string. + */ + Qualifier(final String qualifier) { + Objects.requireNonNull(qualifier, "qualifier cannot be null"); + this.qualifier = qualifier; + this.label = getUniformQualifier(qualifier); + this.priority = DEFAULT_QUALIFIER_NAME.indexOf(label); + this.number = (label.length() < qualifier.length()) ? getQualifierNumber(qualifier) : 0; + } + + + /** + * {@inheritDoc} + */ + @Override + public boolean equals(Object that) { + if (this == that) return true; + if (!(that instanceof Qualifier)) return false; + return qualifier.equalsIgnoreCase(((Qualifier) that).qualifier); + } + + + /** + * {@inheritDoc} + */ + @Override + public int hashCode() { + return Objects.hash(qualifier); + } + + /** + * {@inheritDoc} + */ + @Override + public int compareTo(final Qualifier that) { + int compare = Integer.compare(that.priority, this.priority); + return (compare != 0) ? compare : Integer.compare(that.number, this.number); + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() { + return qualifier; + } + } + + private static int getQualifierNumber(final String qualifier) { + StringBuilder label = new StringBuilder(); + char[] chars = qualifier.toCharArray(); + for (char c : chars) { + if (Character.isDigit(c)) { + label.append(c); + } + } + return label.isEmpty() ? 0 : Integer.parseInt(label.toString()); + } + + private static String getUniformQualifier(final String qualifier) { + StringBuilder label = new StringBuilder(); + char[] chars = qualifier.toCharArray(); + for (char c : chars) { + if (Character.isLetter(c)) { + label.append(c); + } else { + break; + } + } + return label.toString().toUpperCase(); + } +} \ No newline at end of file diff --git a/core/src/test/java/io/kestra/core/docs/ClassPluginDocumentationTest.java b/core/src/test/java/io/kestra/core/docs/ClassPluginDocumentationTest.java index 6a560bd9bea..9c60846da0c 100644 --- a/core/src/test/java/io/kestra/core/docs/ClassPluginDocumentationTest.java +++ b/core/src/test/java/io/kestra/core/docs/ClassPluginDocumentationTest.java @@ -3,6 +3,8 @@ import io.kestra.core.Helpers; import io.kestra.core.models.property.DynamicPropertyExampleTask; import io.kestra.core.models.tasks.runners.TaskRunner; +import io.kestra.core.models.triggers.TriggerInterface; +import io.kestra.core.plugins.PluginClassAndMetadata; import io.kestra.plugin.core.runner.Process; import io.kestra.core.models.tasks.Task; import io.kestra.core.models.triggers.AbstractTrigger; @@ -37,7 +39,8 @@ void tasks() throws URISyntaxException { assertThat(scan.size(), is(1)); assertThat(scan.getFirst().getTasks().size(), is(1)); - ClassPluginDocumentation doc = ClassPluginDocumentation.of(jsonSchemaGenerator, scan.getFirst(), scan.getFirst().getTasks().getFirst(), Task.class); + PluginClassAndMetadata metadata = PluginClassAndMetadata.create(scan.getFirst(), scan.getFirst().getTasks().getFirst(), Task.class, null); + ClassPluginDocumentation doc = ClassPluginDocumentation.of(jsonSchemaGenerator, metadata, false); assertThat(doc.getDocExamples().size(), is(2)); assertThat(doc.getIcon(), is(notNullValue())); @@ -99,7 +102,8 @@ void trigger() throws URISyntaxException { PluginScanner pluginScanner = new PluginScanner(ClassPluginDocumentationTest.class.getClassLoader()); RegisteredPlugin scan = pluginScanner.scan(); - ClassPluginDocumentation doc = ClassPluginDocumentation.of(jsonSchemaGenerator, scan, Schedule.class, null); + PluginClassAndMetadata metadata = PluginClassAndMetadata.create(scan, Schedule.class, AbstractTrigger.class, null); + ClassPluginDocumentation doc = ClassPluginDocumentation.of(jsonSchemaGenerator, metadata, true); assertThat(doc.getDefs().size(), is(1)); assertThat(doc.getDocLicense(), nullValue()); @@ -117,7 +121,8 @@ void taskRunner() throws URISyntaxException { PluginScanner pluginScanner = new PluginScanner(ClassPluginDocumentationTest.class.getClassLoader()); RegisteredPlugin scan = pluginScanner.scan(); - ClassPluginDocumentation> doc = ClassPluginDocumentation.of(jsonSchemaGenerator, scan, Process.class, null); + PluginClassAndMetadata> metadata = PluginClassAndMetadata.create(scan, Process.class, Process.class, null); + ClassPluginDocumentation> doc = ClassPluginDocumentation.of(jsonSchemaGenerator, metadata, false); assertThat((Map) doc.getPropertiesSchema().get("properties"), anEmptyMap()); assertThat(doc.getCls(), is("io.kestra.plugin.core.runner.Process")); @@ -135,12 +140,13 @@ void dynamicProperty() throws URISyntaxException { PluginScanner pluginScanner = new PluginScanner(ClassPluginDocumentationTest.class.getClassLoader()); RegisteredPlugin scan = pluginScanner.scan(); - ClassPluginDocumentation doc = ClassPluginDocumentation.of(jsonSchemaGenerator, scan, DynamicPropertyExampleTask.class, null); + PluginClassAndMetadata metadata = PluginClassAndMetadata.create(scan, DynamicPropertyExampleTask.class, DynamicPropertyExampleTask.class, null); + ClassPluginDocumentation doc = ClassPluginDocumentation.of(jsonSchemaGenerator, metadata, true); assertThat(doc.getCls(), is("io.kestra.core.models.property.DynamicPropertyExampleTask")); assertThat(doc.getDefs(), aMapWithSize(6)); Map properties = (Map) doc.getPropertiesSchema().get("properties"); - assertThat(properties, aMapWithSize(19)); + assertThat(properties, aMapWithSize(20)); Map number = (Map) properties.get("number"); assertThat(number.get("oneOf"), notNullValue()); diff --git a/core/src/test/java/io/kestra/core/docs/DocumentationGeneratorTest.java b/core/src/test/java/io/kestra/core/docs/DocumentationGeneratorTest.java index 42adde428cc..28b9a749d73 100644 --- a/core/src/test/java/io/kestra/core/docs/DocumentationGeneratorTest.java +++ b/core/src/test/java/io/kestra/core/docs/DocumentationGeneratorTest.java @@ -1,6 +1,6 @@ package io.kestra.core.docs; -import io.kestra.core.models.tasks.runners.TaskRunner; +import io.kestra.core.plugins.PluginClassAndMetadata; import io.kestra.plugin.core.runner.Process; import io.kestra.core.models.tasks.Task; import io.kestra.core.plugins.PluginScanner; @@ -41,7 +41,8 @@ void tasks() throws URISyntaxException, IOException { List scan = pluginScanner.scan(plugins); assertThat(scan.size(), is(1)); - ClassPluginDocumentation doc = ClassPluginDocumentation.of(jsonSchemaGenerator, scan.getFirst(), scan.getFirst().getTasks().getFirst(), Task.class); + PluginClassAndMetadata metadata = PluginClassAndMetadata.create(scan.getFirst(), scan.getFirst().getTasks().getFirst(), Task.class, null); + ClassPluginDocumentation doc = ClassPluginDocumentation.of(jsonSchemaGenerator, metadata, false); String render = DocumentationGenerator.render(doc); @@ -59,7 +60,8 @@ void dag() throws IOException { RegisteredPlugin scan = pluginScanner.scan(); Class dag = scan.findClass(Dag.class.getName()).orElseThrow(); - ClassPluginDocumentation doc = ClassPluginDocumentation.of(jsonSchemaGenerator, scan, dag, Task.class); + PluginClassAndMetadata metadata = PluginClassAndMetadata.create(scan,dag, Task.class, null); + ClassPluginDocumentation doc = ClassPluginDocumentation.of(jsonSchemaGenerator, metadata, false); String render = DocumentationGenerator.render(doc); @@ -95,7 +97,8 @@ void returnDoc() throws IOException { RegisteredPlugin scan = pluginScanner.scan(); Class returnTask = scan.findClass(Return.class.getName()).orElseThrow(); - ClassPluginDocumentation doc = ClassPluginDocumentation.of(jsonSchemaGenerator, scan, returnTask, Task.class); + PluginClassAndMetadata metadata = PluginClassAndMetadata.create(scan, returnTask, Task.class, null); + ClassPluginDocumentation doc = ClassPluginDocumentation.of(jsonSchemaGenerator, metadata, false); String render = DocumentationGenerator.render(doc); @@ -113,7 +116,8 @@ void defaultBool() throws IOException { RegisteredPlugin scan = pluginScanner.scan(); Class bash = scan.findClass(Subflow.class.getName()).orElseThrow(); - ClassPluginDocumentation doc = ClassPluginDocumentation.of(jsonSchemaGenerator, scan, bash, Task.class); + PluginClassAndMetadata metadata = PluginClassAndMetadata.create(scan, bash, Task.class, null); + ClassPluginDocumentation doc = ClassPluginDocumentation.of(jsonSchemaGenerator, metadata, false); String render = DocumentationGenerator.render(doc); @@ -127,7 +131,8 @@ void echo() throws IOException { RegisteredPlugin scan = pluginScanner.scan(); Class bash = scan.findClass(Echo.class.getName()).orElseThrow(); - ClassPluginDocumentation doc = ClassPluginDocumentation.of(jsonSchemaGenerator, scan, bash, Task.class); + PluginClassAndMetadata metadata = PluginClassAndMetadata.create(scan, bash, Task.class, null); + ClassPluginDocumentation doc = ClassPluginDocumentation.of(jsonSchemaGenerator, metadata, false); String render = DocumentationGenerator.render(doc); @@ -142,7 +147,8 @@ void state() throws IOException { RegisteredPlugin scan = pluginScanner.scan(); Class set = scan.findClass(Set.class.getName()).orElseThrow(); - ClassPluginDocumentation doc = ClassPluginDocumentation.of(jsonSchemaGenerator, scan, set, Task.class); + PluginClassAndMetadata metadata = PluginClassAndMetadata.create(scan, set, Task.class, null); + ClassPluginDocumentation doc = ClassPluginDocumentation.of(jsonSchemaGenerator, metadata, false); String render = DocumentationGenerator.render(doc); @@ -180,7 +186,8 @@ void taskRunner() throws IOException { RegisteredPlugin scan = pluginScanner.scan(); Class processTaskRunner = scan.findClass(Process.class.getName()).orElseThrow(); - ClassPluginDocumentation doc = ClassPluginDocumentation.of(jsonSchemaGenerator, scan, processTaskRunner, Process.class); + PluginClassAndMetadata metadata = PluginClassAndMetadata.create(scan, processTaskRunner, Process.class, null); + ClassPluginDocumentation doc = ClassPluginDocumentation.of(jsonSchemaGenerator, metadata, false); String render = DocumentationGenerator.render(doc); diff --git a/core/src/test/java/io/kestra/core/plugins/PluginArtifactTest.java b/core/src/test/java/io/kestra/core/plugins/PluginArtifactTest.java new file mode 100644 index 00000000000..c27b7276251 --- /dev/null +++ b/core/src/test/java/io/kestra/core/plugins/PluginArtifactTest.java @@ -0,0 +1,76 @@ +package io.kestra.core.plugins; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; + + +class PluginArtifactTest { + + @Test + void shouldParseGivenValidFilenameWithoutClassifier(){ + String fileName = "io_kestra_plugin__plugin-serdes__0_20_0.jar"; + PluginArtifact artifact = PluginArtifact.fromFileName(fileName); + + assertEquals("io.kestra.plugin", artifact.groupId()); + assertEquals("plugin-serdes", artifact.artifactId()); + assertEquals("jar", artifact.extension()); + assertNull(artifact.classifier()); + assertEquals("0.20.0", artifact.version()); + assertNull(artifact.uri()); + } + + @Test + void shouldParseGivenValidFilenameWithClassifier() { + String fileName = "io_kestra_plugin__plugin-serdes__custom-classifier__0_20_0.jar"; + PluginArtifact artifact = PluginArtifact.fromFileName(fileName); + + assertEquals("io.kestra.plugin", artifact.groupId()); + assertEquals("plugin-serdes", artifact.artifactId()); + assertEquals("jar", artifact.extension()); + assertEquals("custom-classifier", artifact.classifier()); + assertEquals("0.20.0", artifact.version()); + assertNull(artifact.uri()); + } + + @Test + void shouldThrowIllegalArgumentExceptionGivenInvalidFilenameMissingVersion() { + String fileName = "io_kestra_plugin__plugin-serdes__custom-classifier.jar"; + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> PluginArtifact.fromFileName(fileName) + ); + + assertEquals( + "Invalid artifact filename 'io_kestra_plugin__plugin-serdes__custom-classifier.jar', expected format is __[__]__.jar", + exception.getMessage() + ); + } + + @Test + void shouldThrowIllegalArgumentExceptionGivenInvalidFilenameWrongFormat() { + String fileName = "invalid_filename_format.jar"; + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> PluginArtifact.fromFileName(fileName) + ); + + assertEquals( + "Invalid artifact filename 'invalid_filename_format.jar', expected format is __[__]__.jar", + exception.getMessage() + ); + } + + @Test + void shouldParseGivenValidFilenameEdgeCase() { + String fileName = "group__artifact__0_0_1.jar"; + PluginArtifact artifact = PluginArtifact.fromFileName(fileName); + + assertEquals("group", artifact.groupId()); + assertEquals("artifact", artifact.artifactId()); + assertEquals("jar", artifact.extension()); + assertNull(artifact.classifier()); + assertEquals("0.0.1", artifact.version()); + assertNull(artifact.uri()); + } +} \ No newline at end of file diff --git a/core/src/test/java/io/kestra/core/plugins/serdes/PluginDeserializerTest.java b/core/src/test/java/io/kestra/core/plugins/serdes/PluginDeserializerTest.java index ee7e584dcdc..1c7fa6db9e3 100644 --- a/core/src/test/java/io/kestra/core/plugins/serdes/PluginDeserializerTest.java +++ b/core/src/test/java/io/kestra/core/plugins/serdes/PluginDeserializerTest.java @@ -8,7 +8,6 @@ import io.kestra.core.models.Plugin; import io.kestra.core.plugins.PluginRegistry; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; diff --git a/core/src/test/java/io/kestra/core/serializers/YamlParserTest.java b/core/src/test/java/io/kestra/core/serializers/YamlParserTest.java index e11afde5eb6..11fcc216cb6 100644 --- a/core/src/test/java/io/kestra/core/serializers/YamlParserTest.java +++ b/core/src/test/java/io/kestra/core/serializers/YamlParserTest.java @@ -198,7 +198,7 @@ void invalidProperty() { () -> this.parse("flows/invalids/invalid-property.yaml") ); - assertThat(exception.getMessage(), is("Unrecognized field \"invalid\" (class io.kestra.plugin.core.debug.Return), not marked as ignorable (13 known properties: \"logLevel\", \"timeout\", \"retry\", \"allowWarning\", \"format\", \"type\", \"id\", \"description\", \"workerGroup\", \"runIf\", \"logToFile\", \"disabled\", \"allowFailure\"])")); + assertThat(exception.getMessage(), is("Unrecognized field \"invalid\" (class io.kestra.plugin.core.debug.Return), not marked as ignorable (14 known properties: \"logLevel\", \"timeout\", \"retry\", \"allowWarning\", \"format\", \"version\", \"type\", \"id\", \"description\", \"workerGroup\", \"runIf\", \"logToFile\", \"disabled\", \"allowFailure\"])")); assertThat(exception.getConstraintViolations().size(), is(1)); assertThat(exception.getConstraintViolations().iterator().next().getPropertyPath().toString(), is("io.kestra.core.models.flows.Flow[\"tasks\"]->java.util.ArrayList[0]->io.kestra.plugin.core.debug.Return[\"invalid\"]")); } diff --git a/core/src/test/java/io/kestra/core/utils/VersionTest.java b/core/src/test/java/io/kestra/core/utils/VersionTest.java new file mode 100644 index 00000000000..41feb852841 --- /dev/null +++ b/core/src/test/java/io/kestra/core/utils/VersionTest.java @@ -0,0 +1,148 @@ +package io.kestra.core.utils; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +class VersionTest { + + @Test + void shouldCreateVersionFromStringGivenMajorVersion() { + Version version = Version.of("1"); + Assertions.assertEquals(1, version.majorVersion()); + } + + @Test + void shouldCreateVersionFromStringGivenMajorMinorVersion() { + Version version = Version.of("1.2"); + Assertions.assertEquals(1, version.majorVersion()); + Assertions.assertEquals(2, version.minorVersion()); + } + + @Test + void shouldCreateVersionFromStringGivenMajorMinorIncrementVersion() { + Version version = Version.of("1.2.3"); + Assertions.assertEquals(1, version.majorVersion()); + Assertions.assertEquals(2, version.minorVersion()); + Assertions.assertEquals(3, version.incrementalVersion()); + } + + @Test + void shouldCreateVersionFromPrefixedStringGivenMajorMinorIncrementVersion() { + Version version = Version.of("v1.2.3"); + Assertions.assertEquals(1, version.majorVersion()); + Assertions.assertEquals(2, version.minorVersion()); + Assertions.assertEquals(3, version.incrementalVersion()); + } + + @Test + void shouldCreateVersionFromStringGivenMajorMinorIncrementAndQualifierVersion() { + Version version = Version.of("1.2.3-SNAPSHOT"); + Assertions.assertEquals(1, version.majorVersion()); + Assertions.assertEquals(2, version.minorVersion()); + Assertions.assertEquals(3, version.incrementalVersion()); + Assertions.assertEquals("SNAPSHOT", version.qualifier().toString()); + } + + @Test + void shouldCreateVersionFromStringGivenSnapshotSuffixedQualifierVersion() { + Version version = Version.of("1.2.3-RC0-SNAPSHOT"); + Assertions.assertEquals(1, version.majorVersion()); + Assertions.assertEquals(2, version.minorVersion()); + Assertions.assertEquals(3, version.incrementalVersion()); + Assertions.assertEquals("RC0-SNAPSHOT", version.qualifier().toString()); + } + + @Test + void shouldThrowIllegalArgumentGivenInvalidVersion() { + IllegalArgumentException e = Assertions.assertThrows( + IllegalArgumentException.class, + () -> Version.of("bad input")); + + Assertions.assertEquals("Invalid version, cannot parse 'bad input'", e.getMessage()); + } + + @Test + void shouldGetLatestVersionGivenMajorVersions() { + Version result = Version.getLatest(Version.of("1"), Version.of("3"), Version.of("2")); + Assertions.assertEquals(Version.of("3"), result); + } + + @Test + void shouldGetLatestVersionGivenMajorMinorVersions() { + Version result = Version.getLatest(Version.of("1.2"), Version.of("1.0"), Version.of("1.10")); + Assertions.assertEquals(Version.of("1.10"), result); + } + + @Test + void shouldGetLatestVersionGivenMajorMinorIncrementalVersions() { + Version result = Version.getLatest(Version.of("1.0.9"), Version.of("1.0.10"), Version.of("1.0.11")); + Assertions.assertEquals(Version.of("1.0.11"), result); + } + + @Test + public void shouldGetOldestVersionGivenMajorMinorIncrementalVersions() { + Version result = Version.getOldest(Version.of("1.0.9"), Version.of("1.0.10"), Version.of("1.0.11")); + Assertions.assertEquals(Version.of("1.0.9"), result); + } + + @Test + public void shouldGetLatestVersionGivenMajorMinorIncrementalAndSimpleQualifierVersions() { + Version result = Version.getLatest(Version.of("1.0.0"), Version.of("1.0.0-SNAPSHOT")); + Assertions.assertEquals(Version.of("1.0.0"), result); + + result = Version.getLatest(Version.of("1.0.0-ALPHA"), Version.of("1.0.0-BETA")); + Assertions.assertEquals(Version.of("1.0.0-BETA"), result); + + result = Version.getLatest(Version.of("1.0.0-RELEASE"), Version.of("1.0.0-SNAPSHOT")); + Assertions.assertEquals(Version.of("1.0.0-RELEASE"), result); + + result = Version.getLatest(Version.of("1.0.0-RC10"), Version.of("1.0.0-RC12")); + Assertions.assertEquals(Version.of("1.0.0-RC12"), result); + + result = Version.getLatest(Version.of("1.0.0-rc.10"), Version.of("1.0.0-rc.12")); + Assertions.assertEquals(Version.of("1.0.0-rc.12"), result); + } + + @Test + void shouldReturnTrueForEqualsGivenDifferentCase() { + Assertions.assertEquals(Version.of("1.0.0-rc.1"), Version.of("1.0.0-RC.1")); + } + + @Test + void shouldNotFailGivenUnknownQualifier() { + Assertions.assertDoesNotThrow(() -> Version.of("1.0.0-custom10")); + Version result = Version.getLatest(Version.of("1.0.0-custom10"), Version.of("1.0.0-SNAPSHOT")); + Assertions.assertEquals(Version.of("1.0.0-SNAPSHOT"), result); + } + + @Test + void shouldReturnTrueGivenBeforeVersion() { + Assertions.assertTrue(Version.of("1.0.0").isBefore(Version.of("1.0.1"))); + Assertions.assertTrue(Version.of("1.0.0").isBefore(Version.of("1.1.0"))); + Assertions.assertTrue(Version.of("1.0.0").isBefore(Version.of("2.0.0"))); + Assertions.assertTrue(Version.of("1.0.0-SNAPSHOT").isBefore(Version.of("1.0.0"))); + } + + @Test + void shouldReturnFalseGivenNonBeforeVersion() { + Assertions.assertFalse(Version.of("1.0.0").isBefore(Version.of("1.0.0"))); + Assertions.assertFalse(Version.of("1.0.1").isBefore(Version.of("1.0.0"))); + Assertions.assertFalse(Version.of("1.1.0").isBefore(Version.of("1.0.0"))); + Assertions.assertFalse(Version.of("2.0.0").isBefore(Version.of("2.0.0"))); + Assertions.assertFalse(Version.of("1.0.0").isBefore(Version.of("1.0.0-SNAPSHOT"))); + } + + @Test + public void shouldGetStableVersionGivenMajorMinorVersions() { + Version result = Version.getStable(Version.of("1.2.0"), List.of(Version.of("1.2.1"), Version.of("1.2.2"), Version.of("0.99.0"))); + Assertions.assertEquals(Version.of("1.2.2"), result); + } + + @Test + public void shouldGetNullForStableVersionGivenNoCompatibleVersions() { + Version result = Version.getStable(Version.of("1.2.0"), List.of(Version.of("1.3.0"), Version.of("2.0.0"), Version.of("0.99.0"))); + Assertions.assertNull(result); + } +} \ No newline at end of file diff --git a/model/src/main/java/io/kestra/core/models/Plugin.java b/model/src/main/java/io/kestra/core/models/Plugin.java index 093827b64af..b4f5846717a 100644 --- a/model/src/main/java/io/kestra/core/models/Plugin.java +++ b/model/src/main/java/io/kestra/core/models/Plugin.java @@ -1,6 +1,7 @@ package io.kestra.core.models; import io.kestra.core.models.annotations.Plugin.Id; +import jakarta.annotation.Nullable; import jakarta.validation.constraints.NotNull; import java.util.Arrays; diff --git a/platform/build.gradle b/platform/build.gradle index a27ec672acf..763d6eb1174 100644 --- a/platform/build.gradle +++ b/platform/build.gradle @@ -15,7 +15,7 @@ dependencies { def slf4jVersion = "2.0.16" def protobufVersion = "3.25.5" // Orc still uses 3.25.5 see https://github.com/apache/orc/blob/main/java/pom.xml def bouncycastleVersion = "1.80" - def aetherVersion = "1.1.0" + def mavenResolverVersion = "1.9.22" def jollydayVersion = "0.32.0" def jsonschemaVersion = "4.37.0" def kafkaVersion = "3.9.0" @@ -87,14 +87,11 @@ dependencies { api group: 'org.apache.commons', name: 'commons-lang3', version: '3.17.0' api 'ch.qos.logback.contrib:logback-json-classic:0.1.5' api 'ch.qos.logback.contrib:logback-jackson:0.1.5' - api "org.eclipse.aether:aether-api:$aetherVersion" - api "org.eclipse.aether:aether-spi:$aetherVersion" - api "org.eclipse.aether:aether-util:$aetherVersion" - api "org.eclipse.aether:aether-impl:$aetherVersion" - api "org.eclipse.aether:aether-connector-basic:$aetherVersion" - api "org.eclipse.aether:aether-transport-file:$aetherVersion" - api "org.eclipse.aether:aether-transport-http:$aetherVersion" - api 'org.apache.maven:maven-aether-provider:3.3.9' + api group: 'org.apache.maven.resolver', name: 'maven-resolver-impl', version: mavenResolverVersion + api group: 'org.apache.maven.resolver', name: 'maven-resolver-supplier', version: mavenResolverVersion + api group: 'org.apache.maven.resolver', name: 'maven-resolver-connector-basic', version: mavenResolverVersion + api group: 'org.apache.maven.resolver', name: 'maven-resolver-transport-file', version: mavenResolverVersion + api group: 'org.apache.maven.resolver', name: 'maven-resolver-transport-http', version: mavenResolverVersion api 'com.github.oshi:oshi-core:6.6.6' api 'io.pebbletemplates:pebble:3.2.3' api group: 'co.elastic.logging', name: 'logback-ecs-encoder', version: '1.6.0' diff --git a/ui/src/components/plugins/Plugin.vue b/ui/src/components/plugins/Plugin.vue index 73f9826564f..3276f2a36db 100644 --- a/ui/src/components/plugins/Plugin.vue +++ b/ui/src/components/plugins/Plugin.vue @@ -9,6 +9,26 @@