Skip to content

Commit

Permalink
Merge branch 'develop' into ns_4296_fixed
Browse files Browse the repository at this point in the history
  • Loading branch information
@MilosPaunovic
MilosPaunovic authored Feb 28, 2025
2 parents b9aceab + d8295ef commit d92e3e8
Show file tree
Hide file tree
Showing 68 changed files with 2,253 additions and 489 deletions.
12 changes: 0 additions & 12 deletions cli/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,6 @@ dependencies {
implementation 'ch.qos.logback.contrib:logback-json-classic'
implementation 'ch.qos.logback.contrib:logback-jackson'

// plugins
implementation 'org.eclipse.aether:aether-api'
implementation 'org.eclipse.aether:aether-spi'
implementation 'org.eclipse.aether:aether-util'
implementation 'org.eclipse.aether:aether-impl'
implementation 'org.eclipse.aether:aether-connector-basic'
implementation 'org.eclipse.aether:aether-transport-file'
implementation 'org.eclipse.aether:aether-transport-http'
implementation('org.apache.maven:maven-aether-provider') {
// sisu dependency injector is not used
exclude group: 'org.eclipse.sisu'
}
// aether still use javax.inject
compileOnly 'javax.inject:javax.inject:1'

Expand Down
44 changes: 26 additions & 18 deletions cli/src/main/java/io/kestra/cli/AbstractCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,17 @@
import com.google.common.collect.ImmutableMap;
import io.kestra.cli.commands.servers.ServerCommandInterface;
import io.kestra.cli.services.StartupHookInterface;
import io.kestra.core.contexts.KestraContext;
import io.kestra.core.plugins.PluginManager;
import io.kestra.core.plugins.PluginRegistry;
import io.kestra.webserver.services.FlowAutoLoaderService;
import io.micronaut.context.ApplicationContext;
import io.micronaut.context.env.yaml.YamlPropertySourceLoader;
import io.micronaut.core.annotation.Introspected;
import io.micronaut.http.uri.UriBuilder;
import io.micronaut.management.endpoint.EndpointDefaultConfiguration;
import io.micronaut.runtime.server.EmbeddedServer;
import jakarta.inject.Provider;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.utils.URIBuilder;
import io.kestra.core.utils.Rethrow;
import picocli.CommandLine;

Expand All @@ -26,10 +27,13 @@
import java.text.MessageFormat;
import java.time.temporal.ChronoUnit;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.Callable;
import jakarta.inject.Inject;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;

@CommandLine.Command(
@Command(
versionProvider = VersionProvider.class,
mixinStandardHelpOptions = true,
showDefaultValues = true
Expand All @@ -49,22 +53,28 @@ abstract public class AbstractCommand implements Callable<Integer> {
@Inject
private io.kestra.core.utils.VersionProvider versionProvider;

@Inject
protected Provider<PluginRegistry> pluginRegistryProvider;

@Inject
protected Provider<PluginManager> pluginManagerProvider;

private PluginRegistry pluginRegistry;

@CommandLine.Option(names = {"-v", "--verbose"}, description = "Change log level. Multiple -v options increase the verbosity.", showDefaultValue = CommandLine.Help.Visibility.NEVER)
@Option(names = {"-v", "--verbose"}, description = "Change log level. Multiple -v options increase the verbosity.", showDefaultValue = CommandLine.Help.Visibility.NEVER)
private boolean[] verbose = new boolean[0];

@CommandLine.Option(names = {"-l", "--log-level"}, description = "Change log level (values: ${COMPLETION-CANDIDATES})")
@Option(names = {"-l", "--log-level"}, description = "Change log level (values: ${COMPLETION-CANDIDATES})")
private LogLevel logLevel = LogLevel.INFO;

@CommandLine.Option(names = {"--internal-log"}, description = "Change also log level for internal log")
@Option(names = {"--internal-log"}, description = "Change also log level for internal log")
private boolean internalLog = false;

@CommandLine.Option(names = {"-c", "--config"}, description = "Path to a configuration file")
@Option(names = {"-c", "--config"}, description = "Path to a configuration file")
private Path config = Paths.get(System.getProperty("user.home"), ".kestra/config.yml");

@CommandLine.Option(names = {"-p", "--plugins"}, description = "Path to plugins directory")
protected Path pluginsPath = System.getenv("KESTRA_PLUGINS_PATH") != null ? Paths.get(System.getenv("KESTRA_PLUGINS_PATH")) : null;
@Option(names = {"-p", "--plugins"}, description = "Path to plugins directory")
protected Path pluginsPath = Optional.ofNullable(System.getenv("KESTRA_PLUGINS_PATH")).map(Paths::get).orElse(null);

public enum LogLevel {
TRACE,
Expand All @@ -76,16 +86,18 @@ public enum LogLevel {

@Override
public Integer call() throws Exception {
Thread.currentThread().setName(this.getClass().getDeclaredAnnotation(CommandLine.Command.class).name());
Thread.currentThread().setName(this.getClass().getDeclaredAnnotation(Command.class).name());
startLogger();
sendServerLog();
if (this.startupHook != null) {
this.startupHook.start(this);
}

if (this.pluginsPath != null && loadExternalPlugins()) {
pluginRegistry = pluginRegistry();
pluginRegistry = pluginRegistryProvider.get();
pluginRegistry.registerIfAbsent(pluginsPath);
PluginManager manager = pluginManagerProvider.get();
manager.start();
}

startWebserver();
Expand All @@ -102,10 +114,6 @@ protected boolean loadExternalPlugins() {
return true;
}

protected PluginRegistry pluginRegistry() {
return KestraContext.getContext().getPluginRegistry(); // Lazy init
}

private static String message(String message, Object... format) {
return CommandLine.Help.Ansi.AUTO.string(
format.length == 0 ? message : MessageFormat.format(message, format)
Expand Down Expand Up @@ -183,9 +191,9 @@ private void startWebserver() {
if (this.endpointConfiguration.getPort().isPresent()) {
URI endpoint = null;
try {
endpoint = new URIBuilder(server.getURL().toURI())
.setPort(this.endpointConfiguration.getPort().get())
.setPath("/health")
endpoint = UriBuilder.of(server.getURL().toURI())
.port(this.endpointConfiguration.getPort().get())
.path("/health")
.build();
} catch (URISyntaxException e) {
e.printStackTrace();
Expand Down
18 changes: 11 additions & 7 deletions cli/src/main/java/io/kestra/cli/commands/plugins/PluginCommand.java
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
package io.kestra.cli.commands.plugins;

import io.micronaut.configuration.picocli.PicocliRunner;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import io.kestra.cli.AbstractCommand;
import io.kestra.cli.App;
import picocli.CommandLine;
import io.micronaut.configuration.picocli.PicocliRunner;
import lombok.SneakyThrows;
import picocli.CommandLine.Command;

@CommandLine.Command(
@Command(
name = "plugins",
description = "Manage plugins",
mixinStandardHelpOptions = true,
Expand All @@ -18,15 +17,20 @@
PluginSearchCommand.class
}
)
@Slf4j
public class PluginCommand extends AbstractCommand {

@SneakyThrows
@Override
public Integer call() throws Exception {
super.call();

PicocliRunner.call(App.class, "plugins", "--help");
PicocliRunner.call(App.class, "plugins", "--help");

return 0;
}

@Override
protected boolean loadExternalPlugins() {
return false;
}
}
5 changes: 4 additions & 1 deletion cli/src/main/java/io/kestra/cli/commands/plugins/PluginDocCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.google.common.io.Files;
import io.kestra.cli.AbstractCommand;
import io.kestra.core.docs.DocumentationGenerator;
import io.kestra.core.plugins.PluginRegistry;
import io.kestra.core.plugins.RegisteredPlugin;
import io.kestra.core.serializers.JacksonMapper;
import io.micronaut.context.ApplicationContext;
Expand Down Expand Up @@ -42,8 +43,10 @@ public Integer call() throws Exception {
super.call();
DocumentationGenerator documentationGenerator = applicationContext.getBean(DocumentationGenerator.class);

List<RegisteredPlugin> plugins = core ? pluginRegistry().plugins() : pluginRegistry().externalPlugins();
PluginRegistry registry = pluginRegistryProvider.get();
List<RegisteredPlugin> plugins = core ? registry.plugins() : registry.externalPlugins();
boolean hasFailures = false;

for (RegisteredPlugin registeredPlugin : plugins) {
try {
documentationGenerator
Expand Down
105 changes: 55 additions & 50 deletions cli/src/main/java/io/kestra/cli/commands/plugins/PluginInstallCommand.java
View file
Original file line number Diff line number Diff line change
@@ -1,98 +1,103 @@
package io.kestra.cli.commands.plugins;

import org.apache.commons.io.FilenameUtils;
import io.kestra.core.contexts.MavenPluginRepositoryConfig;
import io.kestra.core.plugins.LocalPluginManager;
import io.kestra.core.plugins.MavenPluginDownloader;
import io.kestra.core.plugins.PluginArtifact;
import io.kestra.core.plugins.PluginManager;
import io.micronaut.http.uri.UriBuilder;
import io.kestra.cli.AbstractCommand;
import io.kestra.cli.plugins.PluginDownloader;
import io.kestra.cli.plugins.RepositoryConfig;
import io.kestra.core.utils.IdUtils;
import org.apache.http.client.utils.URIBuilder;
import jakarta.inject.Provider;
import picocli.CommandLine;

import java.net.URI;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import jakarta.inject.Inject;
import picocli.CommandLine.Command;
import picocli.CommandLine.Parameters;
import picocli.CommandLine.Option;
import picocli.CommandLine.Spec;

import static io.kestra.core.utils.Rethrow.throwConsumer;

@CommandLine.Command(
@Command(
name = "install",
description = "Install plugins"
)
public class PluginInstallCommand extends AbstractCommand {
@CommandLine.Parameters(index = "0..*", description = "Plugins to install. Represented as Maven artifact coordinates.")

@Option(names = {"--locally"}, description = "Specifies if plugins must be installed locally. If set to false the installation depends on your Kestra configuration.")
boolean locally = true;

@Parameters(index = "0..*", description = "Plugins to install. Represented as Maven artifact coordinates.")
List<String> dependencies = new ArrayList<>();

@CommandLine.Option(names = {"--repositories"}, description = "URL to additional Maven repositories")
@Option(names = {"--repositories"}, description = "URL to additional Maven repositories")
private URI[] repositories;

@CommandLine.Spec
@Spec
CommandLine.Model.CommandSpec spec;

@Inject
private PluginDownloader pluginDownloader;
Provider<MavenPluginDownloader> mavenPluginRepositoryProvider;

@Override
public Integer call() throws Exception {
super.call();

if (this.pluginsPath == null) {
if (this.locally && this.pluginsPath == null) {
throw new CommandLine.ParameterException(this.spec.commandLine(), "Missing required options '--plugins' " +
"or environment variable 'KESTRA_PLUGINS_PATH"
);
}

if (!pluginsPath.toFile().exists()) {
if (!pluginsPath.toFile().mkdir()) {
throw new RuntimeException("Cannot create directory: " + pluginsPath.toFile().getAbsolutePath());
}
}

List<MavenPluginRepositoryConfig> repositoryConfigs = List.of();
if (repositories != null) {
Arrays.stream(repositories)
.forEach(throwConsumer(s -> {
URIBuilder uriBuilder = new URIBuilder(s);

RepositoryConfig.RepositoryConfigBuilder builder = RepositoryConfig.builder()
repositoryConfigs = Arrays.stream(repositories)
.map(uri -> {
MavenPluginRepositoryConfig.MavenPluginRepositoryConfigBuilder builder = MavenPluginRepositoryConfig
.builder()
.id(IdUtils.create());

if (uriBuilder.getUserInfo() != null) {
int index = uriBuilder.getUserInfo().indexOf(":");

builder.basicAuth(new RepositoryConfig.BasicAuth(
uriBuilder.getUserInfo().substring(0, index),
uriBuilder.getUserInfo().substring(index + 1)
String userInfo = uri.getUserInfo();
if (userInfo != null) {
String[] userInfoParts = userInfo.split(":");
builder = builder.basicAuth(new MavenPluginRepositoryConfig.BasicAuth(
userInfoParts[0],
userInfoParts[1]
));

uriBuilder.setUserInfo(null);
}

builder.url(uriBuilder.build().toString());

pluginDownloader.addRepository(builder.build());
}));
builder.url(UriBuilder.of(uri).userInfo(null).build().toString());
return builder.build();
}).toList();
}

List<URL> resolveUrl = pluginDownloader.resolve(dependencies);
stdOut("Resolved Plugin(s) with {0}", resolveUrl);
final List<PluginArtifact> pluginArtifacts;
try {
pluginArtifacts = dependencies.stream().map(PluginArtifact::fromCoordinates).toList();
} catch (IllegalArgumentException e) {
stdErr(e.getMessage());
return CommandLine.ExitCode.USAGE;
}

for (URL url: resolveUrl) {
Files.copy(
Paths.get(url.toURI()),
Paths.get(pluginsPath.toString(), FilenameUtils.getName(url.toString())),
StandardCopyOption.REPLACE_EXISTING
try (final PluginManager pluginManager = getPluginManager()) {
List<PluginArtifact> installed = pluginManager.install(
pluginArtifacts,
repositoryConfigs,
false,
pluginsPath
);
}

stdOut("Successfully installed plugins {0} into {1}", dependencies, pluginsPath);
List<URI> uris = installed.stream().map(PluginArtifact::uri).toList();
stdOut("Successfully installed plugins {0} into {1}", dependencies, uris);
return CommandLine.ExitCode.OK;
}
}

return 0;
private PluginManager getPluginManager() {
return locally ? new LocalPluginManager(mavenPluginRepositoryProvider.get()) : this.pluginManagerProvider.get();
}

@Override
Expand Down
18 changes: 14 additions & 4 deletions cli/src/main/java/io/kestra/cli/commands/plugins/PluginListCommand.java
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,31 @@
package io.kestra.cli.commands.plugins;

import io.kestra.cli.AbstractCommand;
import io.kestra.core.plugins.PluginRegistry;
import io.kestra.core.plugins.RegisteredPlugin;
import jakarta.inject.Inject;
import jakarta.inject.Provider;
import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
import picocli.CommandLine.Spec;

import java.util.List;

@CommandLine.Command(
@Command(
name = "list",
description = "List all plugins already installed"
)
public class PluginListCommand extends AbstractCommand {
@CommandLine.Spec
@Spec
CommandLine.Model.CommandSpec spec;

@CommandLine.Option(names = {"--core"}, description = "Also write core tasks plugins")
@Option(names = {"--core"}, description = "Also write core tasks plugins")
private boolean core = false;

@Inject
private PluginRegistry registry;

@Override
public Integer call() throws Exception {
super.call();
Expand All @@ -27,7 +36,8 @@ public Integer call() throws Exception {
);
}

List<RegisteredPlugin> plugins = core ? pluginRegistry().plugins() : pluginRegistry().externalPlugins();
List<RegisteredPlugin> plugins = core ? registry.plugins() : registry.externalPlugins();

plugins.forEach(registeredPlugin -> stdOut(registeredPlugin.toString()));

return 0;
Expand Down
Loading

0 comments on commit d92e3e8

Please sign in to comment.