Add support for supplemental configuration (#1371)

Co-authored-by: Musa <[email protected]>

cfg/envconfig/envconfig.go

@@ -30,6 +30,7 @@ const (
 	PodName                   = "POD_NAME"
 	HostIP                    = "HOST_IP"
 	CWConfigContent           = "CW_CONFIG_CONTENT"
+	CWAgentMergedOtelConfig   = "CWAGENT_MERGED_OTEL_CONFIG"
 )
 
 const (

cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent.go

@@ -36,6 +36,8 @@ import (
 	"github.com/aws/amazon-cloudwatch-agent/cfg/envconfig"
 	"github.com/aws/amazon-cloudwatch-agent/cmd/amazon-cloudwatch-agent/internal"
 	"github.com/aws/amazon-cloudwatch-agent/extension/agenthealth/handler/useragent"
+	"github.com/aws/amazon-cloudwatch-agent/internal/mapstructure"
+	"github.com/aws/amazon-cloudwatch-agent/internal/merge/confmap"
 	"github.com/aws/amazon-cloudwatch-agent/internal/version"
 	cwaLogger "github.com/aws/amazon-cloudwatch-agent/logger"
 	"github.com/aws/amazon-cloudwatch-agent/logs"
@@ -46,6 +48,7 @@ import (
 	"github.com/aws/amazon-cloudwatch-agent/service/defaultcomponents"
 	"github.com/aws/amazon-cloudwatch-agent/service/registry"
 	"github.com/aws/amazon-cloudwatch-agent/tool/paths"
+	"github.com/aws/amazon-cloudwatch-agent/translator/tocwconfig/toyamlconfig"
 )
 
 const (
@@ -62,7 +65,7 @@ var fTest = flag.Bool("test", false, "enable test mode: gather metrics, print th
 var fTestWait = flag.Int("test-wait", 0, "wait up to this many seconds for service inputs to complete in test mode")
 var fSchemaTest = flag.Bool("schematest", false, "validate the toml file schema")
 var fTomlConfig = flag.String("config", "", "configuration file to load")
-var fOtelConfig = flag.String("otelconfig", paths.YamlConfigPath, "YAML configuration file to run OTel pipeline")
+var fOtelConfigs configprovider.OtelConfigFlags
 var fEnvConfig = flag.String("envconfig", "", "env configuration file to load")
 var fConfigDirectory = flag.String("config-directory", "",
 	"directory containing additional *.conf files")
@@ -184,7 +187,7 @@ func reloadLoop(
 					_ = f.Close()
 				}
 			}
-			log.Fatalf("E! [telegraf] Error running agent: %v", err)
+			log.Fatalf("E! Error running agent: %v", err)
 		}
 	}
 }
@@ -312,35 +315,65 @@ func runAgent(ctx context.Context,
 		// Always run logAgent as goroutine regardless of whether starting OTEL or Telegraf.
 		go logAgent.Run(ctx)
 
-		// If OTEL config does not exist, then ASSUME just monitoring logs.
+		// If only the default CWAgent config yaml is provided and does not exist, then ASSUME
+		// just monitoring logs since this is the default when no OTEL config flag is provided.
 		// So just start Telegraf.
-		_, err = os.Stat(*fOtelConfig)
-		if errors.Is(err, os.ErrNotExist) {
-			useragent.Get().SetComponents(&otelcol.Config{}, c)
-			return ag.Run(ctx)
+		if len(fOtelConfigs) == 1 && fOtelConfigs[0] == paths.YamlConfigPath {
+			_, err = os.Stat(fOtelConfigs[0])
+			if errors.Is(err, os.ErrNotExist) {
+				useragent.Get().SetComponents(&otelcol.Config{}, c)
+				return ag.Run(ctx)
+			}
 		}
 	}
 	// Else start OTEL and rely on adapter package to start the logfile plugin.
-	yamlConfigPath := *fOtelConfig
 	level := cwaLogger.ConvertToAtomicLevel(wlog.LogLevel())
 	logger, loggerOptions := cwaLogger.NewLogger(writer, level)
-	providerSettings := configprovider.GetSettings(yamlConfigPath, logger)
+
+	uris := fOtelConfigs
+	// merge configs together
+	if len(uris) > 1 {
+		log.Printf("D! Merging multiple OTEL configurations: %s", uris)
+		result := confmap.New()
+		for _, path := range fOtelConfigs {
+			conf, err := confmap.LoadConf(path)
+			if err != nil {
+				return fmt.Errorf("failed to load OTEL configs: %w", err)
+			}
+			if err = result.Merge(conf); err != nil {
+				return fmt.Errorf("failed to merge OTEL configs: %w", err)
+			}
+		}
+		_ = os.Setenv(envconfig.CWAgentMergedOtelConfig, toyamlconfig.ToYamlConfig(result.ToStringMap()))
+		uris = []string{"env:" + envconfig.CWAgentMergedOtelConfig}
+	} else {
+		_ = os.Unsetenv(envconfig.CWAgentMergedOtelConfig)
+	}
+
+	providerSettings := configprovider.GetSettings(uris, logger)
 	provider, err := otelcol.NewConfigProvider(providerSettings)
 	if err != nil {
-		log.Printf("E! Error while initializing config provider: %v\n", err)
-		return err
+		return fmt.Errorf("error while initializing config provider: %v", err)
 	}
 
 	factories, err := components(c)
 	if err != nil {
-		log.Printf("E! Error while adapting telegraf input plugins: %v\n", err)
-		return err
+		return fmt.Errorf("error while adapting telegraf input plugins: %v", err)
 	}
 
 	cfg, err := provider.Get(ctx, factories)
 	if err != nil {
 		return err
 	}
+
+	if _, ok := os.LookupEnv(envconfig.CWAgentMergedOtelConfig); ok {
+		result, err := mapstructure.Marshal(cfg)
+		if err != nil {
+			return fmt.Errorf("failed to marshal OTEL configuration: %v", err)
+		}
+		log.Printf("I! Merged OTEL configuration: \n%s\n", toyamlconfig.ToYamlConfig(result))
+	}
+
 	useragent.Get().SetComponents(cfg, c)
 
 	params := getCollectorParams(factories, providerSettings, loggerOptions)
@@ -352,7 +385,10 @@ func runAgent(ctx context.Context,
 	// The config path below here is actually used that was set in the settings above.
 	// docs: https://github.com/open-telemetry/opentelemetry-collector/blob/93cbae436ae61b832279dbbb18a0d99214b7d305/otelcol/command.go#L63
 	// *************************************************************************************************
-	e := []string{"--config=" + yamlConfigPath}
+	var e []string
+	for _, uri := range uris {
+		e = append(e, "--config="+uri)
+	}
 	cmd.SetArgs(e)
 	return cmd.Execute()
 }
@@ -422,7 +458,11 @@ func (p *program) Stop(_ service.Service) error {
 }
 
 func main() {
+	flag.Var(&fOtelConfigs, configprovider.OtelConfigFlagName, "YAML configuration files to run OTel pipeline")
 	flag.Parse()
+	if len(fOtelConfigs) == 0 {
+		_ = fOtelConfigs.Set(paths.YamlConfigPath)
+	}
 	args := flag.Args()
 	sectionFilters, inputFilters, outputFilters := []string{}, []string{}, []string{}
 	if *fSectionFilters != "" {

cmd/config-downloader/downloader.go

@@ -18,8 +18,8 @@ import (
 
 	configaws "github.com/aws/amazon-cloudwatch-agent/cfg/aws"
 	"github.com/aws/amazon-cloudwatch-agent/cfg/commonconfig"
+	"github.com/aws/amazon-cloudwatch-agent/internal/constants"
 	"github.com/aws/amazon-cloudwatch-agent/translator/config"
-	"github.com/aws/amazon-cloudwatch-agent/translator/context"
 	"github.com/aws/amazon-cloudwatch-agent/translator/util"
 	sdkutil "github.com/aws/amazon-cloudwatch-agent/translator/util"
 )
@@ -173,7 +173,7 @@ func main() {
 					return filepath.SkipDir
 				}
 			}
-			if filepath.Ext(path) == context.TmpFileSuffix {
+			if filepath.Ext(path) == constants.FileSuffixTmp {
 				return os.Remove(path)
 			}
 			return nil
@@ -211,7 +211,7 @@ func main() {
 	}
 
 	if multiConfig != "remove" {
-		outputFilePath = filepath.Join(outputDir, outputFilePath+context.TmpFileSuffix)
+		outputFilePath = filepath.Join(outputDir, outputFilePath+constants.FileSuffixTmp)
 		err = os.WriteFile(outputFilePath, []byte(config), 0644)
 		if err != nil {
 			log.Panicf("E! Failed to write the json file %v: %v", outputFilePath, err)

cmd/start-amazon-cloudwatch-agent/path.go

@@ -17,6 +17,7 @@ import (
 	"github.com/BurntSushi/toml"
 
 	"github.com/aws/amazon-cloudwatch-agent/cfg/envconfig"
+	"github.com/aws/amazon-cloudwatch-agent/internal/util/config"
 	"github.com/aws/amazon-cloudwatch-agent/internal/util/user"
 	"github.com/aws/amazon-cloudwatch-agent/tool/paths"
 )
@@ -28,9 +29,9 @@ func startAgent(writer io.WriteCloser) error {
 			paths.AgentBinaryPath, // when using syscall.Exec, must pass binary name as args[0]
 			"-config", paths.TomlConfigPath,
 			"-envconfig", paths.EnvConfigPath,
-			"-otelconfig", paths.YamlConfigPath,
-			"-pidfile", paths.AgentDir + "/var/amazon-cloudwatch-agent.pid",
 		}
+		execArgs = append(execArgs, config.GetOTELConfigArgs(paths.CONFIG_DIR_IN_CONTAINER)...)
+		execArgs = append(execArgs, "-pidfile", paths.AgentDir+"/var/amazon-cloudwatch-agent.pid")
 		if err := syscall.Exec(paths.AgentBinaryPath, execArgs, os.Environ()); err != nil {
 			return fmt.Errorf("error exec as agent binary: %w", err)
 		}
@@ -69,9 +70,9 @@ func startAgent(writer io.WriteCloser) error {
 		paths.AgentBinaryPath,
 		"-config", paths.TomlConfigPath,
 		"-envconfig", paths.EnvConfigPath,
-		"-otelconfig", paths.YamlConfigPath,
-		"-pidfile", paths.AgentDir + "/var/amazon-cloudwatch-agent.pid",
 	}
+	agentCmd = append(agentCmd, config.GetOTELConfigArgs(paths.ConfigDirPath)...)
+	agentCmd = append(agentCmd, "-pidfile", paths.AgentDir+"/var/amazon-cloudwatch-agent.pid")
 	if err = syscall.Exec(name, agentCmd, os.Environ()); err != nil {
 		// log file is closed, so use fmt here
 		fmt.Printf("E! Exec failed: %v \n", err)

cmd/start-amazon-cloudwatch-agent/path_windows.go

@@ -14,6 +14,7 @@ import (
 	"os/exec"
 
 	"github.com/aws/amazon-cloudwatch-agent/cfg/envconfig"
+	"github.com/aws/amazon-cloudwatch-agent/internal/util/config"
 	"github.com/aws/amazon-cloudwatch-agent/tool/paths"
 )
 
@@ -23,24 +24,24 @@ func startAgent(writer io.WriteCloser) error {
 			log.Printf("E! Cannot close the log file, ERROR is %v \n", err)
 			return err
 		}
-		cmd := exec.Command(
-			paths.AgentBinaryPath,
+		execArgs := []string{
 			"-config", paths.TomlConfigPath,
 			"-envconfig", paths.EnvConfigPath,
-			"-otelconfig", paths.YamlConfigPath,
-		)
+		}
+		execArgs = append(execArgs, config.GetOTELConfigArgs(paths.ConfigDirPath)...)
+		cmd := exec.Command(paths.AgentBinaryPath, execArgs...)
 		stdoutStderr, err := cmd.CombinedOutput()
 		// log file is closed, so use fmt here
 		fmt.Printf("%s \n", stdoutStderr)
 		return err
 	} else {
-		cmd := exec.Command(
-			paths.AgentBinaryPath,
+		execArgs := []string{
 			"-config", paths.TomlConfigPath,
 			"-envconfig", paths.EnvConfigPath,
-			"-otelconfig", paths.YamlConfigPath,
-			"-console", "true",
-		)
+		}
+		execArgs = append(execArgs, config.GetOTELConfigArgs(paths.CONFIG_DIR_IN_CONTAINER)...)
+		execArgs = append(execArgs, "-console", "true")
+		cmd := exec.Command(paths.AgentBinaryPath, execArgs...)
 		cmd.Stdin = os.Stdin
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr

cmd/start-amazon-cloudwatch-agent/start-amazon-cloudwatch-agent.go

@@ -24,7 +24,7 @@ func translateConfig() error {
 	if envconfig.IsRunningInContainer() {
 		args = append(args, "--input-dir", paths.CONFIG_DIR_IN_CONTAINER)
 	} else {
-		args = append(args, "--input", paths.JsonConfigPath, "--input-dir", paths.JsonDirPath, "--config", paths.CommonConfigPath)
+		args = append(args, "--input", paths.JsonConfigPath, "--input-dir", paths.ConfigDirPath, "--config", paths.CommonConfigPath)
 	}
 	cmd := exec.Command(paths.TranslatorBinaryPath, args...)
 	cmd.Stdout = os.Stdout