Changes to binary distribution (#6542)

---------

Co-authored-by: Andrzej Stencel <[email protected]>

_meta/.flavors

@@ -0,0 +1,13 @@
+basic:
+  - agentbeat
+  - endpoint-security
+  - pf-host-agent
+servers:
+  - agentbeat
+  - endpoint-security
+  - pf-host-agent
+  - cloudbeat
+  - apm-server
+  - fleet-server
+  - pf-elastic-symbolizer
+  - pf-elastic-collector

changelog/fragments/1737131552-Changes-to-binary-distribution.yaml

@@ -0,0 +1,43 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: breaking-change
+
+# Change summary; a 80ish characters long description of the change.
+summary: Changes to binary distribution
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: |
+  Default install installs only:
+    - agentbeat
+    - endpoint-security
+    - pf-host-agent
+  additional flag is added that includes components above and:
+    - cloudbeat
+    - apm-server
+    - fleet-server
+    - pf-elastic-symbolizer
+    - pf-elastic-collector
+
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

dev-tools/mage/settings.go

@@ -346,10 +346,15 @@ func PackageManifest() (string, error) {
 		return "", fmt.Errorf("retrieving agent commit hash: %w", err)
 	}
 
-	return GeneratePackageManifest(BeatName, packageVersion, Snapshot, hash, commitHashShort)
+	registry, err := loadFlavorsRegistry()
+	if err != nil {
+		return "", fmt.Errorf("retrieving agent flavors: %w", err)
+	}
+
+	return GeneratePackageManifest(BeatName, packageVersion, Snapshot, hash, commitHashShort, registry)
 }
 
-func GeneratePackageManifest(beatName, packageVersion string, snapshot bool, fullHash, shortHash string) (string, error) {
+func GeneratePackageManifest(beatName, packageVersion string, snapshot bool, fullHash, shortHash string, flavorsRegistry map[string][]string) (string, error) {
 	m := v1.NewManifest()
 	m.Package.Version = packageVersion
 	m.Package.Snapshot = snapshot
@@ -360,6 +365,7 @@ func GeneratePackageManifest(beatName, packageVersion string, snapshot bool, ful
 	m.Package.PathMappings = []map[string]string{{}}
 	m.Package.PathMappings[0][versionedHomePath] = fmt.Sprintf("data/%s-%s%s-%s", beatName, m.Package.Version, GenerateSnapshotSuffix(snapshot), shortHash)
 	m.Package.PathMappings[0][v1.ManifestFileName] = fmt.Sprintf("data/%s-%s%s-%s/%s", beatName, m.Package.Version, GenerateSnapshotSuffix(snapshot), shortHash, v1.ManifestFileName)
+	m.Package.Flavors = flavorsRegistry
 	yamlBytes, err := yaml.Marshal(m)
 	if err != nil {
 		return "", fmt.Errorf("marshaling manifest: %w", err)
@@ -461,6 +467,10 @@ var (
 	beatVersionValue string
 	beatVersionErr   error
 	beatVersionOnce  sync.Once
+
+	flavorsRegistry    map[string][]string
+	flavorsRegistryErr error
+	flavorsOnce        sync.Once
 )
 
 // BeatQualifiedVersion returns the Beat's qualified version.  The value can be overwritten by
@@ -492,6 +502,14 @@ func beatVersion() (string, error) {
 	return beatVersionValue, beatVersionErr
 }
 
+func loadFlavorsRegistry() (map[string][]string, error) {
+	flavorsOnce.Do(func() {
+		flavorsRegistry, flavorsRegistryErr = getBuildVariableSources().GetFlavorsRegistry()
+	})
+
+	return flavorsRegistry, flavorsRegistryErr
+}
+
 var (
 	beatDocBranchRegex     = regexp.MustCompile(`(?m)doc-branch:\s*([^\s]+)\r?$`)
 	beatDocSiteBranchRegex = regexp.MustCompile(`(?m)doc-site-branch:\s*([^\s]+)\r?$`)
@@ -521,9 +539,10 @@ var (
 	// DefaultBeatBuildVariableSources contains the default locations build
 	// variables are read from by Elastic Beats.
 	DefaultBeatBuildVariableSources = &BuildVariableSources{
-		BeatVersion: "{{ elastic_beats_dir }}/version/version.go",
-		GoVersion:   "{{ elastic_beats_dir }}/.go-version",
-		DocBranch:   "{{ elastic_beats_dir }}/version/docs/version.asciidoc",
+		BeatVersion:     "{{ elastic_beats_dir }}/version/version.go",
+		GoVersion:       "{{ elastic_beats_dir }}/.go-version",
+		DocBranch:       "{{ elastic_beats_dir }}/version/docs/version.asciidoc",
+		FlavorsRegistry: "{{ elastic_beats_dir }}/_meta/.flavors",
 	}
 
 	buildVariableSources     *BuildVariableSources
@@ -584,6 +603,9 @@ type BuildVariableSources struct {
 
 	// Parses the documentation branch from the DocBranch file.
 	DocBranchParser func(data []byte) (string, error)
+
+	// File containing definition of flavors.
+	FlavorsRegistry string
 }
 
 func (s *BuildVariableSources) expandVar(in string) (string, error) {
@@ -628,6 +650,26 @@ func (s *BuildVariableSources) GetGoVersion() (string, error) {
 	return s.GoVersionParser(data)
 }
 
+// GetFlavorsRegistry reads the flavors file and parses the list of components of it.
+func (s *BuildVariableSources) GetFlavorsRegistry() (map[string][]string, error) {
+	file, err := s.expandVar(s.FlavorsRegistry)
+	if err != nil {
+		return nil, err
+	}
+
+	data, err := os.ReadFile(file)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read flavors from file=%v: %w", file, err)
+	}
+
+	registry := make(map[string][]string)
+	if err := yaml.Unmarshal(data, registry); err != nil {
+		return nil, fmt.Errorf("failed to parse flavors: %w", err)
+	}
+
+	return registry, nil
+}
+
 // GetDocBranch reads the DocBranch file and parses the branch from it.
 func (s *BuildVariableSources) GetDocBranch() (string, error) {
 	file, err := s.expandVar(s.DocBranch)