refactoring(test): autumn cleanup (#77)

* fix(i18n): fix typo and wording.

* feat(kv): KV storage publication.

* chore(go): go mod tidy.

.vscode/ltex.dictionary.en-US.txt

@@ -0,0 +1,16 @@
+mTLS
+gRPC
+Rundeck
+accessor
+HOCON
+ConfigMap
+protobuf
+NaCL
+saltpack
+secretbox
+Unmarshall
+Unmarshal
+CheatSheet
+Protobuf
+non-instantiable
+Mailgun

.vscode/ltex.hiddenFalsePositives.en-US.txt

@@ -0,0 +1,23 @@
+{"rule":"MORFOLOGIK_RULE_EN_US","sentence":"^\\QHarp is for Harpocrates (Ancient Greek: Ἁρποκράτης) the god of silence, secrets and confidentiality in the Hellenistic religion.\\E$"}
+{"rule":"MORFOLOGIK_RULE_EN_US","sentence":"^\\Q#ChickenEggProblem\\E$"}
+{"rule":"MORFOLOGIK_RULE_EN_US","sentence":"^\\QKubernetes\nHelm\nOpen Policy Agent ConfTest\nSaltPack\nHashicorp Vault\nAWS SDK Go\\E$"}
+{"rule":"ENGLISH_WORD_REPEAT_BEGINNING_RULE","sentence":"^\\QYou need a \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q to \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q your long term cloud secret storage - Harp will help you to create secret containers that can be consumed on deployment.\\E$"}
+{"rule":"ENGLISH_WORD_REPEAT_BEGINNING_RULE","sentence":"^\\QYou want to \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q your secrets from one secret storage to another - Harp provides you a secret container to store these secrets while they are going to be distributed in other secret storage implementations.\\E$"}
+{"rule":"ENGLISH_WORD_REPEAT_BEGINNING_RULE","sentence":"^\\QYou have to \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q a secret (rotation/deprecation/renewal) - Harp provides you a \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q secret \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q, so that you can define a specification to describe how your secret operation is going to be applied offline on the secret container.\\E$"}
+{"rule":"MORFOLOGIK_RULE_EN_US","sentence":"^\\QCustomer BundleTemplate\\E$"}
+{"rule":"MORFOLOGIK_RULE_EN_US","sentence":"^\\QThe \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Qis the BundleTemplate used to generate the \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q.\\E$"}
+{"rule":"MORFOLOGIK_RULE_EN_US","sentence":"^\\QThe \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q is a map of SecretChain where key is the version, and value the data.\\E$"}
+{"rule":"MORFOLOGIK_RULE_EN_US","sentence":"^\\QThe \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q sets the version number of the current SecretChain.\\E$"}
+{"rule":"MASS_AGREEMENT","sentence":"^\\QEach ring evaluate a threat realization impact level when a ring secrets leak occurs.\\E$"}
+{"rule":"MORFOLOGIK_RULE_EN_US","sentence":"^\\QProtobuf definition - InfrastructureNS\\E$"}
+{"rule":"EN_A_VS_AN","sentence":"^\\QAn \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q is a consistent set of resource provided by an infrastructure provider (IaaS).\\E$"}
+{"rule":"MORFOLOGIK_RULE_EN_US","sentence":"^\\QProtobuf definition - PlatformNS\\E$"}
+{"rule":"MORFOLOGIK_RULE_EN_US","sentence":"^\\QProtobuf definition - ProductComponentNS\\E$"}
+{"rule":"MORFOLOGIK_RULE_EN_US","sentence":"^\\QProtobuf definition - ApplicationComponentNS\\E$"}
+{"rule":"EN_A_VS_AN","sentence":"^\\QAn \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q is an instance of a \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q running on a \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q at a \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q stage level.\\E$"}
+{"rule":"MORFOLOGIK_RULE_EN_US","sentence":"^\\QExtracted from Customer BundleTemplate\\E$"}
+{"rule":"MORFOLOGIK_RULE_EN_US","sentence":"^\\QIntroduction\nBundle\nBundleTemplate\nBundlePatch\\E$"}
+{"rule":"A_NNS","sentence":"^\\QCreate a secret container from a structured file\nSecret template\nSample output\nDeploy secrets\nHashicorp Vault\nApplication access profile\nMozilla SOPS\nCreate the identity\nSeal using sops\nRetrieve a bundle from a sops encrypted file\nGenerate a YAML output from a bundle\nComplete scenario\\E$"}
+{"rule":"MORFOLOGIK_RULE_EN_US","sentence":"^\\QGenerate the \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q Vault AppRole and Policy using a terraform script via \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q plugin.\\E$"}
+{"rule":"MORFOLOGIK_RULE_EN_US","sentence":"^\\QGenerate the \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q Vault AppRole and Policy using terraform script via \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q plugin.\\E$"}
+{"rule":"A_NNS","sentence":"^\\QRetrieve a bundle from a sops encrypted file\\E$"}

CHANGELOG.md

@@ -2,10 +2,32 @@
 
 ### Not released yet
 
+BREAKING-CHANGES:
+
+* cmd/ruleset: Ruleset generation from a Bundle has been relocated to `to ruleset` command. [#77](https://github.com/elastic/harp/pull/77)
+* bundle/filter: Parameter `--jmespath` as been renamed to `--query`. [#77](https://github.com/elastic/harp/pull/77)
+* bundle/dump: Parameter `--jmespath` as been renamed to `--query`. [#77](https://github.com/elastic/harp/pull/77)
+* deprecation: Package `github.com/elastic/harp/pkg/bundle/vfs` has been removed. The Golang 1.16 `fs.FS` implementation must be used and located at `github.com/elastic/harp/pkg/bundle/fs`. [#77](https://github.com/elastic/harp/pull/77)
+
+CHANGES:
+
+FEATURES:
+
+* bundle/encryption: Partial bundle encryption based on annotations. [#77](https://github.com/elastic/harp/pull/77)
+* task/bundle: Fully unit tested. [#77](https://github.com/elastic/harp/pull/77)
+* core/kv: Support KV Store publication for Etcd3/Zookeeper/Consul. [#77](https://github.com/elastic/harp/pull/77)
+* value/transformer: Transformer mock is available for testing. [#77](https://github.com/elastic/harp/pull/77)
+* value/encryption: Expose `encryption.Must(value.Transformer, error)` to build a transformer instance with a panic raised on error. [#77](https://github.com/elastic/harp/pull/77)
+* sdk/types: `IsNill()` now recognize nil function pointer. [#77](https://github.com/elastic/harp/pull/77)
+* sdk/cmdutil: `DiscardWriter()` is a `io.Writer` provider used to discard all output. [#77](https://github.com/elastic/harp/pull/77)
+* sdk/cmdutil: `DirectWriter(io.Writer)` is a `io.Writer` provider used to delegate to input writer. [#77](https://github.com/elastic/harp/pull/77)
+* sdk/cmdutil: `NewClosedWriter()` is a `io.Writer` implementation who always return on `Write()` calls. [#77](https://github.com/elastic/harp/pull/77)
+
 DIST:
 
 * go: Build with Golang 1.17.3.
 * tools: Update `golangci-lint` to `v1.43.0`. [#76](https://github.com/elastic/harp/pull/76)
+* docs: General review for typo / grammar.
 
 ## 0.2.0
 

README.md

@@ -34,21 +34,21 @@ and confidentiality in the Hellenistic religion. - [Wikipedia](https://en.wikipe
 
 ## TL;DR.
 
-Harp is a toolset to handle secret data in a `reproducible` way by providing a way
-to describe how your value is technically managed and `consistently` associated
-to a `predictable` secret path with additional metadata (ownership, rotation
-period, leak severity, etc.).
+Harp is a tool set to handle secret data in a `reproducible` way by providing a
+way to describe how your value is technically managed and `consistently`
+associated to a `predictable` secret path with additional metadata (ownership,
+rotation period, leak severity, etc.).
 
-These values (path => value) form a `Bundle` stored in an immutable file named
-a `Container`. This `Container` acts as a pivot format to allow Harp commands
-to communicate and create data management pipelines.
+These values (path ⇒ value) form a `Bundle` stored in an immutable file named a
+`Container`. This `Container` acts as a pivot format to allow Harp commands to
+communicate and create data management pipelines.
 
 In addition to that, it provides a `template engine` used to generate various
-secret values (password, passphrase, crypto keys, etc.) but also as a secret
+confidence values (password, passphrase, crypto keys, etc.) but also as a secret
 consumer, it merges value placeholders from a file content to render a final
 configuration file.
 
-Finally, it provides an `SDK` to allow developers to integrate `Harp` features
+Finally, it provides a `SDK` to allow developers to integrate `Harp` features
 in their products, and/or extend the Harp pipeline features by creating new
 [plugins](#plugins).
 
@@ -74,19 +74,19 @@ in their products, and/or extend the Harp pipeline features by creating new
 
 ## Use cases
 
-* If you want to have a `single secret value` and you are asking yourself
+* You want to have a `single secret value` and you are asking yourself
   `how to generate a strong password` - Harp has a template engine with secret
   value generation functions to allow you to generate such values.
-* If you have `thousands secrets` to handle to deploy your platform/customers
+* You have `thousands secrets` to handle to deploy your platform/customers
   `on multiple cloud providers` with `different secret storages` - Harp will help you
   to define consistent secret provisioning bundles and pipelines.
-* If you are in the situation when you need an `ephemeral secret storage` to
-  `bootstrap` your long term cloud secret storage - Harp will help you to create
+* You need a `ephemeral secret storage` to `bootstrap` your long term cloud
+  secret storage - Harp will help you to create
   secret containers that can be consumed on deployment.
-* If you want to `migrate massively` your secrets from one secret storage to
+* You want to `migrate massively` your secrets from one secret storage to
   another - Harp provides you a secret container to store these secrets while
   they are going to be distributed in other secret storage implementations.
-* If you have to `alter/modifiy` a secret (rotation/deprecation/renewal) - Harp
+* You have to `alter/modifiy` a secret (rotation/deprecation/renewal) - Harp
   provides you a `GitOps-able` secret `storage agnostic operation set`, so that you
   can define a specification to describe how your secret operation is going to
   be applied offline on the secret container.
@@ -99,21 +99,21 @@ in their products, and/or extend the Harp pipeline features by creating new
 
 `harp` allows you to handle secrets using deterministic pipelines expressed
 using an atomic series of CLI operations applied to a commonly shared container
-immutable and standalone filesystem used to store secret collection (Bundle)
+immutable and standalone file system used to store secret collection (Bundle)
 generated from a template engine via user specification, or external secret
 value coming from files or external secret storage.
 
 ![Pipelines](docs/harp/img/SM-HARP.png)
 
-These pipelines use the immutable container filesystem as a data exchange
+These pipelines use the immutable container file system as a data exchange
 protocol and could be extended for new input, intermediary operation or output
 via plugins created with the `harp` SDK.
 
 ### Immutable transformation
 
-Each applied transformation create a container with transformed data inside.
+Each applied transformation creates a container with transformed data inside.
 This will enforce container reproducibility by eliminating cumulative
-side-effects applied to the same container.
+side effects applied to the same container.
 
 The container handles for you the confidentiality and integrity protection applied
 to the secret collection stored inside and manipulated by copy during the
@@ -137,18 +137,18 @@ Harp provides :
   * A path name validation library exposed as `github.com/elastic/harp/pkg/cso`
 * A CLI for secret management implementation
   * CI/CD integration;
-  * Based on human readable definitions (YAML);
+  * Based on human-readable definitions (YAML);
   * In order to create auditable and reproducible pipelines.
   * An extensible tool which can be enhanced via [plugins](https://github.com/elastic/harp-plugins).
 
 And allows :
 
 * Bundle level operations
-  * Create a bundle from scratch / template / json (more via plugins);
+  * Create a bundle from scratch / template / JSON (more via plugins);
   * Generate a complete bundle using a YAML Descriptor (`BundleTemplate`) to describe secret and their usages;
-  * Read value stored in the K/V virtual filesystem;
-  * Update the K/V virtual filesystem;
-  * Reproductible patch applied on immutable container (copy-on-write);
+  * Read value stored in the K/V virtual file system;
+  * Update the K/V virtual file system;
+  * Reproducible patch applied on immutable container (copy-on-write);
   * Import / Export to Vault.
 * Immutable container level operations
   * Seal / Unseal a container for integrity and confidentiality property conservation
@@ -162,11 +162,11 @@ And allows :
   #ChickenEggProblem
 
 * Harp is only supporting `Vault`? - No, it has been published with only vault
-  support builtin, but it supports many other secret storage implementations via
+  support built-in, but it supports many other secret storage implementations via
   plugins.
 
-* What's the difference with `Vault`? - Hashicorp Vault is an encrypted highly
-  available K/V store with advanced autorization engine, it doesn't handle
+* What's the difference with `Vault`? - HashiCorp Vault is an encrypted highly
+  available K/V store with advanced authorization engine, it doesn't handle
   secret provisioning for you. You can't ask Vault to generate secrets for your
   application and store them using a defined logic. Harp is filling this
   requirement.

cmd/harp/internal/cmd/bundle_decrypt.go

@@ -23,18 +23,21 @@ import (
 
 	"github.com/elastic/harp/pkg/sdk/cmdutil"
 	"github.com/elastic/harp/pkg/sdk/log"
+	"github.com/elastic/harp/pkg/sdk/value"
 	"github.com/elastic/harp/pkg/sdk/value/encryption"
 	"github.com/elastic/harp/pkg/tasks/bundle"
 )
 
 // -----------------------------------------------------------------------------
+type bundleDecryptParams struct {
+	inputPath          string
+	outputPath         string
+	keys               []string
+	skipNotDecryptable bool
+}
 
 var bundleDecryptCmd = func() *cobra.Command {
-	var (
-		inputPath  string
-		outputPath string
-		key        string
-	)
+	params := &bundleDecryptParams{}
 
 	cmd := &cobra.Command{
 		Use:   "decrypt",
@@ -44,17 +47,28 @@ var bundleDecryptCmd = func() *cobra.Command {
 			ctx, cancel := cmdutil.Context(cmd.Context(), "harp-bundle-decrypt", conf.Debug.Enable, conf.Instrumentation.Logs.Level)
 			defer cancel()
 
-			// Create transformer according to used encryption key
-			transformer, err := encryption.FromKey(key)
-			if err != nil {
-				log.For(ctx).Fatal("unable to initialize transformer", zap.Error(err))
+			// Prepare transformer collection
+			transformers := []value.Transformer{}
+
+			// Split all alias / key
+			for _, keyRaw := range params.keys {
+				// Create transformer according to used encryption key
+				transformer, err := encryption.FromKey(keyRaw)
+				if err != nil {
+					log.For(ctx).Fatal("unable to initialize transformer", zap.String("key", keyRaw), zap.Error(err))
+					return
+				}
+
+				// Append to collection
+				transformers = append(transformers, transformer)
 			}
 
 			// Prepare task
 			t := &bundle.DecryptTask{
-				ContainerReader: cmdutil.FileReader(inputPath),
-				OutputWriter:    cmdutil.FileWriter(outputPath),
-				Transformer:     transformer,
+				ContainerReader:    cmdutil.FileReader(params.inputPath),
+				OutputWriter:       cmdutil.FileWriter(params.outputPath),
+				Transformers:       transformers,
+				SkipNotDecryptable: params.skipNotDecryptable,
 			}
 
 			// Run the task
@@ -65,10 +79,10 @@ var bundleDecryptCmd = func() *cobra.Command {
 	}
 
 	// Parameters
-	cmd.Flags().StringVar(&inputPath, "in", "", "Container input ('-' for stdin or filename)")
-	cmd.Flags().StringVar(&outputPath, "out", "", "Container output ('-' for stdout or filename)")
-	cmd.Flags().StringVar(&key, "key", "", "Secret value decryption key")
-	log.CheckErr("unable to mark 'key' flag as required.", cmd.MarkFlagRequired("key"))
+	cmd.Flags().StringVar(&params.inputPath, "in", "", "Container input ('-' for stdin or filename)")
+	cmd.Flags().StringVar(&params.outputPath, "out", "", "Container output ('-' for stdout or filename)")
+	cmd.Flags().StringSliceVar(&params.keys, "key", []string{""}, "Secret value decryption key. Repeat to add multiple keys to try.")
+	cmd.Flags().BoolVarP(&params.skipNotDecryptable, "skip-not-decryptable", "s", false, "Skip not decryptable secrets without raising an error.")
 
 	return cmd
 }

cmd/harp/internal/cmd/bundle_diff.go

@@ -27,13 +27,14 @@ import (
 )
 
 // -----------------------------------------------------------------------------
+type bundleDiffParams struct {
+	sourcePath      string
+	destinationPath string
+	generatePatch   bool
+}
 
 var bundleDiffCmd = func() *cobra.Command {
-	var (
-		sourcePath      string
-		destinationPath string
-		generatePatch   bool
-	)
+	params := &bundleDiffParams{}
 
 	cmd := &cobra.Command{
 		Use:   "diff",
@@ -45,10 +46,10 @@ var bundleDiffCmd = func() *cobra.Command {
 
 			// Prepare task
 			t := &bundle.DiffTask{
-				SourceReader:      cmdutil.FileReader(sourcePath),
-				DestinationReader: cmdutil.FileReader(destinationPath),
+				SourceReader:      cmdutil.FileReader(params.sourcePath),
+				DestinationReader: cmdutil.FileReader(params.destinationPath),
 				OutputWriter:      cmdutil.StdoutWriter(),
-				GeneratePatch:     generatePatch,
+				GeneratePatch:     params.generatePatch,
 			}
 
 			// Run the task
@@ -59,10 +60,10 @@ var bundleDiffCmd = func() *cobra.Command {
 	}
 
 	// Parameters
-	cmd.Flags().StringVar(&sourcePath, "old", "", "Container path ('-' for stdin or filename)")
-	cmd.Flags().StringVar(&destinationPath, "new", "", "Container path ('-' for stdin or filename)")
+	cmd.Flags().StringVar(&params.sourcePath, "old", "", "Container path ('-' for stdin or filename)")
+	cmd.Flags().StringVar(&params.destinationPath, "new", "", "Container path ('-' for stdin or filename)")
 	log.CheckErr("unable to mark 'dst' flag as required.", cmd.MarkFlagRequired("dst"))
-	cmd.Flags().BoolVar(&generatePatch, "patch", false, "Output as a bundle patch")
+	cmd.Flags().BoolVar(&params.generatePatch, "patch", false, "Output as a bundle patch")
 
 	return cmd
 }

cmd/harp/internal/cmd/bundle_dump.go

@@ -27,15 +27,17 @@ import (
 )
 
 // -----------------------------------------------------------------------------
+type bundleDumpParams struct {
+	inputPath      string
+	dataOnly       bool
+	metadataOnly   bool
+	pathOnly       bool
+	jmesPathFilter string
+	skipTemplate   bool
+}
 
 var bundleDumpCmd = func() *cobra.Command {
-	var (
-		inputPath      string
-		dataOnly       bool
-		metadataOnly   bool
-		pathOnly       bool
-		jmesPathFilter string
-	)
+	params := &bundleDumpParams{}
 
 	cmd := &cobra.Command{
 		Use:   "dump",
@@ -47,12 +49,13 @@ var bundleDumpCmd = func() *cobra.Command {
 
 			// Prepare task
 			t := &bundle.DumpTask{
-				ContainerReader: cmdutil.FileReader(inputPath),
+				ContainerReader: cmdutil.FileReader(params.inputPath),
 				OutputWriter:    cmdutil.StdoutWriter(),
-				DataOnly:        dataOnly,
-				MetadataOnly:    metadataOnly,
-				PathOnly:        pathOnly,
-				JMESPathFilter:  jmesPathFilter,
+				DataOnly:        params.dataOnly,
+				MetadataOnly:    params.metadataOnly,
+				PathOnly:        params.pathOnly,
+				JMESPathFilter:  params.jmesPathFilter,
+				IgnoreTemplate:  params.skipTemplate,
 			}
 
 			// Run the task
@@ -63,12 +66,13 @@ var bundleDumpCmd = func() *cobra.Command {
 	}
 
 	// Parameters
-	cmd.Flags().StringVar(&inputPath, "in", "", "Container input ('-' for stdin or filename)")
-	cmd.Flags().BoolVar(&dataOnly, "content-only", false, "Display content only (data-only alias)")
-	cmd.Flags().BoolVar(&dataOnly, "data-only", false, "Display data only")
-	cmd.Flags().BoolVar(&metadataOnly, "metadata-only", false, "Display metadata only")
-	cmd.Flags().BoolVar(&pathOnly, "path-only", false, "Display path only")
-	cmd.Flags().StringVar(&jmesPathFilter, "jmespath", "", "Specify a JMESPath query to format output")
+	cmd.Flags().StringVar(&params.inputPath, "in", "", "Container input ('-' for stdin or filename)")
+	cmd.Flags().BoolVar(&params.dataOnly, "content-only", false, "Display content only (data-only alias)")
+	cmd.Flags().BoolVar(&params.dataOnly, "data-only", false, "Display data only")
+	cmd.Flags().BoolVar(&params.metadataOnly, "metadata-only", false, "Display metadata only")
+	cmd.Flags().BoolVar(&params.pathOnly, "path-only", false, "Display path only")
+	cmd.Flags().StringVar(&params.jmesPathFilter, "query", "", "Specify a JMESPath query to format output")
+	cmd.Flags().BoolVar(&params.skipTemplate, "skip-template", false, "Drop template from dump")
 
 	return cmd
 }