Skip to content

Commit

Permalink
Restructure to allow import of protobuf API with minimal dependencies
Browse files Browse the repository at this point in the history
and also for selective import of storage backends - massively reducing
input tree for dependees.

Signed-off-by: Silas Davis <[email protected]>
  • Loading branch information
Silas Davis committed May 24, 2019
1 parent ae90c85 commit a50cde4
Show file tree
Hide file tree
Showing 57 changed files with 548 additions and 546 deletions.
1 change: 1 addition & 0 deletions .circleci/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ RUN rm -rf protobuf protoc-*
RUN go get -u golang.org/x/tools/cmd/goimports
RUN go get -u github.com/golang/protobuf/protoc-gen-go
RUN go get -u golang.org/x/lint/golint
RUN go get -u github.com/goreleaser/goreleaser
ENV GO111MODULE=on

# install aws auth binaries
Expand Down
2 changes: 1 addition & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

## [4.0.0] - 2019-05-21
###Fixed
- [BUILD] Change hoard.pb.go to services/services.pb.go
- [BUILD] Change hoard.pb.go to services/api.pb.go


## [3.2.1] - 2019-04-24
Expand Down
2 changes: 1 addition & 1 deletion NOTES.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
###Fixed
- [BUILD] Change hoard.pb.go to services/services.pb.go
- [BUILD] Change hoard.pb.go to services/api.pb.go

181 changes: 90 additions & 91 deletions services/services.pb.go → api/api.pb.go

Large diffs are not rendered by default.

10 changes: 5 additions & 5 deletions cmd/hoarctl/encryption.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ import (
"os"

cli "github.com/jawher/mow.cli"
"github.com/monax/hoard/v4/api"
"github.com/monax/hoard/v4/reference"
"github.com/monax/hoard/v4/services"
)

// Decrypt does what it says on the tin
Expand All @@ -19,12 +19,12 @@ func (client *Client) Decrypt(cmd *cli.Cmd) {
cmd.Action = func() {
encryptedData := readData()
plaintext, err := client.encryption.Decrypt(context.Background(),
&services.ReferenceAndCiphertext{
&api.ReferenceAndCiphertext{
Reference: &reference.Ref{
SecretKey: readBase64(secretKey),
Salt: parseSalt(salt),
},
Ciphertext: &services.Ciphertext{
Ciphertext: &api.Ciphertext{
EncryptedData: encryptedData,
},
})
Expand All @@ -45,7 +45,7 @@ func (client *Client) Encrypt(cmd *cli.Cmd) {
fatalf("could read bytes from STDIN to store: %v", err)
}
refAndCiphertext, err := client.encryption.Encrypt(context.Background(),
&services.Plaintext{
&api.Plaintext{
Data: data,
Salt: parseSalt(salt),
})
Expand All @@ -63,7 +63,7 @@ func (client *Client) Ref(cmd *cli.Cmd) {
cmd.Action = func() {
data := readData()
refAndCiphertext, err := client.encryption.Encrypt(context.Background(),
&services.Plaintext{
&api.Plaintext{
Data: data,
Salt: parseSalt(salt),
})
Expand Down
10 changes: 5 additions & 5 deletions cmd/hoarctl/grants.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ import (
"os"

cli "github.com/jawher/mow.cli"
"github.com/monax/hoard/v4/api"
"github.com/monax/hoard/v4/grant"
"github.com/monax/hoard/v4/services"
)

// PutSeal encrypts and stores data then prints a grant
Expand All @@ -29,8 +29,8 @@ func (client *Client) PutSeal(cmd *cli.Cmd) {

data := readData()
seal, err = client.grant.PutSeal(context.Background(),
&services.PlaintextAndGrantSpec{
Plaintext: &services.Plaintext{
&api.PlaintextAndGrantSpec{
Plaintext: &api.Plaintext{
Data: data,
Salt: parseSalt(salt),
},
Expand Down Expand Up @@ -61,7 +61,7 @@ func (client *Client) Seal(cmd *cli.Cmd) {

ref := readReference(address)
seal, err := client.grant.Seal(context.Background(),
&services.ReferenceAndGrantSpec{
&api.ReferenceAndGrantSpec{
Reference: ref,
GrantSpec: &spec,
},
Expand Down Expand Up @@ -90,7 +90,7 @@ func (client *Client) Reseal(cmd *cli.Cmd) {
}

ref, err := client.grant.Reseal(context.Background(),
&services.GrantAndGrantSpec{
&api.GrantAndGrantSpec{
Grant: prev,
GrantSpec: &next,
})
Expand Down
19 changes: 10 additions & 9 deletions cmd/hoarctl/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,14 @@ import (
"os"
"time"

"github.com/monax/hoard/v4/api"

cli "github.com/jawher/mow.cli"
"github.com/monax/hoard/v4/cmd"
"github.com/monax/hoard/v4/config"
"github.com/monax/hoard/v4/grant"
"github.com/monax/hoard/v4/reference"
"github.com/monax/hoard/v4/server"
"github.com/monax/hoard/v4/services"
"google.golang.org/grpc"
)

Expand All @@ -31,10 +32,10 @@ const (

// Client scopes the available hoard clients
type Client struct {
cleartext services.CleartextClient
encryption services.EncryptionClient
grant services.GrantClient
storage services.StorageClient
cleartext api.CleartextClient
encryption api.EncryptionClient
grant api.GrantClient
storage api.StorageClient
}

func main() {
Expand Down Expand Up @@ -62,10 +63,10 @@ func main() {
if err != nil {
fatalf("Could not dial hoard server on %s: %v", *dialURL, err)
}
client.cleartext = services.NewCleartextClient(conn)
client.encryption = services.NewEncryptionClient(conn)
client.grant = services.NewGrantClient(conn)
client.storage = services.NewStorageClient(conn)
client.cleartext = api.NewCleartextClient(conn)
client.encryption = api.NewEncryptionClient(conn)
client.grant = api.NewGrantClient(conn)
client.storage = api.NewStorageClient(conn)
}

cmd.AddVersionCommand(hoarctlApp)
Expand Down
10 changes: 5 additions & 5 deletions cmd/hoarctl/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ import (
"os"

cli "github.com/jawher/mow.cli"
"github.com/monax/hoard/v4/api"
"github.com/monax/hoard/v4/reference"
"github.com/monax/hoard/v4/services"
)

// Cat retrieves encrypted data from store
Expand All @@ -17,7 +17,7 @@ func (client *Client) Cat(cmd *cli.Cmd) {
cmd.Action = func() {
ref := readReference(address)
ciphertext, err := client.storage.Pull(context.Background(),
&services.Address{Address: ref.Address})
&api.Address{Address: ref.Address})
if err != nil {
fatalf("Error querying data: %v", err)
}
Expand Down Expand Up @@ -58,7 +58,7 @@ func (client *Client) Insert(cmd *cli.Cmd) {
data := readData()
// If given address use it
address, err := client.storage.Push(context.Background(),
&services.Ciphertext{EncryptedData: data})
&api.Ciphertext{EncryptedData: data})
if err != nil {
fatalf("Error querying data: %v", err)
}
Expand All @@ -73,7 +73,7 @@ func (client *Client) Put(cmd *cli.Cmd) {
cmd.Action = func() {
data := readData()
ref, err := client.cleartext.Put(context.Background(),
&services.Plaintext{
&api.Plaintext{
Data: data,
Salt: parseSalt(salt),
})
Expand All @@ -91,7 +91,7 @@ func (client *Client) Stat(cmd *cli.Cmd) {
cmd.Action = func() {
ref := readReference(address)
statInfo, err := client.storage.Stat(context.Background(),
&services.Address{Address: ref.Address})
&api.Address{Address: ref.Address})
if err != nil {
fatalf("Error querying data: %v", err)
}
Expand Down
40 changes: 18 additions & 22 deletions cmd/hoard/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,6 @@ import (
cli "github.com/jawher/mow.cli"
"github.com/monax/hoard/v4/cmd"
"github.com/monax/hoard/v4/config"
"github.com/monax/hoard/v4/config/logging"
"github.com/monax/hoard/v4/config/secrets"
"github.com/monax/hoard/v4/config/source"
"github.com/monax/hoard/v4/config/storage"
"github.com/monax/hoard/v4/server"
)

Expand All @@ -37,7 +33,7 @@ func main() {

environmentOpt := hoardApp.BoolOpt("e env", false,
fmt.Sprintf("Parse the contents of the environment variable %s as a complete JSON config",
source.DefaultJSONConfigEnvironmentVariable))
config.DefaultJSONConfigEnvironmentVariable))

// This string spec is parsed by mow.cli and has actual semantic significance
// around optionality and ordering of options and arguments
Expand All @@ -54,22 +50,22 @@ func main() {
var logger log.Logger

if *loggingOpt {
logger, err = logging.LoggerFromLoggingConfig(conf.Logging, os.Stderr)
logger, err = config.Logger(conf.Logging, os.Stderr)
if err != nil {
fatalf("Could not create logging form logging config: %s", err)
}
}

store, err := storage.StoreFromStorageConfig(conf.Storage, logger)
store, err := StoreFromStorageConfig(conf.Storage, logger)
if err != nil {
fatalf("Could not configure store from storage config: %s", err)
}
if *listenAddressOpt != "" {
conf.ListenAddress = *listenAddressOpt
}
symmetricProvider := secrets.ProviderFromConfig(conf.Secrets)
openPGPConf := secrets.OpenPGPFromConfig(conf.Secrets)
sm := secrets.Manager{Provider: symmetricProvider, OpenPGP: openPGPConf}
symmetricProvider := config.NewSymmetricProvider(conf.Secrets)
openPGPConf := config.NewOpenPGPSecret(conf.Secrets)
sm := config.SecretsManager{Provider: symmetricProvider, OpenPGP: openPGPConf}

serv := server.New(conf.ListenAddress, store, sm, logger)
// Catch interrupt etc
Expand Down Expand Up @@ -114,20 +110,20 @@ func main() {

secretsOpt := configCmd.StringsOpt("s secret", nil, "Pairs of PublicID and Passphrase to use as symmetric secrets in config")

arg := configCmd.StringArg("CONFIG", "", fmt.Sprintf("Config type to generate, one of: %s",
arg := configCmd.StringArg("CONFIG", "", fmt.Sprintf("Storage type to generate, one of: %s",
strings.Join(configTypes(), ", ")))

configCmd.Spec = "[--json | --yaml] | (([--output=<output file>] | [--init]) [--force]) CONFIG [--secret=<PublicID:Passphrase>...]"

configCmd.Action = func() {
store, err := storage.GetDefaultConfig(*arg)
store, err := config.GetDefaultStorage(*arg)
if err != nil {
fatalf("Error fetching default config for %v: %v", arg, err)
}
conf.Storage = store
if len(*secretsOpt) > 0 {
conf.Secrets = &secrets.SecretsConfig{
Symmetric: make([]secrets.SymmetricSecret, len(*secretsOpt)),
conf.Secrets = &config.Secrets{
Symmetric: make([]config.SymmetricSecret, len(*secretsOpt)),
}
for i, ss := range *secretsOpt {
pair := strings.Split(ss, ":")
Expand All @@ -149,7 +145,7 @@ func main() {
}
if *initOpt {
configFileName, err := xdgbasedir.GetConfigFileLocation(
source.DefaultHoardConfigFileName)
config.DefaultHoardConfigFileName)
if err != nil {
fatalf("Error getting config file location: %s", err)
}
Expand Down Expand Up @@ -180,12 +176,12 @@ func fatalf(format string, args ...interface{}) {
os.Exit(1)
}

func hoardConfigCascade(env bool, configFile string) source.ConfigProvider {
return source.Cascade(os.Stderr, true,
source.Environment(source.DefaultJSONConfigEnvironmentVariable).SetSkip(!env),
source.File(configFile).SetSkip(configFile == ""),
source.XDGBaseDir(),
source.Default())
func hoardConfigCascade(env bool, configFile string) config.Provider {
return config.Cascade(os.Stderr, true,
config.Environment(config.DefaultJSONConfigEnvironmentVariable).SetSkip(!env),
config.File(configFile).SetSkip(configFile == ""),
config.XDGBaseDir(),
config.Default())
}

func writeFile(filename string, data []byte, overwrite bool) error {
Expand All @@ -196,7 +192,7 @@ func writeFile(filename string, data []byte, overwrite bool) error {
}

func configTypes() []string {
storageTypes := storage.GetStorageTypes()
storageTypes := config.GetStorageTypes()
configTypes := make([]string, len(storageTypes))
for i, st := range storageTypes {
configTypes[i] = string(st)
Expand Down
73 changes: 73 additions & 0 deletions cmd/hoard/stores.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package main

import (
"errors"
"fmt"

"github.com/go-kit/kit/log"
"github.com/monax/hoard/v4/config"
"github.com/monax/hoard/v4/stores"
"github.com/monax/hoard/v4/stores/cloud"
"github.com/monax/hoard/v4/stores/ipfs"
)

func StoreFromStorageConfig(storageConfig *config.Storage, logger log.Logger) (stores.NamedStore, error) {
addressEncoding, err := stores.GetAddressEncoding(storageConfig.AddressEncoding)
if err != nil {
return nil, err
}

switch storageConfig.StorageType {
case config.Memory, config.Unspecified:
return stores.NewMemoryStore(), nil

case config.Filesystem:
fsConf := storageConfig.FileSystemConfig
if fsConf == nil {
return nil, errors.New("filesystem storage configuration must be " +
"supplied to use the filesystem storage backend")
}
if fsConf.RootDirectory == "" {
return nil, errors.New("rootDirectory key must be non-empty in " + "filesystem storage")

}
return stores.NewFileSystemStore(fsConf.RootDirectory, addressEncoding)

case config.IPFS:
ipfsConf := storageConfig.IPFSConfig
if ipfsConf == nil {
return nil, errors.New("IPFS storage configuration must be " +
"supplied to use the filesystem storage backend")
}
if ipfsConf.RemoteAPI == "" {
return nil, errors.New("http api url must be non-empty in " +
"ipfs storage config")
}
return ipfs.NewStore(ipfsConf.RemoteAPI, addressEncoding)

case config.AWS:
awsConf := storageConfig.Cloud
if awsConf == nil {
return nil, errors.New("aws configuration must be supplied")
}
return cloud.NewStore(cloud.AWS, awsConf.Bucket, awsConf.Prefix, awsConf.Region, addressEncoding, logger)

case config.Azure:
azureConf := storageConfig.Cloud
if azureConf == nil {
return nil, errors.New("azure configuration must be supplied")
}
return cloud.NewStore(cloud.Azure, azureConf.Bucket, azureConf.Prefix, azureConf.Region, addressEncoding, logger)

case config.GCP:
gcpConf := storageConfig.Cloud
if gcpConf == nil {
return nil, errors.New("gcp configuration must be supplied")
}
return cloud.NewStore(cloud.GCP, gcpConf.Bucket, gcpConf.Prefix, gcpConf.Region, addressEncoding, logger)

default:
return nil, fmt.Errorf("did not recognise storage type '%s'",
storageConfig.StorageType)
}
}
Loading

0 comments on commit a50cde4

Please sign in to comment.