diff --git a/.circleci/Dockerfile b/.circleci/Dockerfile index 054ce90c..638bda56 100644 --- a/.circleci/Dockerfile +++ b/.circleci/Dockerfile @@ -33,6 +33,7 @@ RUN rm -rf protobuf protoc-* RUN go get -u golang.org/x/tools/cmd/goimports RUN go get -u github.com/golang/protobuf/protoc-gen-go RUN go get -u golang.org/x/lint/golint +RUN go get -u github.com/goreleaser/goreleaser ENV GO111MODULE=on # install aws auth binaries diff --git a/CHANGELOG.md b/CHANGELOG.md index 08d814b7..a1b4721d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ## [4.0.0] - 2019-05-21 ###Fixed -- [BUILD] Change hoard.pb.go to services/services.pb.go +- [BUILD] Change hoard.pb.go to services/api.pb.go ## [3.2.1] - 2019-04-24 diff --git a/NOTES.md b/NOTES.md index 8e3ba860..2fb2a133 100644 --- a/NOTES.md +++ b/NOTES.md @@ -1,3 +1,3 @@ ###Fixed -- [BUILD] Change hoard.pb.go to services/services.pb.go +- [BUILD] Change hoard.pb.go to services/api.pb.go diff --git a/services/services.pb.go b/api/api.pb.go similarity index 82% rename from services/services.pb.go rename to api/api.pb.go index 909a3942..9c8ce689 100644 --- a/services/services.pb.go +++ b/api/api.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: services.proto +// source: api.proto -package services +package api import ( context "context" @@ -9,7 +9,7 @@ import ( proto "github.com/gogo/protobuf/proto" grant "github.com/monax/hoard/v4/grant" reference "github.com/monax/hoard/v4/reference" - storage "github.com/monax/hoard/v4/storage" + stores "github.com/monax/hoard/v4/stores" grpc "google.golang.org/grpc" math "math" ) @@ -38,7 +38,7 @@ func (m *GrantAndGrantSpec) Reset() { *m = GrantAndGrantSpec{} } func (m *GrantAndGrantSpec) String() string { return proto.CompactTextString(m) } func (*GrantAndGrantSpec) ProtoMessage() {} func (*GrantAndGrantSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_8e16ccb8c5307b32, []int{0} + return fileDescriptor_00212fb1f9d3bf1c, []int{0} } func (m *GrantAndGrantSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GrantAndGrantSpec.Unmarshal(m, b) @@ -85,7 +85,7 @@ func (m *PlaintextAndGrantSpec) Reset() { *m = PlaintextAndGrantSpec{} } func (m *PlaintextAndGrantSpec) String() string { return proto.CompactTextString(m) } func (*PlaintextAndGrantSpec) ProtoMessage() {} func (*PlaintextAndGrantSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_8e16ccb8c5307b32, []int{1} + return fileDescriptor_00212fb1f9d3bf1c, []int{1} } func (m *PlaintextAndGrantSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PlaintextAndGrantSpec.Unmarshal(m, b) @@ -132,7 +132,7 @@ func (m *ReferenceAndGrantSpec) Reset() { *m = ReferenceAndGrantSpec{} } func (m *ReferenceAndGrantSpec) String() string { return proto.CompactTextString(m) } func (*ReferenceAndGrantSpec) ProtoMessage() {} func (*ReferenceAndGrantSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_8e16ccb8c5307b32, []int{2} + return fileDescriptor_00212fb1f9d3bf1c, []int{2} } func (m *ReferenceAndGrantSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ReferenceAndGrantSpec.Unmarshal(m, b) @@ -178,7 +178,7 @@ func (m *Plaintext) Reset() { *m = Plaintext{} } func (m *Plaintext) String() string { return proto.CompactTextString(m) } func (*Plaintext) ProtoMessage() {} func (*Plaintext) Descriptor() ([]byte, []int) { - return fileDescriptor_8e16ccb8c5307b32, []int{3} + return fileDescriptor_00212fb1f9d3bf1c, []int{3} } func (m *Plaintext) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Plaintext.Unmarshal(m, b) @@ -223,7 +223,7 @@ func (m *Ciphertext) Reset() { *m = Ciphertext{} } func (m *Ciphertext) String() string { return proto.CompactTextString(m) } func (*Ciphertext) ProtoMessage() {} func (*Ciphertext) Descriptor() ([]byte, []int) { - return fileDescriptor_8e16ccb8c5307b32, []int{4} + return fileDescriptor_00212fb1f9d3bf1c, []int{4} } func (m *Ciphertext) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Ciphertext.Unmarshal(m, b) @@ -262,7 +262,7 @@ func (m *ReferenceAndCiphertext) Reset() { *m = ReferenceAndCiphertext{} func (m *ReferenceAndCiphertext) String() string { return proto.CompactTextString(m) } func (*ReferenceAndCiphertext) ProtoMessage() {} func (*ReferenceAndCiphertext) Descriptor() ([]byte, []int) { - return fileDescriptor_8e16ccb8c5307b32, []int{5} + return fileDescriptor_00212fb1f9d3bf1c, []int{5} } func (m *ReferenceAndCiphertext) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ReferenceAndCiphertext.Unmarshal(m, b) @@ -307,7 +307,7 @@ func (m *Address) Reset() { *m = Address{} } func (m *Address) String() string { return proto.CompactTextString(m) } func (*Address) ProtoMessage() {} func (*Address) Descriptor() ([]byte, []int) { - return fileDescriptor_8e16ccb8c5307b32, []int{6} + return fileDescriptor_00212fb1f9d3bf1c, []int{6} } func (m *Address) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Address.Unmarshal(m, b) @@ -335,51 +335,50 @@ func (m *Address) GetAddress() []byte { } func init() { - proto.RegisterType((*GrantAndGrantSpec)(nil), "services.GrantAndGrantSpec") - proto.RegisterType((*PlaintextAndGrantSpec)(nil), "services.PlaintextAndGrantSpec") - proto.RegisterType((*ReferenceAndGrantSpec)(nil), "services.ReferenceAndGrantSpec") - proto.RegisterType((*Plaintext)(nil), "services.Plaintext") - proto.RegisterType((*Ciphertext)(nil), "services.Ciphertext") - proto.RegisterType((*ReferenceAndCiphertext)(nil), "services.ReferenceAndCiphertext") - proto.RegisterType((*Address)(nil), "services.Address") -} - -func init() { proto.RegisterFile("services.proto", fileDescriptor_8e16ccb8c5307b32) } - -var fileDescriptor_8e16ccb8c5307b32 = []byte{ - // 501 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0x41, 0x6b, 0xdb, 0x30, - 0x14, 0x26, 0x6d, 0x16, 0xcf, 0x2f, 0x69, 0x47, 0xb4, 0x75, 0x04, 0xef, 0xb0, 0xe2, 0x95, 0xb1, - 0x95, 0x62, 0xb3, 0xa4, 0x83, 0x1d, 0xdb, 0xb5, 0xa3, 0xec, 0x16, 0x64, 0x76, 0xd9, 0x4d, 0xb1, - 0x95, 0xc4, 0xe0, 0xca, 0x46, 0x96, 0x4b, 0x07, 0xbb, 0xee, 0xbe, 0xeb, 0xfe, 0xed, 0xb0, 0x2c, - 0x5b, 0xb6, 0x23, 0x28, 0x39, 0x59, 0xef, 0x7d, 0xdf, 0x7b, 0xdf, 0x7b, 0xd6, 0x87, 0xe0, 0x38, - 0xa7, 0xfc, 0x21, 0x0e, 0x69, 0xee, 0x65, 0x3c, 0x15, 0x29, 0x7a, 0x5e, 0xc7, 0xce, 0x0b, 0x4e, - 0xd7, 0x94, 0x53, 0x16, 0xd2, 0x0a, 0x72, 0xc6, 0x1b, 0x4e, 0x98, 0x50, 0xc1, 0x51, 0x2e, 0x52, - 0x4e, 0x36, 0x0a, 0x73, 0x57, 0x30, 0xbd, 0x2b, 0xd1, 0x6b, 0x16, 0xc9, 0x6f, 0x90, 0xd1, 0x10, - 0xb9, 0xf0, 0x4c, 0x06, 0xb3, 0xc1, 0xe9, 0xe0, 0xc3, 0x78, 0x3e, 0xf1, 0xaa, 0x06, 0x32, 0x87, - 0x2b, 0x08, 0x7d, 0x04, 0xbb, 0x29, 0x98, 0x1d, 0x48, 0xde, 0x58, 0xf1, 0xca, 0x14, 0xd6, 0xa8, - 0x5b, 0xc0, 0xc9, 0x32, 0x21, 0x31, 0x13, 0xf4, 0xb1, 0xab, 0xf3, 0x09, 0xec, 0x06, 0x50, 0x5a, - 0x2f, 0xbd, 0x66, 0xaf, 0x06, 0xc2, 0x9a, 0xb5, 0x8f, 0x6c, 0x06, 0x27, 0xb8, 0xfe, 0x13, 0x1d, - 0xd9, 0x0b, 0xb0, 0x1b, 0x40, 0xc9, 0x1e, 0x7b, 0xfa, 0xa7, 0x61, 0xba, 0xc6, 0x9a, 0xb0, 0x8f, - 0xe2, 0xa2, 0xb5, 0x0f, 0x42, 0x30, 0xbc, 0x25, 0x82, 0x48, 0x81, 0x09, 0x96, 0xe7, 0x32, 0x17, - 0x90, 0x44, 0xc8, 0x36, 0x13, 0x2c, 0xcf, 0xee, 0x1c, 0xe0, 0x26, 0xce, 0xb6, 0x94, 0xcb, 0xaa, - 0x33, 0x38, 0xfa, 0xc6, 0x42, 0xfe, 0x2b, 0x13, 0x34, 0x6a, 0x95, 0x77, 0x93, 0xee, 0x6f, 0x78, - 0xdd, 0x5e, 0xad, 0x55, 0xbf, 0xdf, 0x6e, 0x97, 0x6d, 0x6d, 0xb5, 0xdc, 0x2b, 0x7d, 0x03, 0x1a, - 0xc3, 0x2d, 0x9e, 0xfb, 0x0e, 0xac, 0xeb, 0x28, 0xe2, 0x34, 0xcf, 0xd1, 0xac, 0x39, 0xaa, 0x41, - 0xeb, 0x70, 0xfe, 0xe7, 0x40, 0x99, 0x08, 0x7d, 0x86, 0x61, 0x40, 0x49, 0x82, 0xde, 0xea, 0xc6, - 0xc6, 0x7b, 0x71, 0x3a, 0x3e, 0x43, 0xef, 0x61, 0xf4, 0x83, 0xe5, 0x65, 0x61, 0x27, 0xef, 0xf4, - 0xd6, 0x41, 0x0b, 0x18, 0x61, 0x2a, 0x79, 0x6f, 0xb4, 0xc0, 0x8e, 0xa7, 0x7b, 0xcd, 0xbf, 0x80, - 0xb5, 0x2c, 0x44, 0x7f, 0x2c, 0xa3, 0x4b, 0x7b, 0x95, 0x1e, 0xd8, 0xd5, 0x58, 0x77, 0x54, 0xf4, - 0x26, 0x33, 0x79, 0x77, 0x1e, 0x82, 0x7d, 0x93, 0x50, 0x52, 0xdd, 0xce, 0x39, 0x1c, 0x2e, 0x0b, - 0x81, 0x4c, 0xc4, 0x9d, 0xbd, 0xce, 0xe1, 0xb0, 0x94, 0xe8, 0xa5, 0xcd, 0x22, 0x7f, 0x07, 0x00, - 0xca, 0x21, 0x71, 0xca, 0xd0, 0x15, 0x58, 0x2a, 0x32, 0x4b, 0x9d, 0x9a, 0x6f, 0xa2, 0x65, 0xa3, - 0x2b, 0xb0, 0x6e, 0x69, 0xd5, 0xe1, 0x49, 0xb2, 0x79, 0xa4, 0x7f, 0x03, 0xb0, 0x82, 0xea, 0xa9, - 0x41, 0x3e, 0x0c, 0x97, 0x45, 0xbe, 0x45, 0x46, 0x6b, 0x39, 0x53, 0x9d, 0xad, 0x6d, 0x25, 0x0b, - 0x92, 0x04, 0xed, 0x42, 0x8e, 0xb1, 0x07, 0xba, 0x80, 0x61, 0x20, 0x88, 0x30, 0x15, 0x4c, 0xbd, - 0xfa, 0xc5, 0x2b, 0x19, 0xdf, 0xd9, 0x3a, 0xfd, 0x7a, 0xf6, 0xd3, 0xdd, 0xc4, 0x62, 0x5b, 0xac, - 0xbc, 0x30, 0xbd, 0xf7, 0xef, 0x53, 0x46, 0x1e, 0xfd, 0x6d, 0x4a, 0x78, 0xe4, 0x3f, 0x5c, 0xfa, - 0x75, 0x83, 0xd5, 0x48, 0xbe, 0x90, 0x8b, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0xac, 0x80, 0xc4, - 0xc0, 0x6a, 0x05, 0x00, 0x00, + proto.RegisterType((*GrantAndGrantSpec)(nil), "api.GrantAndGrantSpec") + proto.RegisterType((*PlaintextAndGrantSpec)(nil), "api.PlaintextAndGrantSpec") + proto.RegisterType((*ReferenceAndGrantSpec)(nil), "api.ReferenceAndGrantSpec") + proto.RegisterType((*Plaintext)(nil), "api.Plaintext") + proto.RegisterType((*Ciphertext)(nil), "api.Ciphertext") + proto.RegisterType((*ReferenceAndCiphertext)(nil), "api.ReferenceAndCiphertext") + proto.RegisterType((*Address)(nil), "api.Address") +} + +func init() { proto.RegisterFile("api.proto", fileDescriptor_00212fb1f9d3bf1c) } + +var fileDescriptor_00212fb1f9d3bf1c = []byte{ + // 486 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0x51, 0x8b, 0xd3, 0x40, + 0x10, 0xa6, 0xb6, 0x36, 0x64, 0x1a, 0x3d, 0x5d, 0xf0, 0x28, 0x11, 0x51, 0xa2, 0x3d, 0x3c, 0x90, + 0x44, 0x5a, 0xb9, 0xf7, 0xf3, 0x4e, 0x0e, 0xdf, 0xca, 0x06, 0x1f, 0xf4, 0x6d, 0xdb, 0x4c, 0xdb, + 0x40, 0x2e, 0x1b, 0x36, 0x5b, 0x3d, 0x41, 0xf0, 0x37, 0xfb, 0x0f, 0x24, 0x9b, 0x6d, 0xb2, 0xd9, + 0x1c, 0x07, 0x7d, 0xca, 0xce, 0x37, 0xdf, 0x37, 0xdf, 0xce, 0xce, 0x10, 0x70, 0x59, 0x91, 0x86, + 0x85, 0xe0, 0x92, 0x93, 0x21, 0x2b, 0x52, 0xff, 0x44, 0xe0, 0x06, 0x05, 0xe6, 0x6b, 0xac, 0x51, + 0x7f, 0xb2, 0x15, 0x2c, 0x97, 0x3a, 0xf0, 0x4a, 0xc9, 0x05, 0x96, 0x75, 0x14, 0xac, 0xe0, 0xf9, + 0x4d, 0x95, 0xbc, 0xcc, 0x13, 0xf5, 0x8d, 0x0b, 0x5c, 0x93, 0x00, 0x1e, 0xab, 0x60, 0x3a, 0x78, + 0x33, 0x78, 0x3f, 0x99, 0x7b, 0x61, 0xad, 0x57, 0x18, 0xad, 0x53, 0xe4, 0x1c, 0xdc, 0x46, 0x30, + 0x7d, 0xa4, 0x78, 0x13, 0xcd, 0xab, 0x20, 0xda, 0x66, 0x83, 0x02, 0x5e, 0x2c, 0x33, 0x96, 0xe6, + 0x12, 0xef, 0xba, 0x3e, 0x1f, 0xc0, 0x6d, 0x12, 0xda, 0xeb, 0x69, 0x58, 0x35, 0xd3, 0xa0, 0xb4, + 0x25, 0x1c, 0xe9, 0x48, 0x0f, 0x6f, 0x60, 0x3b, 0x36, 0x89, 0xc6, 0xb1, 0x7d, 0x2e, 0x8a, 0x1b, + 0xda, 0x12, 0x8e, 0x71, 0x5c, 0x18, 0xad, 0x10, 0x02, 0xa3, 0x6b, 0x26, 0x99, 0x32, 0xf0, 0xa8, + 0x3a, 0x57, 0x58, 0xcc, 0x32, 0xa9, 0xca, 0x78, 0x54, 0x9d, 0x83, 0x39, 0xc0, 0x55, 0x5a, 0xec, + 0x50, 0x28, 0xd5, 0x3b, 0x78, 0xf2, 0x25, 0x5f, 0x8b, 0xdf, 0x85, 0xc4, 0xc4, 0x90, 0x77, 0xc1, + 0xe0, 0x17, 0x9c, 0x9a, 0xad, 0x19, 0xfa, 0xe3, 0x7a, 0x8b, 0x4c, 0x6f, 0xdd, 0xdc, 0x89, 0x7a, + 0xfc, 0x16, 0xa6, 0x06, 0x25, 0x78, 0x0b, 0xce, 0x65, 0x92, 0x08, 0x2c, 0x4b, 0x32, 0x6d, 0x8e, + 0xfa, 0x8e, 0x87, 0x70, 0xfe, 0x6f, 0xa0, 0x57, 0x87, 0x7c, 0x84, 0x51, 0x8c, 0x2c, 0x23, 0xbe, + 0xaa, 0x79, 0xef, 0x34, 0xfc, 0xce, 0x62, 0x91, 0x33, 0x18, 0x7f, 0xcb, 0xcb, 0x4a, 0xd3, 0xc1, + 0x7d, 0xab, 0x09, 0x12, 0xc2, 0x98, 0xa2, 0xe2, 0x9d, 0xaa, 0xda, 0xbd, 0xfd, 0xb5, 0xea, 0x2e, + 0xc0, 0x59, 0xee, 0xa5, 0x71, 0x99, 0x7b, 0x97, 0xd1, 0x12, 0x9d, 0x83, 0x5b, 0x5f, 0xe6, 0x06, + 0x65, 0xef, 0x3e, 0x9d, 0x22, 0xf3, 0xef, 0xe0, 0x5e, 0x65, 0xc8, 0xea, 0x21, 0xcc, 0x60, 0xb8, + 0xdc, 0x4b, 0x62, 0x71, 0x7a, 0x3d, 0xcc, 0x60, 0x58, 0x15, 0xb6, 0xe0, 0x5e, 0xe9, 0x3f, 0x00, + 0x7a, 0xfa, 0x29, 0xcf, 0xc9, 0x05, 0x38, 0x3a, 0xea, 0xd5, 0x7f, 0xd9, 0x7b, 0x65, 0x63, 0x31, + 0x2e, 0xc0, 0xb9, 0xc6, 0x5a, 0xf7, 0x10, 0xaf, 0xe7, 0xfe, 0x17, 0x9c, 0x58, 0x72, 0xc1, 0xb6, + 0x48, 0x66, 0x30, 0x5a, 0xee, 0xcb, 0x1d, 0xb1, 0x37, 0xc4, 0xf7, 0x14, 0x70, 0x58, 0x0c, 0x45, + 0xcb, 0xaa, 0x01, 0x1a, 0xa8, 0x6f, 0x8b, 0xc8, 0x19, 0x8c, 0x62, 0xc9, 0xa4, 0x45, 0x7b, 0x16, + 0xea, 0x3f, 0x53, 0x95, 0xfb, 0x9a, 0x6f, 0xf8, 0xe7, 0xd7, 0x3f, 0x5e, 0x6d, 0x53, 0xb9, 0xdb, + 0xaf, 0xc2, 0x35, 0xbf, 0x8d, 0x6e, 0x79, 0xce, 0xee, 0xa2, 0x1d, 0x67, 0x22, 0x89, 0x7e, 0x7e, + 0x8a, 0x58, 0x91, 0xae, 0xc6, 0xea, 0x27, 0xb6, 0xf8, 0x1f, 0x00, 0x00, 0xff, 0xff, 0x9f, 0x0f, + 0x13, 0x6f, 0x02, 0x05, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -417,7 +416,7 @@ func NewGrantClient(cc *grpc.ClientConn) GrantClient { func (c *grantClient) Seal(ctx context.Context, in *ReferenceAndGrantSpec, opts ...grpc.CallOption) (*grant.Grant, error) { out := new(grant.Grant) - err := c.cc.Invoke(ctx, "/services.Grant/Seal", in, out, opts...) + err := c.cc.Invoke(ctx, "/api.Grant/Seal", in, out, opts...) if err != nil { return nil, err } @@ -426,7 +425,7 @@ func (c *grantClient) Seal(ctx context.Context, in *ReferenceAndGrantSpec, opts func (c *grantClient) Unseal(ctx context.Context, in *grant.Grant, opts ...grpc.CallOption) (*reference.Ref, error) { out := new(reference.Ref) - err := c.cc.Invoke(ctx, "/services.Grant/Unseal", in, out, opts...) + err := c.cc.Invoke(ctx, "/api.Grant/Unseal", in, out, opts...) if err != nil { return nil, err } @@ -435,7 +434,7 @@ func (c *grantClient) Unseal(ctx context.Context, in *grant.Grant, opts ...grpc. func (c *grantClient) Reseal(ctx context.Context, in *GrantAndGrantSpec, opts ...grpc.CallOption) (*grant.Grant, error) { out := new(grant.Grant) - err := c.cc.Invoke(ctx, "/services.Grant/Reseal", in, out, opts...) + err := c.cc.Invoke(ctx, "/api.Grant/Reseal", in, out, opts...) if err != nil { return nil, err } @@ -444,7 +443,7 @@ func (c *grantClient) Reseal(ctx context.Context, in *GrantAndGrantSpec, opts .. func (c *grantClient) PutSeal(ctx context.Context, in *PlaintextAndGrantSpec, opts ...grpc.CallOption) (*grant.Grant, error) { out := new(grant.Grant) - err := c.cc.Invoke(ctx, "/services.Grant/PutSeal", in, out, opts...) + err := c.cc.Invoke(ctx, "/api.Grant/PutSeal", in, out, opts...) if err != nil { return nil, err } @@ -453,7 +452,7 @@ func (c *grantClient) PutSeal(ctx context.Context, in *PlaintextAndGrantSpec, op func (c *grantClient) UnsealGet(ctx context.Context, in *grant.Grant, opts ...grpc.CallOption) (*Plaintext, error) { out := new(Plaintext) - err := c.cc.Invoke(ctx, "/services.Grant/UnsealGet", in, out, opts...) + err := c.cc.Invoke(ctx, "/api.Grant/UnsealGet", in, out, opts...) if err != nil { return nil, err } @@ -489,7 +488,7 @@ func _Grant_Seal_Handler(srv interface{}, ctx context.Context, dec func(interfac } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/services.Grant/Seal", + FullMethod: "/api.Grant/Seal", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(GrantServer).Seal(ctx, req.(*ReferenceAndGrantSpec)) @@ -507,7 +506,7 @@ func _Grant_Unseal_Handler(srv interface{}, ctx context.Context, dec func(interf } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/services.Grant/Unseal", + FullMethod: "/api.Grant/Unseal", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(GrantServer).Unseal(ctx, req.(*grant.Grant)) @@ -525,7 +524,7 @@ func _Grant_Reseal_Handler(srv interface{}, ctx context.Context, dec func(interf } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/services.Grant/Reseal", + FullMethod: "/api.Grant/Reseal", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(GrantServer).Reseal(ctx, req.(*GrantAndGrantSpec)) @@ -543,7 +542,7 @@ func _Grant_PutSeal_Handler(srv interface{}, ctx context.Context, dec func(inter } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/services.Grant/PutSeal", + FullMethod: "/api.Grant/PutSeal", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(GrantServer).PutSeal(ctx, req.(*PlaintextAndGrantSpec)) @@ -561,7 +560,7 @@ func _Grant_UnsealGet_Handler(srv interface{}, ctx context.Context, dec func(int } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/services.Grant/UnsealGet", + FullMethod: "/api.Grant/UnsealGet", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(GrantServer).UnsealGet(ctx, req.(*grant.Grant)) @@ -570,7 +569,7 @@ func _Grant_UnsealGet_Handler(srv interface{}, ctx context.Context, dec func(int } var _Grant_serviceDesc = grpc.ServiceDesc{ - ServiceName: "services.Grant", + ServiceName: "api.Grant", HandlerType: (*GrantServer)(nil), Methods: []grpc.MethodDesc{ { @@ -595,7 +594,7 @@ var _Grant_serviceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{}, - Metadata: "services.proto", + Metadata: "api.proto", } // CleartextClient is the client API for Cleartext service. @@ -620,7 +619,7 @@ func NewCleartextClient(cc *grpc.ClientConn) CleartextClient { func (c *cleartextClient) Put(ctx context.Context, in *Plaintext, opts ...grpc.CallOption) (*reference.Ref, error) { out := new(reference.Ref) - err := c.cc.Invoke(ctx, "/services.Cleartext/Put", in, out, opts...) + err := c.cc.Invoke(ctx, "/api.Cleartext/Put", in, out, opts...) if err != nil { return nil, err } @@ -629,7 +628,7 @@ func (c *cleartextClient) Put(ctx context.Context, in *Plaintext, opts ...grpc.C func (c *cleartextClient) Get(ctx context.Context, in *reference.Ref, opts ...grpc.CallOption) (*Plaintext, error) { out := new(Plaintext) - err := c.cc.Invoke(ctx, "/services.Cleartext/Get", in, out, opts...) + err := c.cc.Invoke(ctx, "/api.Cleartext/Get", in, out, opts...) if err != nil { return nil, err } @@ -660,7 +659,7 @@ func _Cleartext_Put_Handler(srv interface{}, ctx context.Context, dec func(inter } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/services.Cleartext/Put", + FullMethod: "/api.Cleartext/Put", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CleartextServer).Put(ctx, req.(*Plaintext)) @@ -678,7 +677,7 @@ func _Cleartext_Get_Handler(srv interface{}, ctx context.Context, dec func(inter } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/services.Cleartext/Get", + FullMethod: "/api.Cleartext/Get", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(CleartextServer).Get(ctx, req.(*reference.Ref)) @@ -687,7 +686,7 @@ func _Cleartext_Get_Handler(srv interface{}, ctx context.Context, dec func(inter } var _Cleartext_serviceDesc = grpc.ServiceDesc{ - ServiceName: "services.Cleartext", + ServiceName: "api.Cleartext", HandlerType: (*CleartextServer)(nil), Methods: []grpc.MethodDesc{ { @@ -700,7 +699,7 @@ var _Cleartext_serviceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{}, - Metadata: "services.proto", + Metadata: "api.proto", } // EncryptionClient is the client API for Encryption service. @@ -725,7 +724,7 @@ func NewEncryptionClient(cc *grpc.ClientConn) EncryptionClient { func (c *encryptionClient) Encrypt(ctx context.Context, in *Plaintext, opts ...grpc.CallOption) (*ReferenceAndCiphertext, error) { out := new(ReferenceAndCiphertext) - err := c.cc.Invoke(ctx, "/services.Encryption/Encrypt", in, out, opts...) + err := c.cc.Invoke(ctx, "/api.Encryption/Encrypt", in, out, opts...) if err != nil { return nil, err } @@ -734,7 +733,7 @@ func (c *encryptionClient) Encrypt(ctx context.Context, in *Plaintext, opts ...g func (c *encryptionClient) Decrypt(ctx context.Context, in *ReferenceAndCiphertext, opts ...grpc.CallOption) (*Plaintext, error) { out := new(Plaintext) - err := c.cc.Invoke(ctx, "/services.Encryption/Decrypt", in, out, opts...) + err := c.cc.Invoke(ctx, "/api.Encryption/Decrypt", in, out, opts...) if err != nil { return nil, err } @@ -765,7 +764,7 @@ func _Encryption_Encrypt_Handler(srv interface{}, ctx context.Context, dec func( } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/services.Encryption/Encrypt", + FullMethod: "/api.Encryption/Encrypt", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(EncryptionServer).Encrypt(ctx, req.(*Plaintext)) @@ -783,7 +782,7 @@ func _Encryption_Decrypt_Handler(srv interface{}, ctx context.Context, dec func( } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/services.Encryption/Decrypt", + FullMethod: "/api.Encryption/Decrypt", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(EncryptionServer).Decrypt(ctx, req.(*ReferenceAndCiphertext)) @@ -792,7 +791,7 @@ func _Encryption_Decrypt_Handler(srv interface{}, ctx context.Context, dec func( } var _Encryption_serviceDesc = grpc.ServiceDesc{ - ServiceName: "services.Encryption", + ServiceName: "api.Encryption", HandlerType: (*EncryptionServer)(nil), Methods: []grpc.MethodDesc{ { @@ -805,7 +804,7 @@ var _Encryption_serviceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{}, - Metadata: "services.proto", + Metadata: "api.proto", } // StorageClient is the client API for Storage service. @@ -818,7 +817,7 @@ type StorageClient interface { Pull(ctx context.Context, in *Address, opts ...grpc.CallOption) (*Ciphertext, error) // Get some information about the encrypted blob stored at an address, // including whether it exists. - Stat(ctx context.Context, in *Address, opts ...grpc.CallOption) (*storage.StatInfo, error) + Stat(ctx context.Context, in *Address, opts ...grpc.CallOption) (*stores.StatInfo, error) } type storageClient struct { @@ -831,7 +830,7 @@ func NewStorageClient(cc *grpc.ClientConn) StorageClient { func (c *storageClient) Push(ctx context.Context, in *Ciphertext, opts ...grpc.CallOption) (*Address, error) { out := new(Address) - err := c.cc.Invoke(ctx, "/services.Storage/Push", in, out, opts...) + err := c.cc.Invoke(ctx, "/api.Storage/Push", in, out, opts...) if err != nil { return nil, err } @@ -840,16 +839,16 @@ func (c *storageClient) Push(ctx context.Context, in *Ciphertext, opts ...grpc.C func (c *storageClient) Pull(ctx context.Context, in *Address, opts ...grpc.CallOption) (*Ciphertext, error) { out := new(Ciphertext) - err := c.cc.Invoke(ctx, "/services.Storage/Pull", in, out, opts...) + err := c.cc.Invoke(ctx, "/api.Storage/Pull", in, out, opts...) if err != nil { return nil, err } return out, nil } -func (c *storageClient) Stat(ctx context.Context, in *Address, opts ...grpc.CallOption) (*storage.StatInfo, error) { - out := new(storage.StatInfo) - err := c.cc.Invoke(ctx, "/services.Storage/Stat", in, out, opts...) +func (c *storageClient) Stat(ctx context.Context, in *Address, opts ...grpc.CallOption) (*stores.StatInfo, error) { + out := new(stores.StatInfo) + err := c.cc.Invoke(ctx, "/api.Storage/Stat", in, out, opts...) if err != nil { return nil, err } @@ -864,7 +863,7 @@ type StorageServer interface { Pull(context.Context, *Address) (*Ciphertext, error) // Get some information about the encrypted blob stored at an address, // including whether it exists. - Stat(context.Context, *Address) (*storage.StatInfo, error) + Stat(context.Context, *Address) (*stores.StatInfo, error) } func RegisterStorageServer(s *grpc.Server, srv StorageServer) { @@ -881,7 +880,7 @@ func _Storage_Push_Handler(srv interface{}, ctx context.Context, dec func(interf } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/services.Storage/Push", + FullMethod: "/api.Storage/Push", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(StorageServer).Push(ctx, req.(*Ciphertext)) @@ -899,7 +898,7 @@ func _Storage_Pull_Handler(srv interface{}, ctx context.Context, dec func(interf } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/services.Storage/Pull", + FullMethod: "/api.Storage/Pull", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(StorageServer).Pull(ctx, req.(*Address)) @@ -917,7 +916,7 @@ func _Storage_Stat_Handler(srv interface{}, ctx context.Context, dec func(interf } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/services.Storage/Stat", + FullMethod: "/api.Storage/Stat", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(StorageServer).Stat(ctx, req.(*Address)) @@ -926,7 +925,7 @@ func _Storage_Stat_Handler(srv interface{}, ctx context.Context, dec func(interf } var _Storage_serviceDesc = grpc.ServiceDesc{ - ServiceName: "services.Storage", + ServiceName: "api.Storage", HandlerType: (*StorageServer)(nil), Methods: []grpc.MethodDesc{ { @@ -943,5 +942,5 @@ var _Storage_serviceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{}, - Metadata: "services.proto", + Metadata: "api.proto", } diff --git a/cmd/hoarctl/encryption.go b/cmd/hoarctl/encryption.go index 5d8902ec..a2c2ab8b 100644 --- a/cmd/hoarctl/encryption.go +++ b/cmd/hoarctl/encryption.go @@ -7,8 +7,8 @@ import ( "os" cli "github.com/jawher/mow.cli" + "github.com/monax/hoard/v4/api" "github.com/monax/hoard/v4/reference" - "github.com/monax/hoard/v4/services" ) // Decrypt does what it says on the tin @@ -19,12 +19,12 @@ func (client *Client) Decrypt(cmd *cli.Cmd) { cmd.Action = func() { encryptedData := readData() plaintext, err := client.encryption.Decrypt(context.Background(), - &services.ReferenceAndCiphertext{ + &api.ReferenceAndCiphertext{ Reference: &reference.Ref{ SecretKey: readBase64(secretKey), Salt: parseSalt(salt), }, - Ciphertext: &services.Ciphertext{ + Ciphertext: &api.Ciphertext{ EncryptedData: encryptedData, }, }) @@ -45,7 +45,7 @@ func (client *Client) Encrypt(cmd *cli.Cmd) { fatalf("could read bytes from STDIN to store: %v", err) } refAndCiphertext, err := client.encryption.Encrypt(context.Background(), - &services.Plaintext{ + &api.Plaintext{ Data: data, Salt: parseSalt(salt), }) @@ -63,7 +63,7 @@ func (client *Client) Ref(cmd *cli.Cmd) { cmd.Action = func() { data := readData() refAndCiphertext, err := client.encryption.Encrypt(context.Background(), - &services.Plaintext{ + &api.Plaintext{ Data: data, Salt: parseSalt(salt), }) diff --git a/cmd/hoarctl/grants.go b/cmd/hoarctl/grants.go index 0a31fd05..0607b099 100644 --- a/cmd/hoarctl/grants.go +++ b/cmd/hoarctl/grants.go @@ -6,8 +6,8 @@ import ( "os" cli "github.com/jawher/mow.cli" + "github.com/monax/hoard/v4/api" "github.com/monax/hoard/v4/grant" - "github.com/monax/hoard/v4/services" ) // PutSeal encrypts and stores data then prints a grant @@ -29,8 +29,8 @@ func (client *Client) PutSeal(cmd *cli.Cmd) { data := readData() seal, err = client.grant.PutSeal(context.Background(), - &services.PlaintextAndGrantSpec{ - Plaintext: &services.Plaintext{ + &api.PlaintextAndGrantSpec{ + Plaintext: &api.Plaintext{ Data: data, Salt: parseSalt(salt), }, @@ -61,7 +61,7 @@ func (client *Client) Seal(cmd *cli.Cmd) { ref := readReference(address) seal, err := client.grant.Seal(context.Background(), - &services.ReferenceAndGrantSpec{ + &api.ReferenceAndGrantSpec{ Reference: ref, GrantSpec: &spec, }, @@ -90,7 +90,7 @@ func (client *Client) Reseal(cmd *cli.Cmd) { } ref, err := client.grant.Reseal(context.Background(), - &services.GrantAndGrantSpec{ + &api.GrantAndGrantSpec{ Grant: prev, GrantSpec: &next, }) diff --git a/cmd/hoarctl/main.go b/cmd/hoarctl/main.go index 5051ca57..d6fdf0ef 100644 --- a/cmd/hoarctl/main.go +++ b/cmd/hoarctl/main.go @@ -10,13 +10,14 @@ import ( "os" "time" + "github.com/monax/hoard/v4/api" + cli "github.com/jawher/mow.cli" "github.com/monax/hoard/v4/cmd" "github.com/monax/hoard/v4/config" "github.com/monax/hoard/v4/grant" "github.com/monax/hoard/v4/reference" "github.com/monax/hoard/v4/server" - "github.com/monax/hoard/v4/services" "google.golang.org/grpc" ) @@ -31,10 +32,10 @@ const ( // Client scopes the available hoard clients type Client struct { - cleartext services.CleartextClient - encryption services.EncryptionClient - grant services.GrantClient - storage services.StorageClient + cleartext api.CleartextClient + encryption api.EncryptionClient + grant api.GrantClient + storage api.StorageClient } func main() { @@ -62,10 +63,10 @@ func main() { if err != nil { fatalf("Could not dial hoard server on %s: %v", *dialURL, err) } - client.cleartext = services.NewCleartextClient(conn) - client.encryption = services.NewEncryptionClient(conn) - client.grant = services.NewGrantClient(conn) - client.storage = services.NewStorageClient(conn) + client.cleartext = api.NewCleartextClient(conn) + client.encryption = api.NewEncryptionClient(conn) + client.grant = api.NewGrantClient(conn) + client.storage = api.NewStorageClient(conn) } cmd.AddVersionCommand(hoarctlApp) diff --git a/cmd/hoarctl/store.go b/cmd/hoarctl/store.go index 268199d6..c9f48056 100644 --- a/cmd/hoarctl/store.go +++ b/cmd/hoarctl/store.go @@ -6,8 +6,8 @@ import ( "os" cli "github.com/jawher/mow.cli" + "github.com/monax/hoard/v4/api" "github.com/monax/hoard/v4/reference" - "github.com/monax/hoard/v4/services" ) // Cat retrieves encrypted data from store @@ -17,7 +17,7 @@ func (client *Client) Cat(cmd *cli.Cmd) { cmd.Action = func() { ref := readReference(address) ciphertext, err := client.storage.Pull(context.Background(), - &services.Address{Address: ref.Address}) + &api.Address{Address: ref.Address}) if err != nil { fatalf("Error querying data: %v", err) } @@ -58,7 +58,7 @@ func (client *Client) Insert(cmd *cli.Cmd) { data := readData() // If given address use it address, err := client.storage.Push(context.Background(), - &services.Ciphertext{EncryptedData: data}) + &api.Ciphertext{EncryptedData: data}) if err != nil { fatalf("Error querying data: %v", err) } @@ -73,7 +73,7 @@ func (client *Client) Put(cmd *cli.Cmd) { cmd.Action = func() { data := readData() ref, err := client.cleartext.Put(context.Background(), - &services.Plaintext{ + &api.Plaintext{ Data: data, Salt: parseSalt(salt), }) @@ -91,7 +91,7 @@ func (client *Client) Stat(cmd *cli.Cmd) { cmd.Action = func() { ref := readReference(address) statInfo, err := client.storage.Stat(context.Background(), - &services.Address{Address: ref.Address}) + &api.Address{Address: ref.Address}) if err != nil { fatalf("Error querying data: %v", err) } diff --git a/cmd/hoard/main.go b/cmd/hoard/main.go index 7027ae31..4aedf2cb 100644 --- a/cmd/hoard/main.go +++ b/cmd/hoard/main.go @@ -13,10 +13,6 @@ import ( cli "github.com/jawher/mow.cli" "github.com/monax/hoard/v4/cmd" "github.com/monax/hoard/v4/config" - "github.com/monax/hoard/v4/config/logging" - "github.com/monax/hoard/v4/config/secrets" - "github.com/monax/hoard/v4/config/source" - "github.com/monax/hoard/v4/config/storage" "github.com/monax/hoard/v4/server" ) @@ -37,7 +33,7 @@ func main() { environmentOpt := hoardApp.BoolOpt("e env", false, fmt.Sprintf("Parse the contents of the environment variable %s as a complete JSON config", - source.DefaultJSONConfigEnvironmentVariable)) + config.DefaultJSONConfigEnvironmentVariable)) // This string spec is parsed by mow.cli and has actual semantic significance // around optionality and ordering of options and arguments @@ -54,22 +50,22 @@ func main() { var logger log.Logger if *loggingOpt { - logger, err = logging.LoggerFromLoggingConfig(conf.Logging, os.Stderr) + logger, err = config.Logger(conf.Logging, os.Stderr) if err != nil { fatalf("Could not create logging form logging config: %s", err) } } - store, err := storage.StoreFromStorageConfig(conf.Storage, logger) + store, err := StoreFromStorageConfig(conf.Storage, logger) if err != nil { fatalf("Could not configure store from storage config: %s", err) } if *listenAddressOpt != "" { conf.ListenAddress = *listenAddressOpt } - symmetricProvider := secrets.ProviderFromConfig(conf.Secrets) - openPGPConf := secrets.OpenPGPFromConfig(conf.Secrets) - sm := secrets.Manager{Provider: symmetricProvider, OpenPGP: openPGPConf} + symmetricProvider := config.NewSymmetricProvider(conf.Secrets) + openPGPConf := config.NewOpenPGPSecret(conf.Secrets) + sm := config.SecretsManager{Provider: symmetricProvider, OpenPGP: openPGPConf} serv := server.New(conf.ListenAddress, store, sm, logger) // Catch interrupt etc @@ -114,20 +110,20 @@ func main() { secretsOpt := configCmd.StringsOpt("s secret", nil, "Pairs of PublicID and Passphrase to use as symmetric secrets in config") - arg := configCmd.StringArg("CONFIG", "", fmt.Sprintf("Config type to generate, one of: %s", + arg := configCmd.StringArg("CONFIG", "", fmt.Sprintf("Storage type to generate, one of: %s", strings.Join(configTypes(), ", "))) configCmd.Spec = "[--json | --yaml] | (([--output=] | [--init]) [--force]) CONFIG [--secret=...]" configCmd.Action = func() { - store, err := storage.GetDefaultConfig(*arg) + store, err := config.GetDefaultStorage(*arg) if err != nil { fatalf("Error fetching default config for %v: %v", arg, err) } conf.Storage = store if len(*secretsOpt) > 0 { - conf.Secrets = &secrets.SecretsConfig{ - Symmetric: make([]secrets.SymmetricSecret, len(*secretsOpt)), + conf.Secrets = &config.Secrets{ + Symmetric: make([]config.SymmetricSecret, len(*secretsOpt)), } for i, ss := range *secretsOpt { pair := strings.Split(ss, ":") @@ -149,7 +145,7 @@ func main() { } if *initOpt { configFileName, err := xdgbasedir.GetConfigFileLocation( - source.DefaultHoardConfigFileName) + config.DefaultHoardConfigFileName) if err != nil { fatalf("Error getting config file location: %s", err) } @@ -180,12 +176,12 @@ func fatalf(format string, args ...interface{}) { os.Exit(1) } -func hoardConfigCascade(env bool, configFile string) source.ConfigProvider { - return source.Cascade(os.Stderr, true, - source.Environment(source.DefaultJSONConfigEnvironmentVariable).SetSkip(!env), - source.File(configFile).SetSkip(configFile == ""), - source.XDGBaseDir(), - source.Default()) +func hoardConfigCascade(env bool, configFile string) config.Provider { + return config.Cascade(os.Stderr, true, + config.Environment(config.DefaultJSONConfigEnvironmentVariable).SetSkip(!env), + config.File(configFile).SetSkip(configFile == ""), + config.XDGBaseDir(), + config.Default()) } func writeFile(filename string, data []byte, overwrite bool) error { @@ -196,7 +192,7 @@ func writeFile(filename string, data []byte, overwrite bool) error { } func configTypes() []string { - storageTypes := storage.GetStorageTypes() + storageTypes := config.GetStorageTypes() configTypes := make([]string, len(storageTypes)) for i, st := range storageTypes { configTypes[i] = string(st) diff --git a/cmd/hoard/stores.go b/cmd/hoard/stores.go new file mode 100644 index 00000000..f2b93302 --- /dev/null +++ b/cmd/hoard/stores.go @@ -0,0 +1,73 @@ +package main + +import ( + "errors" + "fmt" + + "github.com/go-kit/kit/log" + "github.com/monax/hoard/v4/config" + "github.com/monax/hoard/v4/stores" + "github.com/monax/hoard/v4/stores/cloud" + "github.com/monax/hoard/v4/stores/ipfs" +) + +func StoreFromStorageConfig(storageConfig *config.Storage, logger log.Logger) (stores.NamedStore, error) { + addressEncoding, err := stores.GetAddressEncoding(storageConfig.AddressEncoding) + if err != nil { + return nil, err + } + + switch storageConfig.StorageType { + case config.Memory, config.Unspecified: + return stores.NewMemoryStore(), nil + + case config.Filesystem: + fsConf := storageConfig.FileSystemConfig + if fsConf == nil { + return nil, errors.New("filesystem storage configuration must be " + + "supplied to use the filesystem storage backend") + } + if fsConf.RootDirectory == "" { + return nil, errors.New("rootDirectory key must be non-empty in " + "filesystem storage") + + } + return stores.NewFileSystemStore(fsConf.RootDirectory, addressEncoding) + + case config.IPFS: + ipfsConf := storageConfig.IPFSConfig + if ipfsConf == nil { + return nil, errors.New("IPFS storage configuration must be " + + "supplied to use the filesystem storage backend") + } + if ipfsConf.RemoteAPI == "" { + return nil, errors.New("http api url must be non-empty in " + + "ipfs storage config") + } + return ipfs.NewStore(ipfsConf.RemoteAPI, addressEncoding) + + case config.AWS: + awsConf := storageConfig.Cloud + if awsConf == nil { + return nil, errors.New("aws configuration must be supplied") + } + return cloud.NewStore(cloud.AWS, awsConf.Bucket, awsConf.Prefix, awsConf.Region, addressEncoding, logger) + + case config.Azure: + azureConf := storageConfig.Cloud + if azureConf == nil { + return nil, errors.New("azure configuration must be supplied") + } + return cloud.NewStore(cloud.Azure, azureConf.Bucket, azureConf.Prefix, azureConf.Region, addressEncoding, logger) + + case config.GCP: + gcpConf := storageConfig.Cloud + if gcpConf == nil { + return nil, errors.New("gcp configuration must be supplied") + } + return cloud.NewStore(cloud.GCP, gcpConf.Bucket, gcpConf.Prefix, gcpConf.Region, addressEncoding, logger) + + default: + return nil, fmt.Errorf("did not recognise storage type '%s'", + storageConfig.StorageType) + } +} diff --git a/config/storage/cloud.go b/config/cloud.go similarity index 53% rename from config/storage/cloud.go rename to config/cloud.go index 302eff1e..77a3e04d 100644 --- a/config/storage/cloud.go +++ b/config/cloud.go @@ -1,20 +1,20 @@ -package storage +package config import ( "fmt" ) -type CloudConfig struct { +type Cloud struct { Bucket string Prefix string Region string } -func NewCloudConfig(encoding, cloud, bucket, prefix, region string) (*StorageConfig, error) { - return &StorageConfig{ +func NewCloud(encoding, cloud, bucket, prefix, region string) (*Storage, error) { + return &Storage{ StorageType: StorageType(cloud), AddressEncoding: encoding, - CloudConfig: &CloudConfig{ + Cloud: &Cloud{ Bucket: bucket, Prefix: prefix, Region: region, @@ -22,8 +22,8 @@ func NewCloudConfig(encoding, cloud, bucket, prefix, region string) (*StorageCon }, nil } -func DefaultCloudConfig(cloud string) *StorageConfig { - conf, err := NewCloudConfig(DefaultAddressEncodingName, +func DefaultCloud(cloud string) *Storage { + conf, err := NewCloud(DefaultAddressEncodingName, cloud, "hoard", "store", diff --git a/config/cloud_test.go b/config/cloud_test.go new file mode 100644 index 00000000..c019ee7a --- /dev/null +++ b/config/cloud_test.go @@ -0,0 +1,9 @@ +package config + +import "testing" + +func TestDefaultCloudConfig(t *testing.T) { + assertStorageConfigSerialisation(t, DefaultCloud("aws")) + assertStorageConfigSerialisation(t, DefaultCloud("azure")) + assertStorageConfigSerialisation(t, DefaultCloud("gcp")) +} diff --git a/config/config.go b/config/config.go index c5bd1412..38544e32 100644 --- a/config/config.go +++ b/config/config.go @@ -6,26 +6,22 @@ import ( "fmt" "github.com/BurntSushi/toml" - "github.com/monax/hoard/v4/config/logging" - "github.com/monax/hoard/v4/config/secrets" - "github.com/monax/hoard/v4/config/storage" yaml "gopkg.in/yaml.v2" ) const DefaultListenAddress = "tcp://:53431" var DefaultHoardConfig = NewHoardConfig(DefaultListenAddress, - storage.DefaultConfig, logging.DefaultConfig) + DefaultStorage, DefaultLogging) type HoardConfig struct { ListenAddress string - Storage *storage.StorageConfig - Logging *logging.LoggingConfig - Secrets *secrets.SecretsConfig + Storage *Storage + Logging *Logging + Secrets *Secrets } -func NewHoardConfig(listenAddress string, storageConfig *storage.StorageConfig, - loggingConfig *logging.LoggingConfig) *HoardConfig { +func NewHoardConfig(listenAddress string, storageConfig *Storage, loggingConfig *Logging) *HoardConfig { return &HoardConfig{ ListenAddress: listenAddress, Storage: storageConfig, diff --git a/config/storage/filesystem.go b/config/filesystem.go similarity index 73% rename from config/storage/filesystem.go rename to config/filesystem.go index 58f95554..9fb85225 100644 --- a/config/storage/filesystem.go +++ b/config/filesystem.go @@ -1,19 +1,20 @@ -package storage +package config import ( "fmt" "path" + "github.com/monax/hoard/v4/stores" + "github.com/cep21/xdgbasedir" - "github.com/monax/hoard/v4/storage" ) type FileSystemConfig struct { RootDirectory string } -func NewFileSystemConfig(addressEncoding, rootDirectory string) *StorageConfig { - return &StorageConfig{ +func NewFileSystemConfig(addressEncoding, rootDirectory string) *Storage { + return &Storage{ StorageType: Filesystem, AddressEncoding: addressEncoding, FileSystemConfig: &FileSystemConfig{ @@ -22,12 +23,12 @@ func NewFileSystemConfig(addressEncoding, rootDirectory string) *StorageConfig { } } -func DefaultFileSystemConfig() *StorageConfig { +func DefaultFileSystemConfig() *Storage { dataDir, err := xdgbasedir.DataHomeDirectory() if err != nil { panic(fmt.Errorf("could not get XDG data dir: %s", err)) } // Avoid '/' character for filesystem storage - return NewFileSystemConfig(storage.Base32EncodingName, + return NewFileSystemConfig(stores.Base32EncodingName, path.Join(dataDir, "hoard")) } diff --git a/config/storage/filesystem_test.go b/config/filesystem_test.go similarity index 89% rename from config/storage/filesystem_test.go rename to config/filesystem_test.go index faf88839..f8751ba1 100644 --- a/config/storage/filesystem_test.go +++ b/config/filesystem_test.go @@ -1,4 +1,4 @@ -package storage +package config import "testing" diff --git a/config/storage/ipfs.go b/config/ipfs.go similarity index 61% rename from config/storage/ipfs.go rename to config/ipfs.go index 92f63670..80959d8d 100644 --- a/config/storage/ipfs.go +++ b/config/ipfs.go @@ -1,11 +1,11 @@ -package storage +package config type IPFSConfig struct { RemoteAPI string } -func NewIPFSConfig(addressEncoding, host string) *StorageConfig { - return &StorageConfig{ +func NewIPFSConfig(addressEncoding, host string) *Storage { + return &Storage{ StorageType: IPFS, AddressEncoding: addressEncoding, IPFSConfig: &IPFSConfig{ @@ -14,7 +14,7 @@ func NewIPFSConfig(addressEncoding, host string) *StorageConfig { } } -func DefaultIPFSConfig() *StorageConfig { +func DefaultIPFSConfig() *Storage { return NewIPFSConfig(DefaultAddressEncodingName, "http://:5001", ) diff --git a/config/storage/ipfs_test.go b/config/ipfs_test.go similarity index 88% rename from config/storage/ipfs_test.go rename to config/ipfs_test.go index f681e5f1..16106eea 100644 --- a/config/storage/ipfs_test.go +++ b/config/ipfs_test.go @@ -1,4 +1,4 @@ -package storage +package config import "testing" diff --git a/config/logging/logging.go b/config/logging.go similarity index 84% rename from config/logging/logging.go rename to config/logging.go index a8955a0a..bd756446 100644 --- a/config/logging/logging.go +++ b/config/logging.go @@ -1,4 +1,4 @@ -package logging +package config import ( "io" @@ -13,7 +13,7 @@ import ( type LoggingType string -var DefaultConfig = NewLoggingConfig(Json, structure.InfoChannel, +var DefaultLogging = NewLogging(Json, structure.InfoChannel, structure.TraceChannel) const ( @@ -21,25 +21,24 @@ const ( Json LoggingType = "json" ) -// LoggingConfig describes the channels to listen on, +// Logging describes the channels to listen on, // messages not on these channels will be filtered // and leaving empty disables logging -type LoggingConfig struct { +type Logging struct { LoggingType LoggingType Channels []structure.Channel } -func NewLoggingConfig(loggingType LoggingType, - channels ...structure.Channel) *LoggingConfig { +func NewLogging(loggingType LoggingType, + channels ...structure.Channel) *Logging { - return &LoggingConfig{ + return &Logging{ LoggingType: loggingType, Channels: channels, } } -func LoggerFromLoggingConfig(loggingConfig *LoggingConfig, - writer io.Writer) (log.Logger, error) { +func Logger(loggingConfig *Logging, writer io.Writer) (log.Logger, error) { terminalLogger, err := NewTerminalLogger(loggingConfig.LoggingType, writer) if err != nil { return nil, err diff --git a/config/logging/logging_test.go b/config/logging_test.go similarity index 84% rename from config/logging/logging_test.go rename to config/logging_test.go index 736b9a33..4368e766 100644 --- a/config/logging/logging_test.go +++ b/config/logging_test.go @@ -1,4 +1,4 @@ -package logging +package config import ( "testing" @@ -19,7 +19,7 @@ func TestTerminalLogger(t *testing.T) { func TestLoggerFromLoggingConfig(t *testing.T) { buf := new(bytes.Buffer) - logger, err := LoggerFromLoggingConfig(NewLoggingConfig(Logfmt, structure.TraceChannel), buf) + logger, err := Logger(NewLogging(Logfmt, structure.TraceChannel), buf) assert.NoError(t, err) logger.Log(structure.ChannelKey, structure.TraceChannel, "foo", "bar") assert.Equal(t, "channel=trace foo=bar\n", buf.String()) diff --git a/config/memory.go b/config/memory.go new file mode 100644 index 00000000..e6929585 --- /dev/null +++ b/config/memory.go @@ -0,0 +1,9 @@ +package config + +func NewMemory(addressEncoding string) *Storage { + return NewStorage(Memory, addressEncoding) +} + +func DefaultMemory() *Storage { + return NewMemory(DefaultAddressEncodingName) +} diff --git a/config/memory_test.go b/config/memory_test.go new file mode 100644 index 00000000..732386c9 --- /dev/null +++ b/config/memory_test.go @@ -0,0 +1,7 @@ +package config + +import "testing" + +func TestDefaultMemoryConfig(t *testing.T) { + assertStorageConfigSerialisation(t, DefaultMemory()) +} diff --git a/config/secrets/secrets.go b/config/secrets.go similarity index 86% rename from config/secrets/secrets.go rename to config/secrets.go index 36c77bcd..7d5d80e0 100644 --- a/config/secrets/secrets.go +++ b/config/secrets.go @@ -1,14 +1,14 @@ -package secrets +package config import ( "fmt" "io/ioutil" ) -// SecretsConfig lists the configured secrets, +// Secrets lists the configured secrets, // Symmetric secrets are those local to the running daemon // and OpenPGP identifies an entity in the given keyring -type SecretsConfig struct { +type Secrets struct { Symmetric []SymmetricSecret OpenPGP *OpenPGPSecret } @@ -27,7 +27,7 @@ type OpenPGPSecret struct { Data []byte } -type Manager struct { +type SecretsManager struct { Provider SymmetricProvider OpenPGP *OpenPGPSecret } @@ -35,7 +35,7 @@ type Manager struct { type SymmetricProvider func(secretID string) ([]byte, error) // NoopSecretManager is an empty secret manager -var NoopSecretManager = Manager{ +var NoopSecretManager = SecretsManager{ Provider: NoopSymmetricProvider, OpenPGP: nil, } @@ -46,7 +46,7 @@ func NoopSymmetricProvider(_ string) ([]byte, error) { } // ProviderFromConfig creates a secret reader from a set of symmetric secrets -func ProviderFromConfig(conf *SecretsConfig) SymmetricProvider { +func NewSymmetricProvider(conf *Secrets) SymmetricProvider { if conf == nil || len(conf.Symmetric) == 0 { return NoopSymmetricProvider } @@ -66,7 +66,7 @@ func ProviderFromConfig(conf *SecretsConfig) SymmetricProvider { } // OpenPGPFromConfig reads a given PGP keyring -func OpenPGPFromConfig(conf *SecretsConfig) *OpenPGPSecret { +func NewOpenPGPSecret(conf *Secrets) *OpenPGPSecret { if conf == nil || conf.OpenPGP == nil { return nil } diff --git a/config/source/source.go b/config/source.go similarity index 74% rename from config/source/source.go rename to config/source.go index f887210d..e3175eb6 100644 --- a/config/source/source.go +++ b/config/source.go @@ -1,4 +1,4 @@ -package source +package config import ( "errors" @@ -9,37 +9,36 @@ import ( "strings" "github.com/cep21/xdgbasedir" - "github.com/monax/hoard/v4/config" ) const DefaultHoardConfigFileName = "hoard.conf" const DefaultJSONConfigEnvironmentVariable = "HOARD_JSON_CONFIG" const STDINFileIdentifier = "-" -type ConfigProvider interface { +type Provider interface { // Description of where this provider sources its config from From() string // Get the config possibly overriding values passed in from baseConfig - Get(baseConfig *config.HoardConfig) (*config.HoardConfig, error) + Get(baseConfig *HoardConfig) (*HoardConfig, error) // Return a copy of the provider that does nothing if skip is true - SetSkip(skip bool) ConfigProvider + SetSkip(skip bool) Provider // Whether to skip this provider Skip() bool } -var _ ConfigProvider = &configSource{} +var _ Provider = &configSource{} type configSource struct { from string skip bool - provider func(baseConfig *config.HoardConfig) (*config.HoardConfig, error) + provider func(baseConfig *HoardConfig) (*HoardConfig, error) } func (cs *configSource) From() string { return cs.from } -func (cs *configSource) Get(baseConfig *config.HoardConfig) (*config.HoardConfig, error) { +func (cs *configSource) Get(baseConfig *HoardConfig) (*HoardConfig, error) { return cs.provider(baseConfig) } @@ -48,7 +47,7 @@ func (cs *configSource) Skip() bool { } // Returns a copy of the configSource with skip set as passed in -func (cs *configSource) SetSkip(skip bool) ConfigProvider { +func (cs *configSource) SetSkip(skip bool) Provider { return &configSource{ skip: skip, from: cs.from, @@ -56,7 +55,7 @@ func (cs *configSource) SetSkip(skip bool) ConfigProvider { } } -func Cascade(logWriter io.Writer, shortCircuit bool, sources ...ConfigProvider) *configSource { +func Cascade(logWriter io.Writer, shortCircuit bool, sources ...Provider) *configSource { var fromStrings []string for _, source := range sources { if !source.Skip() { @@ -65,7 +64,7 @@ func Cascade(logWriter io.Writer, shortCircuit bool, sources ...ConfigProvider) } return &configSource{ from: strings.Join(fromStrings, " then "), - provider: func(baseConfig *config.HoardConfig) (*config.HoardConfig, error) { + provider: func(baseConfig *HoardConfig) (*HoardConfig, error) { for _, source := range sources { if !source.Skip() { writeLog(logWriter, fmt.Sprintf("Trying to source config from %s", source.From())) @@ -97,8 +96,8 @@ func Cascade(logWriter io.Writer, shortCircuit bool, sources ...ConfigProvider) func File(configFile string) *configSource { return &configSource{ skip: configFile == "", - from: fmt.Sprintf("Config file at '%s'", configFile), - provider: func(baseConfig *config.HoardConfig) (*config.HoardConfig, error) { + from: fmt.Sprintf("Storage file at '%s'", configFile), + provider: func(baseConfig *HoardConfig) (*HoardConfig, error) { return fromFile(configFile) }, } @@ -117,7 +116,7 @@ func XDGBaseDir() *configSource { return &configSource{ skip: skip, from: fmt.Sprintf("XDG base dir"), - provider: func(baseConfig *config.HoardConfig) (*config.HoardConfig, error) { + provider: func(baseConfig *HoardConfig) (*HoardConfig, error) { if err != nil { return nil, err } @@ -132,8 +131,8 @@ func Environment(key string) *configSource { return &configSource{ skip: jsonString == "", from: fmt.Sprintf("'%s' environment variable (as JSON)", key), - provider: func(baseConfig *config.HoardConfig) (*config.HoardConfig, error) { - conf, err := config.HoardConfigFromJSONString(jsonString) + provider: func(baseConfig *HoardConfig) (*HoardConfig, error) { + conf, err := HoardConfigFromJSONString(jsonString) if err != nil { return nil, err } @@ -145,13 +144,13 @@ func Environment(key string) *configSource { func Default() *configSource { return &configSource{ from: "defaults", - provider: func(baseConfig *config.HoardConfig) (*config.HoardConfig, error) { - return config.DefaultHoardConfig, nil + provider: func(baseConfig *HoardConfig) (*HoardConfig, error) { + return DefaultHoardConfig, nil }, } } -func fromFile(configFile string) (*config.HoardConfig, error) { +func fromFile(configFile string) (*HoardConfig, error) { bs, err := readFile(configFile) if err != nil { return nil, fmt.Errorf("could not read config file '%s': %s", @@ -162,10 +161,10 @@ func fromFile(configFile string) (*config.HoardConfig, error) { } configSpec := string(bs) - if hoardConf, err := config.HoardConfigFromTOMLString(configSpec); err == nil { + if hoardConf, err := HoardConfigFromTOMLString(configSpec); err == nil { return hoardConf, err } - return config.HoardConfigFromYAMLString(configSpec) + return HoardConfigFromYAMLString(configSpec) } func readFile(configFile string) ([]byte, error) { diff --git a/config/source/source_test.go b/config/source_test.go similarity index 67% rename from config/source/source_test.go rename to config/source_test.go index a99e640f..d6f5da16 100644 --- a/config/source/source_test.go +++ b/config/source_test.go @@ -1,39 +1,35 @@ -package source +package config import ( + "io/ioutil" "os" "testing" - "io/ioutil" - - "github.com/monax/hoard/v4/config" - "github.com/monax/hoard/v4/config/logging" - "github.com/monax/hoard/v4/config/storage" "github.com/stretchr/testify/assert" ) func TestEnvironment(t *testing.T) { - jsonString := config.DefaultHoardConfig.JSONString() + jsonString := DefaultHoardConfig.JSONString() os.Setenv(DefaultJSONConfigEnvironmentVariable, jsonString) - conf, err := Environment(DefaultJSONConfigEnvironmentVariable).Get(&config.HoardConfig{}) + conf, err := Environment(DefaultJSONConfigEnvironmentVariable).Get(&HoardConfig{}) assert.NoError(t, err) assert.Equal(t, jsonString, conf.JSONString()) } func TestTOMLFile(t *testing.T) { - tomlString := config.DefaultHoardConfig.TOMLString() + tomlString := DefaultHoardConfig.TOMLString() file := writeConfigFile(t, tomlString) defer os.Remove(file) - conf, err := File(file).Get(&config.HoardConfig{}) + conf, err := File(file).Get(&HoardConfig{}) assert.NoError(t, err) assert.Equal(t, tomlString, conf.TOMLString()) } func TestYAMLFile(t *testing.T) { - yamlString := config.DefaultHoardConfig.YAMLString() + yamlString := DefaultHoardConfig.YAMLString() file := writeConfigFile(t, yamlString) defer os.Remove(file) - conf, err := File(file).Get(&config.HoardConfig{}) + conf, err := File(file).Get(&HoardConfig{}) assert.NoError(t, err) assert.Equal(t, yamlString, conf.YAMLString()) } @@ -42,27 +38,26 @@ func TestCascade(t *testing.T) { // Both fall through so baseConfig returned conf, err := Cascade(os.Stderr, true, Environment(DefaultJSONConfigEnvironmentVariable), - File("")).Get(config.DefaultHoardConfig) + File("")).Get(DefaultHoardConfig) assert.NoError(t, err) - assert.Equal(t, *config.DefaultHoardConfig, *conf) + assert.Equal(t, *DefaultHoardConfig, *conf) // Env not set so falls through to file - fileConfig := config.DefaultHoardConfig.TOMLString() + fileConfig := DefaultHoardConfig.TOMLString() file := writeConfigFile(t, fileConfig) defer os.Remove(file) conf, err = Cascade(os.Stderr, true, Environment(DefaultJSONConfigEnvironmentVariable), - File(file)).Get(&config.HoardConfig{}) + File(file)).Get(&HoardConfig{}) assert.NoError(t, err) assert.Equal(t, fileConfig, conf.TOMLString()) // Env set so caught by environment source - envConfig := config.NewHoardConfig("unix:///tmp/hoard.sock'", storage.DefaultCloudConfig("aws"), - logging.DefaultConfig) + envConfig := NewHoardConfig("unix:///tmp/hoard.sock'", DefaultCloud("aws"), DefaultLogging) os.Setenv(DefaultJSONConfigEnvironmentVariable, envConfig.JSONString()) conf, err = Cascade(os.Stderr, true, Environment(DefaultJSONConfigEnvironmentVariable), - File(file)).Get(config.DefaultHoardConfig) + File(file)).Get(DefaultHoardConfig) assert.NoError(t, err) assert.Equal(t, envConfig.TOMLString(), conf.TOMLString()) } diff --git a/config/storage.go b/config/storage.go new file mode 100644 index 00000000..60fd6733 --- /dev/null +++ b/config/storage.go @@ -0,0 +1,97 @@ +package config + +import ( + "fmt" + + "github.com/monax/hoard/v4/stores" + + "bytes" + + "github.com/BurntSushi/toml" +) + +const DefaultAddressEncodingName = stores.Base64EncodingName + +var DefaultStorage = NewMemory(DefaultAddressEncodingName) + +type StorageType string + +const ( + Unspecified StorageType = "" + Memory StorageType = "memory" + Filesystem StorageType = "filesystem" + AWS StorageType = "aws" + Azure StorageType = "azure" + GCP StorageType = "gcp" + IPFS StorageType = "ipfs" +) + +// Storage identifies the configured back-end +type Storage struct { + // Acts a string enum + StorageType StorageType + // Address encoding name + AddressEncoding string + // Embedding a pointer to each type of config struct allows us to access the + // relevant one, while at the same time those that are left as nil will be + // omitted from being serialised. + *FileSystemConfig + *Cloud + *IPFSConfig +} + +func NewStorage(storageType StorageType, addressEncoding string) *Storage { + return &Storage{ + StorageType: storageType, + AddressEncoding: addressEncoding, + } +} + +func GetStorageTypes() []StorageType { + return []StorageType{ + Memory, + Filesystem, + AWS, + Azure, + GCP, + IPFS, + } +} + +func GetDefaultStorage(c string) (*Storage, error) { + switch StorageType(c) { + case Memory, Unspecified: + return DefaultMemory(), nil + case Filesystem: + return DefaultFileSystemConfig(), nil + case IPFS: + return DefaultIPFSConfig(), nil + case AWS: + return DefaultCloud(c), nil + case Azure: + return DefaultCloud(c), nil + case GCP: + return DefaultCloud(c), nil + default: + return nil, fmt.Errorf("did not recognise storage type '%s'", c) + } +} + +func ConfigFromString(tomlString string) (*Storage, error) { + storageConfig := new(Storage) + _, err := toml.Decode(tomlString, storageConfig) + if err != nil { + return nil, err + } + return storageConfig, nil +} + +func (storageConfig *Storage) TOMLString() string { + buf := new(bytes.Buffer) + encoder := toml.NewEncoder(buf) + err := encoder.Encode(storageConfig) + if err != nil { + return fmt.Sprintf("") + } + return buf.String() +} diff --git a/config/storage/cloud_test.go b/config/storage/cloud_test.go deleted file mode 100644 index 29b5a1df..00000000 --- a/config/storage/cloud_test.go +++ /dev/null @@ -1,9 +0,0 @@ -package storage - -import "testing" - -func TestDefaultCloudConfig(t *testing.T) { - assertStorageConfigSerialisation(t, DefaultCloudConfig("aws")) - assertStorageConfigSerialisation(t, DefaultCloudConfig("azure")) - assertStorageConfigSerialisation(t, DefaultCloudConfig("gcp")) -} diff --git a/config/storage/memory.go b/config/storage/memory.go deleted file mode 100644 index 7dd96c75..00000000 --- a/config/storage/memory.go +++ /dev/null @@ -1,9 +0,0 @@ -package storage - -func NewMemoryConfig(addressEncoding string) *StorageConfig { - return NewStorageConfig(Memory, addressEncoding) -} - -func DefaultMemoryConfig() *StorageConfig { - return NewMemoryConfig(DefaultAddressEncodingName) -} diff --git a/config/storage/memory_test.go b/config/storage/memory_test.go deleted file mode 100644 index a00536c7..00000000 --- a/config/storage/memory_test.go +++ /dev/null @@ -1,7 +0,0 @@ -package storage - -import "testing" - -func TestDefaultMemoryConfig(t *testing.T) { - assertStorageConfigSerialisation(t, DefaultMemoryConfig()) -} diff --git a/config/storage/storage.go b/config/storage/storage.go deleted file mode 100644 index 5922d844..00000000 --- a/config/storage/storage.go +++ /dev/null @@ -1,160 +0,0 @@ -package storage - -import ( - "fmt" - - "bytes" - - "errors" - - "github.com/BurntSushi/toml" - "github.com/go-kit/kit/log" - "github.com/monax/hoard/v4/storage" -) - -const DefaultAddressEncodingName = storage.Base64EncodingName - -var DefaultConfig = NewMemoryConfig(DefaultAddressEncodingName) - -type StorageType string - -const ( - Unspecified StorageType = "" - Memory StorageType = "memory" - Filesystem StorageType = "filesystem" - AWS StorageType = "aws" - Azure StorageType = "azure" - GCP StorageType = "gcp" - IPFS StorageType = "ipfs" -) - -// StorageConfig identifies the configured back-end -type StorageConfig struct { - // Acts a string enum - StorageType StorageType - // Address encoding name - AddressEncoding string - // Embedding a pointer to each type of config struct allows us to access the - // relevant one, while at the same time those that are left as nil will be - // omitted from being serialised. - *FileSystemConfig - *CloudConfig - *IPFSConfig -} - -func NewStorageConfig(storageType StorageType, addressEncoding string) *StorageConfig { - return &StorageConfig{ - StorageType: storageType, - AddressEncoding: addressEncoding, - } -} - -func GetStorageTypes() []StorageType { - return []StorageType{ - Memory, - Filesystem, - AWS, - Azure, - GCP, - IPFS, - } -} - -func GetDefaultConfig(c string) (*StorageConfig, error) { - switch StorageType(c) { - case Memory, Unspecified: - return DefaultMemoryConfig(), nil - case Filesystem: - return DefaultFileSystemConfig(), nil - case IPFS: - return DefaultIPFSConfig(), nil - case AWS: - return DefaultCloudConfig(c), nil - case Azure: - return DefaultCloudConfig(c), nil - case GCP: - return DefaultCloudConfig(c), nil - default: - return nil, fmt.Errorf("did not recognise storage type '%s'", c) - } -} - -func StoreFromStorageConfig(storageConfig *StorageConfig, logger log.Logger) (storage.NamedStore, error) { - addressEncoding, err := storage.GetAddressEncoding(storageConfig.AddressEncoding) - if err != nil { - return nil, err - } - - switch storageConfig.StorageType { - case Memory, Unspecified: - return storage.NewMemoryStore(), nil - - case Filesystem: - fsConf := storageConfig.FileSystemConfig - if fsConf == nil { - return nil, errors.New("filesystem storage configuration must be " + - "supplied to use the filesystem storage backend") - } - if fsConf.RootDirectory == "" { - return nil, errors.New("rootDirectory key must be non-empty in " + - "filesystem storage config") - } - return storage.NewFileSystemStore(fsConf.RootDirectory, addressEncoding) - - case IPFS: - ipfsConf := storageConfig.IPFSConfig - if ipfsConf == nil { - return nil, errors.New("IPFS storage configuration must be " + - "supplied to use the filesystem storage backend") - } - if ipfsConf.RemoteAPI == "" { - return nil, errors.New("http api url must be non-empty in " + - "ipfs storage config") - } - return storage.NewIPFSStore(ipfsConf.RemoteAPI, addressEncoding) - - case AWS: - awsConf := storageConfig.CloudConfig - if awsConf == nil { - return nil, errors.New("aws configuration must be supplied") - } - return storage.NewCloudStore(storage.CloudType(AWS), awsConf.Bucket, awsConf.Prefix, awsConf.Region, addressEncoding, logger) - - case Azure: - azureConf := storageConfig.CloudConfig - if azureConf == nil { - return nil, errors.New("azure configuration must be supplied") - } - return storage.NewCloudStore(storage.CloudType(Azure), azureConf.Bucket, azureConf.Prefix, azureConf.Region, addressEncoding, logger) - - case GCP: - gcpConf := storageConfig.CloudConfig - if gcpConf == nil { - return nil, errors.New("gcp configuration must be supplied") - } - return storage.NewCloudStore(storage.CloudType(GCP), gcpConf.Bucket, gcpConf.Prefix, gcpConf.Region, addressEncoding, logger) - - default: - return nil, fmt.Errorf("did not recognise storage type '%s'", - storageConfig.StorageType) - } -} - -func ConfigFromString(tomlString string) (*StorageConfig, error) { - storageConfig := new(StorageConfig) - _, err := toml.Decode(tomlString, storageConfig) - if err != nil { - return nil, err - } - return storageConfig, nil -} - -func (storageConfig *StorageConfig) TOMLString() string { - buf := new(bytes.Buffer) - encoder := toml.NewEncoder(buf) - err := encoder.Encode(storageConfig) - if err != nil { - return fmt.Sprintf("") - } - return buf.String() -} diff --git a/config/storage/storage_test.go b/config/storage_test.go similarity index 91% rename from config/storage/storage_test.go rename to config/storage_test.go index 5811e8fb..50484460 100644 --- a/config/storage/storage_test.go +++ b/config/storage_test.go @@ -1,4 +1,4 @@ -package storage +package config import ( "testing" @@ -7,7 +7,7 @@ import ( ) func assertStorageConfigSerialisation(t *testing.T, - storageConfig *StorageConfig) { + storageConfig *Storage) { // We are panicking on serialisation errors so check here defer func() { if r := recover(); r != nil { diff --git a/grant/grant.go b/grant/grant.go index 609439ac..bffe40e1 100644 --- a/grant/grant.go +++ b/grant/grant.go @@ -3,12 +3,12 @@ package grant import ( "fmt" - "github.com/monax/hoard/v4/config/secrets" + "github.com/monax/hoard/v4/config" "github.com/monax/hoard/v4/reference" ) // Seal this reference into a Grant as specified by Spec -func Seal(secret secrets.Manager, ref *reference.Ref, spec *Spec) (*Grant, error) { +func Seal(secret config.SecretsManager, ref *reference.Ref, spec *Spec) (*Grant, error) { grt := &Grant{Spec: spec} if s := spec.GetPlaintext(); s != nil { @@ -37,7 +37,7 @@ func Seal(secret secrets.Manager, ref *reference.Ref, spec *Spec) (*Grant, error } // Unseal a Grant exposing its secret reference -func Unseal(secret secrets.Manager, grt *Grant) (*reference.Ref, error) { +func Unseal(secret config.SecretsManager, grt *Grant) (*reference.Ref, error) { if s := grt.Spec.GetPlaintext(); s != nil { return PlaintextReference(grt.EncryptedReference), nil } else if s := grt.Spec.GetSymmetric(); s != nil { diff --git a/grant/grant_test.go b/grant/grant_test.go index c483cca6..ae2c467a 100644 --- a/grant/grant_test.go +++ b/grant/grant_test.go @@ -4,7 +4,7 @@ import ( "io/ioutil" "testing" - "github.com/monax/hoard/v4/config/secrets" + "github.com/monax/hoard/v4/config" "github.com/monax/hoard/v4/reference" "github.com/stretchr/testify/assert" ) @@ -15,12 +15,12 @@ func TestGrants(t *testing.T) { keyPrivate, err := ioutil.ReadFile("private.key.asc") assert.NoError(t, err) - testPGP := secrets.OpenPGPSecret{ + testPGP := config.OpenPGPSecret{ PrivateID: "10449759736975846181", Data: keyPrivate, } - testSecrets := secrets.Manager{ + testSecrets := config.SecretsManager{ Provider: func(_ string) ([]byte, error) { return nil, nil }, diff --git a/grant/openpgp.go b/grant/openpgp.go index da8c6c7f..6c586073 100644 --- a/grant/openpgp.go +++ b/grant/openpgp.go @@ -5,18 +5,19 @@ import ( "fmt" "strconv" + "github.com/monax/hoard/v4/config" + "bytes" "io" "io/ioutil" - "github.com/monax/hoard/v4/config/secrets" "github.com/monax/hoard/v4/reference" "golang.org/x/crypto/openpgp" "golang.org/x/crypto/openpgp/armor" ) // OpenPGPGrant encrypts and signs a given reference -func OpenPGPGrant(ref *reference.Ref, public string, keyring *secrets.OpenPGPSecret) ([]byte, error) { +func OpenPGPGrant(ref *reference.Ref, public string, keyring *config.OpenPGPSecret) ([]byte, error) { if keyring == nil { return nil, fmt.Errorf("cannot encrypt because no private key was provided") } @@ -73,7 +74,7 @@ func OpenPGPGrant(ref *reference.Ref, public string, keyring *secrets.OpenPGPSec } // OpenPGPReference decrypts a given grant -func OpenPGPReference(grant []byte, keyring *secrets.OpenPGPSecret) (*reference.Ref, error) { +func OpenPGPReference(grant []byte, keyring *config.OpenPGPSecret) (*reference.Ref, error) { if keyring == nil { return nil, fmt.Errorf("cannot decrypt because no private key was provided") } diff --git a/grant/openpgp_test.go b/grant/openpgp_test.go index 17d88f5e..c982f494 100644 --- a/grant/openpgp_test.go +++ b/grant/openpgp_test.go @@ -4,7 +4,8 @@ import ( "io/ioutil" "testing" - "github.com/monax/hoard/v4/config/secrets" + "github.com/monax/hoard/v4/config" + "github.com/stretchr/testify/assert" ) @@ -16,7 +17,7 @@ func TestOpenPGPGrant(t *testing.T) { keyPrivate, err := ioutil.ReadFile("private.key.asc") assert.NoError(t, err) - testPGP := secrets.OpenPGPSecret{ + testPGP := config.OpenPGPSecret{ PrivateID: "10449759736975846181", Data: keyPrivate, } diff --git a/services/grpc_service.go b/grpc_service.go similarity index 68% rename from services/grpc_service.go rename to grpc_service.go index 5cef05d8..dd66e211 100644 --- a/services/grpc_service.go +++ b/grpc_service.go @@ -1,94 +1,94 @@ -package services +package hoard import ( "context" - "github.com/monax/hoard/v4" + "github.com/monax/hoard/v4/api" "github.com/monax/hoard/v4/grant" "github.com/monax/hoard/v4/reference" - "github.com/monax/hoard/v4/storage" + "github.com/monax/hoard/v4/stores" ) // Here we implement the GRPC Hoard service. It should mostly be plumbing to // a DeterministicEncryptedStore (for which hoard.hoard is the canonical example) // and also to Grants. type grpcService struct { - des hoard.DeterministicEncryptedStore - gs hoard.GrantService + des DeterministicEncryptedStore + gs GrantService } -func NewHoardServer(des hoard.DeterministicEncryptedStore, gs hoard.GrantService) *grpcService { +func NewServer(des DeterministicEncryptedStore, gs GrantService) *grpcService { return &grpcService{ des: des, gs: gs, } } -func (service *grpcService) Get(ctx context.Context, ref *reference.Ref) (*Plaintext, error) { +func (service *grpcService) Get(ctx context.Context, ref *reference.Ref) (*api.Plaintext, error) { data, err := service.des.Get(ref) if err != nil { return nil, err } - return &Plaintext{ + return &api.Plaintext{ Data: data, Salt: ref.Salt, }, nil } -func (service *grpcService) Put(ctx context.Context, plaintext *Plaintext) (*reference.Ref, error) { +func (service *grpcService) Put(ctx context.Context, plaintext *api.Plaintext) (*reference.Ref, error) { return service.des.Put(plaintext.Data, plaintext.Salt) } -func (service *grpcService) Encrypt(ctx context.Context, plaintext *Plaintext) (*ReferenceAndCiphertext, error) { +func (service *grpcService) Encrypt(ctx context.Context, plaintext *api.Plaintext) (*api.ReferenceAndCiphertext, error) { ref, encryptedData, err := service.des.Encrypt(plaintext.Data, plaintext.Salt) if err != nil { return nil, err } - return &ReferenceAndCiphertext{ + return &api.ReferenceAndCiphertext{ Reference: ref, - Ciphertext: &Ciphertext{ + Ciphertext: &api.Ciphertext{ EncryptedData: encryptedData, }, }, nil } -func (service *grpcService) Decrypt(ctx context.Context, refAndCiphertext *ReferenceAndCiphertext) (*Plaintext, error) { +func (service *grpcService) Decrypt(ctx context.Context, refAndCiphertext *api.ReferenceAndCiphertext) (*api.Plaintext, error) { data, err := service.des.Decrypt(refAndCiphertext.Reference, refAndCiphertext.Ciphertext.EncryptedData) if err != nil { return nil, err } - return &Plaintext{ + return &api.Plaintext{ Data: data, Salt: refAndCiphertext.Reference.Salt, }, nil } // StorageServer -func (service *grpcService) Push(ctx context.Context, ciphertext *Ciphertext) (*Address, error) { +func (service *grpcService) Push(ctx context.Context, ciphertext *api.Ciphertext) (*api.Address, error) { address, err := service.des.Store().Put(ciphertext.EncryptedData) if err != nil { return nil, err } - return &Address{ + return &api.Address{ Address: address, }, nil } -func (service *grpcService) Pull(ctx context.Context, address *Address) (*Ciphertext, error) { +func (service *grpcService) Pull(ctx context.Context, address *api.Address) (*api.Ciphertext, error) { // Get from the underlying store encryptedData, err := service.des.Store().Get(address.Address) if err != nil { return nil, err } - return &Ciphertext{ + return &api.Ciphertext{ EncryptedData: encryptedData, }, nil } -func (service *grpcService) Stat(ctx context.Context, address *Address) (*storage.StatInfo, error) { +func (service *grpcService) Stat(ctx context.Context, address *api.Address) (*stores.StatInfo, error) { statInfo, err := service.des.Store().Stat(address.Address) if err != nil { return nil, err @@ -102,7 +102,7 @@ func (service *grpcService) Stat(ctx context.Context, address *Address) (*storag // GrantServer -func (service *grpcService) Seal(ctx context.Context, arg *ReferenceAndGrantSpec) (*grant.Grant, error) { +func (service *grpcService) Seal(ctx context.Context, arg *api.ReferenceAndGrantSpec) (*grant.Grant, error) { return service.gs.Seal(arg.Reference, arg.GrantSpec) } @@ -110,7 +110,7 @@ func (service *grpcService) Unseal(ctx context.Context, grt *grant.Grant) (*refe return service.gs.Unseal(grt) } -func (service *grpcService) Reseal(ctx context.Context, arg *GrantAndGrantSpec) (*grant.Grant, error) { +func (service *grpcService) Reseal(ctx context.Context, arg *api.GrantAndGrantSpec) (*grant.Grant, error) { ref, err := service.gs.Unseal(arg.Grant) if err != nil { return nil, err @@ -118,7 +118,7 @@ func (service *grpcService) Reseal(ctx context.Context, arg *GrantAndGrantSpec) return service.gs.Seal(ref, arg.GrantSpec) } -func (service *grpcService) PutSeal(ctx context.Context, arg *PlaintextAndGrantSpec) (*grant.Grant, error) { +func (service *grpcService) PutSeal(ctx context.Context, arg *api.PlaintextAndGrantSpec) (*grant.Grant, error) { ref, err := service.des.Put(arg.Plaintext.Data, arg.Plaintext.Salt) if err != nil { return nil, err @@ -126,7 +126,7 @@ func (service *grpcService) PutSeal(ctx context.Context, arg *PlaintextAndGrantS return service.gs.Seal(ref, arg.GrantSpec) } -func (service *grpcService) UnsealGet(ctx context.Context, grt *grant.Grant) (*Plaintext, error) { +func (service *grpcService) UnsealGet(ctx context.Context, grt *grant.Grant) (*api.Plaintext, error) { ref, err := service.gs.Unseal(grt) if err != nil { return nil, err diff --git a/hoard-js/index.js b/hoard-js/index.js index bd3fb3cd..28b57d6d 100644 --- a/hoard-js/index.js +++ b/hoard-js/index.js @@ -4,7 +4,7 @@ const path = require('path') const LOCAL_PATH = path.join(__dirname, './protobuf'); const PROTO_PATH = path.join(__dirname, '../protobuf'); -const PROTO_FILE = 'services.proto'; +const PROTO_FILE = 'api.proto'; const protoLoader = require('@grpc/proto-loader'); const grpc = require('grpc') @@ -169,4 +169,4 @@ HoardClient.prototype.base64ify = function (obj) { }; HoardClientDynamic.prototype = Object.create(HoardClient.prototype); -module.exports.Client = HoardClientDynamic; \ No newline at end of file +module.exports.Client = HoardClientDynamic; diff --git a/hoard.go b/hoard.go index 5f79adca..78039ada 100644 --- a/hoard.go +++ b/hoard.go @@ -4,11 +4,11 @@ import ( "crypto/sha256" "github.com/go-kit/kit/log" - "github.com/monax/hoard/v4/config/secrets" + "github.com/monax/hoard/v4/config" "github.com/monax/hoard/v4/encryption" "github.com/monax/hoard/v4/grant" "github.com/monax/hoard/v4/reference" - "github.com/monax/hoard/v4/storage" + "github.com/monax/hoard/v4/stores" ) type DeterministicEncryptor interface { @@ -26,7 +26,7 @@ type DeterministicEncryptedStore interface { // Encrypt data and put it in underlying storage Put(data, salt []byte) (*reference.Ref, error) // Get the underlying ContentAddressedStore - Store() storage.ContentAddressedStore + Store() stores.ContentAddressedStore } type GrantService interface { @@ -42,20 +42,20 @@ type GrantService interface { // hoard.proto interface. type Hoard struct { name string - store storage.ContentAddressedStore - secrets secrets.Manager + store stores.ContentAddressedStore + secrets config.SecretsManager logger log.Logger } -func NewHoard(store storage.NamedStore, secrets secrets.Manager, logger log.Logger) *Hoard { +func NewHoard(store stores.NamedStore, secrets config.SecretsManager, logger log.Logger) *Hoard { if logger == nil { logger = log.NewNopLogger() } return &Hoard{ name: store.Name(), - store: storage.NewContentAddressedStore(storage.MakeAddresser(sha256.New), - storage.NewLoggingStore(storage.NewSyncStore(store), logger)), + store: stores.NewContentAddressedStore(stores.MakeAddresser(sha256.New), + stores.NewLoggingStore(stores.NewSyncStore(store), logger)), secrets: secrets, logger: log.With(logger, "scope", "NewHoard"), } @@ -87,7 +87,7 @@ func (hrd *Hoard) Get(ref *reference.Ref) ([]byte, error) { return data, nil } -// Encrypts data and stores it in underlying store and returns the address +// Encrypts data and storage it in underlying store and returns the address func (hrd *Hoard) Put(data, salt []byte) (*reference.Ref, error) { blob, err := encryption.EncryptConvergent(data, salt) if err != nil { @@ -120,6 +120,6 @@ func (hrd *Hoard) Decrypt(ref *reference.Ref, encryptedData []byte) ([]byte, err return data, nil } -func (hrd *Hoard) Store() storage.ContentAddressedStore { +func (hrd *Hoard) Store() stores.ContentAddressedStore { return hrd.store } diff --git a/hoard_test.go b/hoard_test.go index d7722852..f3828a9b 100644 --- a/hoard_test.go +++ b/hoard_test.go @@ -3,16 +3,15 @@ package hoard import ( "testing" - "github.com/monax/hoard/v4/config/secrets" - "github.com/go-kit/kit/log" + "github.com/monax/hoard/v4/config" "github.com/monax/hoard/v4/reference" - "github.com/monax/hoard/v4/storage" + "github.com/monax/hoard/v4/stores" "github.com/stretchr/testify/assert" ) func TestDeterministicEncryptedStore(t *testing.T) { - hrd := NewHoard(storage.NewMemoryStore(), secrets.NoopSecretManager, log.NewNopLogger()) + hrd := NewHoard(stores.NewMemoryStore(), config.NoopSecretManager, log.NewNopLogger()) bunsIn := bs("hot buns") ref, err := hrd.Put(bunsIn, nil) diff --git a/project/history.go b/project/history.go index 4df79275..365b705b 100644 --- a/project/history.go +++ b/project/history.go @@ -33,7 +33,7 @@ var History relic.ImmutableHistory = relic.NewHistory("Monax Hoard", "https://gi ``, "4.0.0 - 2019-05-21", `###Fixed -- [BUILD] Change hoard.pb.go to services/services.pb.go +- [BUILD] Change hoard.pb.go to services/api.pb.go `, "3.2.1 - 2019-04-24", `### Fixed diff --git a/protobuf/services.proto b/protobuf/api.proto similarity index 94% rename from protobuf/services.proto rename to protobuf/api.proto index 6a3e5c41..3ad7a3d9 100644 --- a/protobuf/services.proto +++ b/protobuf/api.proto @@ -2,11 +2,11 @@ syntax = "proto3"; import "reference.proto"; import "grant.proto"; -import "storage.proto"; +import "stores.proto"; -package services; +package api; -option go_package = "github.com/monax/hoard/v4/services"; +option go_package = "github.com/monax/hoard/v4/api"; service Grant { // Seal a Reference to create a Grant @@ -58,7 +58,7 @@ service Storage { // Get some information about the encrypted blob stored at an address, // including whether it exists. - rpc Stat (Address) returns (storage.StatInfo); + rpc Stat (Address) returns (stores.StatInfo); } message GrantAndGrantSpec { diff --git a/protobuf/storage.proto b/protobuf/stores.proto similarity index 88% rename from protobuf/storage.proto rename to protobuf/stores.proto index 64ce5638..77da5162 100644 --- a/protobuf/storage.proto +++ b/protobuf/stores.proto @@ -1,8 +1,8 @@ syntax = "proto3"; -package storage; +package stores; -option go_package = "github.com/monax/hoard/v4/storage"; +option go_package = "github.com/monax/hoard/v4/stores"; message StatInfo { // The address will be the same as the one passed in but is repeated to diff --git a/server/server.go b/server/server.go index 56c73054..9ff5c46a 100644 --- a/server/server.go +++ b/server/server.go @@ -8,11 +8,11 @@ import ( "github.com/go-kit/kit/log" "github.com/monax/hoard/v4" - "github.com/monax/hoard/v4/config/secrets" + "github.com/monax/hoard/v4/api" + "github.com/monax/hoard/v4/config" "github.com/monax/hoard/v4/logging" "github.com/monax/hoard/v4/logging/loggers" - "github.com/monax/hoard/v4/services" - "github.com/monax/hoard/v4/storage" + "github.com/monax/hoard/v4/stores" "google.golang.org/grpc" "google.golang.org/grpc/reflection" ) @@ -26,7 +26,7 @@ type Server struct { logger log.Logger } -func New(listenURL string, store storage.NamedStore, secretManager secrets.Manager, logger log.Logger) *Server { +func New(listenURL string, store stores.NamedStore, secretManager config.SecretsManager, logger log.Logger) *Server { return &Server{ listenURL: listenURL, hoard: hoard.NewHoard(store, secretManager, logger), @@ -55,11 +55,11 @@ func (serv *Server) Serve() error { logging.InfoMsg(serv.logger, "Initialising Hoard server", "store_name", serv.hoard.Name()) - hoardServer := services.NewHoardServer(serv.hoard, serv.hoard) - services.RegisterCleartextServer(serv.grpcServer, hoardServer) - services.RegisterEncryptionServer(serv.grpcServer, hoardServer) - services.RegisterStorageServer(serv.grpcServer, hoardServer) - services.RegisterGrantServer(serv.grpcServer, hoardServer) + hoardServer := hoard.NewServer(serv.hoard, serv.hoard) + api.RegisterCleartextServer(serv.grpcServer, hoardServer) + api.RegisterEncryptionServer(serv.grpcServer, hoardServer) + api.RegisterStorageServer(serv.grpcServer, hoardServer) + api.RegisterGrantServer(serv.grpcServer, hoardServer) // Register reflection service on gRPC server. reflection.Register(serv.grpcServer) // Announce ready diff --git a/storage/memory_test.go b/storage/memory_test.go deleted file mode 100644 index b9a6385e..00000000 --- a/storage/memory_test.go +++ /dev/null @@ -1,7 +0,0 @@ -package storage - -import "testing" - -func TestMemoryStore(t *testing.T) { - testStore(t, NewMemoryStore()) -} diff --git a/storage/address_encoding.go b/stores/address_encoding.go similarity index 98% rename from storage/address_encoding.go rename to stores/address_encoding.go index 1097dd71..dcd113cd 100644 --- a/storage/address_encoding.go +++ b/stores/address_encoding.go @@ -1,4 +1,4 @@ -package storage +package stores import ( "encoding/base32" diff --git a/storage/cloud.go b/stores/cloud/cloud.go similarity index 90% rename from storage/cloud.go rename to stores/cloud/cloud.go index 1e5fee87..f689dc70 100644 --- a/storage/cloud.go +++ b/stores/cloud/cloud.go @@ -1,4 +1,4 @@ -package storage +package cloud import ( "bytes" @@ -8,6 +8,8 @@ import ( "os" "strings" + "github.com/monax/hoard/v4/stores" + "github.com/Azure/azure-storage-blob-go/azblob" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" @@ -23,12 +25,12 @@ import ( "golang.org/x/oauth2/google" ) -type CloudType string +type Type string const ( - AWS CloudType = "aws" - Azure CloudType = "azure" - GCP CloudType = "gcp" + AWS Type = "aws" + Azure Type = "azure" + GCP Type = "gcp" ) type cloudStore struct { @@ -36,11 +38,11 @@ type cloudStore struct { blob *blob.Bucket bucket string prefix string - encoding AddressEncoding + encoding stores.AddressEncoding logger log.Logger } -func NewCloudStore(cloud CloudType, bucket, prefix, region string, addrenc AddressEncoding, logger log.Logger) (*cloudStore, error) { +func NewStore(cloud Type, bucket, prefix, region string, addrenc stores.AddressEncoding, logger log.Logger) (*cloudStore, error) { if logger == nil { logger = log.NewNopLogger() } @@ -151,10 +153,10 @@ func (inv *cloudStore) Get(address []byte) ([]byte, error) { return buf.Bytes(), nil } -func (inv *cloudStore) Stat(address []byte) (*StatInfo, error) { +func (inv *cloudStore) Stat(address []byte) (*stores.StatInfo, error) { reader, err := inv.blob.NewReader(inv.back, fmt.Sprintf("%s/%s", inv.prefix, inv.encode(address)), nil) if err != nil { - return &StatInfo{ + return &stores.StatInfo{ Exists: false, }, nil } @@ -167,7 +169,7 @@ func (inv *cloudStore) Stat(address []byte) (*StatInfo, error) { inv.logger.Log("method", "Stat", "encoded_address", inv.encode(address)) - return &StatInfo{ + return &stores.StatInfo{ Exists: true, Size_: uint64(n), }, nil diff --git a/storage/cloud_test.go b/stores/cloud/cloud_test.go similarity index 78% rename from storage/cloud_test.go rename to stores/cloud/cloud_test.go index aaed6a3f..fd2018eb 100644 --- a/storage/cloud_test.go +++ b/stores/cloud/cloud_test.go @@ -1,41 +1,39 @@ // +build integration -package storage +package cloud import ( + "context" "encoding/base32" "os" "testing" - "cloud.google.com/go/storage" - "google.golang.org/api/option" - - "context" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/monax/hoard/v4/stores" "github.com/stretchr/testify/assert" "golang.org/x/oauth2/google" + "google.golang.org/api/option" ) func TestStoreGCS(t *testing.T) { bucket := "monax-hoard" prefix := "test-store" deleteGCSPrefix(bucket, prefix) - store, err := NewCloudStore(GCP, bucket, prefix, "", base32.StdEncoding, nil) + store, err := NewStore(GCP, bucket, prefix, "", base32.StdEncoding, nil) assert.NoError(t, err) - testStore(t, store) + stores.RunTests(t, store) } func TestStoreS3(t *testing.T) { bucket := "monax-hoard-test" prefix := "TestS3Store/" deleteS3Prefix(bucket, prefix) - store, err := NewCloudStore(AWS, bucket, prefix, "", base32.StdEncoding, nil) + store, err := NewStore(AWS, bucket, prefix, "", base32.StdEncoding, nil) assert.NoError(t, err) - testStore(t, store) + stores.RunTests(t, store) } func deleteS3Prefix(bucket, prefix string) { @@ -55,14 +53,14 @@ func deleteGCSPrefix(bucket, prefix string) { panic(err) } - client, err := storage.NewClient(ctx, option.WithCredentials(creds)) + client, err := cloudstores.NewClient(ctx, option.WithCredentials(creds)) if err != nil { panic(err) } defer client.Close() bkt := client.Bucket(bucket) - objs := bkt.Objects(ctx, &storage.Query{Prefix: prefix}) + objs := bkt.Objects(ctx, &cloudstores.Query{Prefix: prefix}) for obj, _ := objs.Next(); obj != nil; obj, _ = objs.Next() { bkt.Object(obj.Name).Delete(ctx) } diff --git a/storage/filesystem.go b/stores/filesystem.go similarity index 98% rename from storage/filesystem.go rename to stores/filesystem.go index 255eb426..f95369dc 100644 --- a/storage/filesystem.go +++ b/stores/filesystem.go @@ -1,4 +1,4 @@ -package storage +package stores import ( "fmt" diff --git a/storage/filesystem_test.go b/stores/filesystem_test.go similarity index 91% rename from storage/filesystem_test.go rename to stores/filesystem_test.go index df09da54..03effcc8 100644 --- a/storage/filesystem_test.go +++ b/stores/filesystem_test.go @@ -1,12 +1,11 @@ -package storage +package stores import ( + "encoding/base64" "io/ioutil" "os" "testing" - "encoding/base64" - "github.com/stretchr/testify/assert" ) @@ -23,5 +22,5 @@ func TestFileSystemStore(t *testing.T) { fss, err := NewFileSystemStore(tempDir, base64.URLEncoding) assert.NoError(t, err) - testStore(t, fss) + RunTests(t, fss) } diff --git a/storage/ipfs.go b/stores/ipfs/ipfs.go similarity index 87% rename from storage/ipfs.go rename to stores/ipfs/ipfs.go index f3bbc1a7..75526df0 100644 --- a/storage/ipfs.go +++ b/stores/ipfs/ipfs.go @@ -1,4 +1,4 @@ -package storage +package ipfs import ( "bytes" @@ -8,14 +8,16 @@ import ( "mime/multipart" "net/http" "strings" + + "github.com/monax/hoard/v4/stores" ) type ipfsStore struct { host string - encoding AddressEncoding + encoding stores.AddressEncoding } -func NewIPFSStore(host string, encoding AddressEncoding) (*ipfsStore, error) { +func NewStore(host string, encoding stores.AddressEncoding) (*ipfsStore, error) { host = fmt.Sprintf("%s/api/v0", strings.TrimRight(host, "/")) _, err := http.Get(host) if err != nil { @@ -77,11 +79,11 @@ func (inv *ipfsStore) Get(address []byte) ([]byte, error) { return body, nil } -func (inv *ipfsStore) Stat(address []byte) (*StatInfo, error) { +func (inv *ipfsStore) Stat(address []byte) (*stores.StatInfo, error) { url := fmt.Sprintf("%s/cat?arg=%s", inv.host, string(address)) resp, err := http.Get(url) if err != nil || resp.StatusCode != 200 { - return &StatInfo{ + return &stores.StatInfo{ Exists: false, }, nil } @@ -92,7 +94,7 @@ func (inv *ipfsStore) Stat(address []byte) (*StatInfo, error) { return nil, err } - return &StatInfo{ + return &stores.StatInfo{ Exists: true, Size_: uint64(len(body)), }, nil diff --git a/storage/ipfs_test.go b/stores/ipfs/ipfs_test.go similarity index 76% rename from storage/ipfs_test.go rename to stores/ipfs/ipfs_test.go index 79c2bb25..efaec422 100644 --- a/storage/ipfs_test.go +++ b/stores/ipfs/ipfs_test.go @@ -1,4 +1,4 @@ -package storage +package ipfs import ( "net/http" @@ -12,7 +12,7 @@ import ( func TestIPFSStore(t *testing.T) { srv := &http.Server{Addr: ":5001"} go srv.ListenAndServe() - inv, err := NewIPFSStore("http://localhost:5001", base64.URLEncoding) + inv, err := NewStore("http://localhost:5001", base64.URLEncoding) assert.NoError(t, err) assert.Equal(t, "http://localhost:5001/api/v0", inv.host) srv.Close() diff --git a/storage/logging_store.go b/stores/logging_store.go similarity index 99% rename from storage/logging_store.go rename to stores/logging_store.go index 0d9f0a35..82e6833e 100644 --- a/storage/logging_store.go +++ b/stores/logging_store.go @@ -1,8 +1,7 @@ -package storage +package stores import ( "encoding/base64" - "fmt" "github.com/go-kit/kit/log" diff --git a/storage/memory.go b/stores/memory.go similarity index 98% rename from storage/memory.go rename to stores/memory.go index f67bf528..f08599fd 100644 --- a/storage/memory.go +++ b/stores/memory.go @@ -1,4 +1,4 @@ -package storage +package stores import ( "fmt" diff --git a/stores/memory_test.go b/stores/memory_test.go new file mode 100644 index 00000000..d488dc79 --- /dev/null +++ b/stores/memory_test.go @@ -0,0 +1,9 @@ +package stores + +import ( + "testing" +) + +func TestMemoryStore(t *testing.T) { + RunTests(t, NewMemoryStore()) +} diff --git a/storage/storage.go b/stores/storage.go similarity index 92% rename from storage/storage.go rename to stores/storage.go index 24eeab1d..c3381a63 100644 --- a/storage/storage.go +++ b/stores/storage.go @@ -1,4 +1,6 @@ -package storage +// Contains core types and logic pertaining to Hoard's backend storage services - but not the implementations of those +// stores to avoid a large number of possibly unwanted dependencies +package stores import ( "encoding/base64" diff --git a/storage/storage.pb.go b/stores/stores.pb.go similarity index 67% rename from storage/storage.pb.go rename to stores/stores.pb.go index 5e2b8c5d..bc60576f 100644 --- a/storage/storage.pb.go +++ b/stores/stores.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: storage.proto +// source: stores.proto -package storage +package stores import ( fmt "fmt" @@ -41,7 +41,7 @@ func (m *StatInfo) Reset() { *m = StatInfo{} } func (m *StatInfo) String() string { return proto.CompactTextString(m) } func (*StatInfo) ProtoMessage() {} func (*StatInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_0d2c4ccf1453ffdb, []int{0} + return fileDescriptor_78bd58a4c4375fca, []int{0} } func (m *StatInfo) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StatInfo.Unmarshal(m, b) @@ -90,22 +90,22 @@ func (m *StatInfo) GetLocation() string { } func init() { - proto.RegisterType((*StatInfo)(nil), "storage.StatInfo") + proto.RegisterType((*StatInfo)(nil), "stores.StatInfo") } -func init() { proto.RegisterFile("storage.proto", fileDescriptor_0d2c4ccf1453ffdb) } +func init() { proto.RegisterFile("stores.proto", fileDescriptor_78bd58a4c4375fca) } -var fileDescriptor_0d2c4ccf1453ffdb = []byte{ - // 168 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x2d, 0x2e, 0xc9, 0x2f, - 0x4a, 0x4c, 0x4f, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x87, 0x72, 0x95, 0x72, 0xb8, - 0x38, 0x82, 0x4b, 0x12, 0x4b, 0x3c, 0xf3, 0xd2, 0xf2, 0x85, 0x24, 0xb8, 0xd8, 0x1d, 0x53, 0x52, - 0x8a, 0x52, 0x8b, 0x8b, 0x25, 0x18, 0x15, 0x18, 0x35, 0x78, 0x82, 0x60, 0x5c, 0x21, 0x31, 0x2e, - 0x36, 0xd7, 0x8a, 0xcc, 0xe2, 0x92, 0x62, 0x09, 0x26, 0x05, 0x46, 0x0d, 0x8e, 0x20, 0x28, 0x4f, - 0x48, 0x88, 0x8b, 0x25, 0x38, 0xb3, 0x2a, 0x55, 0x82, 0x59, 0x81, 0x51, 0x83, 0x25, 0x08, 0xcc, - 0x16, 0x92, 0xe2, 0xe2, 0xf0, 0xc9, 0x4f, 0x4e, 0x2c, 0xc9, 0xcc, 0xcf, 0x93, 0x60, 0x51, 0x60, - 0xd4, 0xe0, 0x0c, 0x82, 0xf3, 0x9d, 0x94, 0xa3, 0x14, 0xd3, 0x33, 0x4b, 0x32, 0x4a, 0x93, 0xf4, - 0x92, 0xf3, 0x73, 0xf5, 0x73, 0xf3, 0xf3, 0x12, 0x2b, 0xf4, 0x33, 0xf2, 0x13, 0x8b, 0x52, 0xf4, - 0xcb, 0x4c, 0xf4, 0xa1, 0x4e, 0x4a, 0x62, 0x03, 0x3b, 0xd1, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff, - 0x49, 0x0a, 0x33, 0x8c, 0xb3, 0x00, 0x00, 0x00, +var fileDescriptor_78bd58a4c4375fca = []byte{ + // 167 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x29, 0x2e, 0xc9, 0x2f, + 0x4a, 0x2d, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x83, 0xf0, 0x94, 0x72, 0xb8, 0x38, + 0x82, 0x4b, 0x12, 0x4b, 0x3c, 0xf3, 0xd2, 0xf2, 0x85, 0x24, 0xb8, 0xd8, 0x1d, 0x53, 0x52, 0x8a, + 0x52, 0x8b, 0x8b, 0x25, 0x18, 0x15, 0x18, 0x35, 0x78, 0x82, 0x60, 0x5c, 0x21, 0x31, 0x2e, 0x36, + 0xd7, 0x8a, 0xcc, 0xe2, 0x92, 0x62, 0x09, 0x26, 0x05, 0x46, 0x0d, 0x8e, 0x20, 0x28, 0x4f, 0x48, + 0x88, 0x8b, 0x25, 0x38, 0xb3, 0x2a, 0x55, 0x82, 0x59, 0x81, 0x51, 0x83, 0x25, 0x08, 0xcc, 0x16, + 0x92, 0xe2, 0xe2, 0xf0, 0xc9, 0x4f, 0x4e, 0x2c, 0xc9, 0xcc, 0xcf, 0x93, 0x60, 0x51, 0x60, 0xd4, + 0xe0, 0x0c, 0x82, 0xf3, 0x9d, 0x94, 0xa2, 0x14, 0xd2, 0x33, 0x4b, 0x32, 0x4a, 0x93, 0xf4, 0x92, + 0xf3, 0x73, 0xf5, 0x73, 0xf3, 0xf3, 0x12, 0x2b, 0xf4, 0x33, 0xf2, 0x13, 0x8b, 0x52, 0xf4, 0xcb, + 0x4c, 0xf4, 0x21, 0x2e, 0x4a, 0x62, 0x03, 0x3b, 0xd0, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff, 0xb4, + 0x7f, 0x21, 0xed, 0xb0, 0x00, 0x00, 0x00, } diff --git a/storage/sync_store.go b/stores/sync_store.go similarity index 99% rename from storage/sync_store.go rename to stores/sync_store.go index b9f526c5..57e33257 100644 --- a/storage/sync_store.go +++ b/stores/sync_store.go @@ -1,4 +1,4 @@ -package storage +package stores import ( "fmt" diff --git a/storage/storage_test.go b/stores/test.go similarity index 97% rename from storage/storage_test.go rename to stores/test.go index 4aab0cc7..dd4b2556 100644 --- a/storage/storage_test.go +++ b/stores/test.go @@ -1,4 +1,4 @@ -package storage +package stores import ( "testing" @@ -14,7 +14,7 @@ import ( const concurrentAccessGoRoutines = 200 // Generic test suite for all Stores -func testStore(t *testing.T, store Store) { +func RunTests(t *testing.T, store Store) { address := bs("address") data := bs("data") getPutGet(t, store, address, data)