From 6d84b7acd2aec37d42661d623c9ed7aea3cec9b0 Mon Sep 17 00:00:00 2001 From: Martin Garton Date: Fri, 11 Oct 2019 21:25:02 +0100 Subject: [PATCH] Move backends to seperate packages This moves the backends each to its own package, so that users do not have to include support for every backend (and the somewhat large dependencies associated with them) even if they are not needed. To enable a backend for url based usage, simply do an anonymous import, just as you would with a sql driver. Note that this represents a breaking API change. Fixes #20 Updates #21 and #22 --- straw_gcs.go => gcs/straw_gcs.go | 20 ++++++++++++--- straw_s3.go => s3/straw_s3.go | 25 ++++++++++++++++--- straw_s3_options.go => s3/straw_s3_options.go | 2 +- straw_sftp.go => sftp/straw_sftp.go | 15 ++++++++--- strawurl.go | 23 +---------------- 5 files changed, 50 insertions(+), 35 deletions(-) rename straw_gcs.go => gcs/straw_gcs.go (92%) rename straw_s3.go => s3/straw_s3.go (92%) rename straw_s3_options.go => s3/straw_s3_options.go (96%) rename straw_sftp.go => sftp/straw_sftp.go (89%) diff --git a/straw_gcs.go b/gcs/straw_gcs.go similarity index 92% rename from straw_gcs.go rename to gcs/straw_gcs.go index 233c509..46628ac 100644 --- a/straw_gcs.go +++ b/gcs/straw_gcs.go @@ -1,9 +1,10 @@ -package straw +package gcs import ( "context" "fmt" "io" + "net/url" "os" "path/filepath" "sort" @@ -11,11 +12,22 @@ import ( "time" "cloud.google.com/go/storage" + "github.com/uw-labs/straw" "google.golang.org/api/iterator" "google.golang.org/api/option" ) -var _ StreamStore = &GCSStreamStore{} +var _ straw.StreamStore = &GCSStreamStore{} + +func init() { + straw.Register("gs", func(u *url.URL) (straw.StreamStore, error) { + creds := u.Query().Get("credentialsfile") + if creds == "" { + return nil, fmt.Errorf("gs URLs must provide a `credentialsfile` parameter") + } + return NewGCSStreamStore(creds, u.Host) + }) +} func NewGCSStreamStore(credentialsFile string, bucket string) (*GCSStreamStore, error) { ctx := context.Background() @@ -131,7 +143,7 @@ func (sr *gcsStatResult) Sys() interface{} { return nil } -func (fs *GCSStreamStore) OpenReadCloser(name string) (StrawReader, error) { +func (fs *GCSStreamStore) OpenReadCloser(name string) (straw.StrawReader, error) { fi, err := fs.Stat(name) if err != nil { return nil, err @@ -233,7 +245,7 @@ func (fs *GCSStreamStore) Remove(name string) error { return fs.client.Bucket(fs.bucket).Object(name).Delete(context.Background()) } -func (fs *GCSStreamStore) CreateWriteCloser(name string) (StrawWriter, error) { +func (fs *GCSStreamStore) CreateWriteCloser(name string) (straw.StrawWriter, error) { name = fs.noSlashPrefix(name) if err := fs.checkParentDir(name); err != nil { diff --git a/straw_s3.go b/s3/straw_s3.go similarity index 92% rename from straw_s3.go rename to s3/straw_s3.go index 7e626e4..abb0725 100644 --- a/straw_s3.go +++ b/s3/straw_s3.go @@ -1,10 +1,11 @@ -package straw +package s3 import ( "fmt" "io" "io/ioutil" "log" + "net/url" "os" "path/filepath" "sort" @@ -16,6 +17,7 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/uw-labs/straw" ) type ServerSideEncryptionType string @@ -25,7 +27,22 @@ const ( ServerSideEncryptionTypeAES256 ServerSideEncryptionType = "AES256" ) -var _ StreamStore = &S3StreamStore{} +var _ straw.StreamStore = &S3StreamStore{} + +func init() { + straw.Register("s3", func(u *url.URL) (straw.StreamStore, error) { + sse := u.Query().Get("sse") + var opts []S3Option + switch sse { + case "": + case "AES256": + opts = append(opts, S3ServerSideEncoding(ServerSideEncryptionTypeAES256)) + default: + return nil, fmt.Errorf("unknown server side encryption type '%s'", sse) + } + return NewS3StreamStore(u.Host, opts...) + }) +} func NewS3StreamStore(bucket string, options ...S3Option) (*S3StreamStore, error) { sess, err := session.NewSessionWithOptions( @@ -159,7 +176,7 @@ func (sr *s3StatResult) Sys() interface{} { return nil } -func (fs *S3StreamStore) OpenReadCloser(name string) (StrawReader, error) { +func (fs *S3StreamStore) OpenReadCloser(name string) (straw.StrawReader, error) { fi, err := fs.Stat(name) if err != nil { return nil, err @@ -299,7 +316,7 @@ func (fs *S3StreamStore) Remove(name string) error { return err } -func (fs *S3StreamStore) CreateWriteCloser(name string) (StrawWriter, error) { +func (fs *S3StreamStore) CreateWriteCloser(name string) (straw.StrawWriter, error) { name = fs.noSlashPrefix(name) if err := fs.checkParentDir(name); err != nil { diff --git a/straw_s3_options.go b/s3/straw_s3_options.go similarity index 96% rename from straw_s3_options.go rename to s3/straw_s3_options.go index 18cc6e3..4e19344 100644 --- a/straw_s3_options.go +++ b/s3/straw_s3_options.go @@ -1,4 +1,4 @@ -package straw +package s3 // S3Option is an option to the s3 backend type S3Option interface { diff --git a/straw_sftp.go b/sftp/straw_sftp.go similarity index 89% rename from straw_sftp.go rename to sftp/straw_sftp.go index bc37ee8..c285c0e 100644 --- a/straw_sftp.go +++ b/sftp/straw_sftp.go @@ -1,4 +1,4 @@ -package straw +package sftp import ( "errors" @@ -12,10 +12,17 @@ import ( "sync" "github.com/pkg/sftp" + "github.com/uw-labs/straw" "golang.org/x/crypto/ssh" ) -var _ StreamStore = &SFTPStreamStore{} +var _ straw.StreamStore = &SFTPStreamStore{} + +func init() { + straw.Register("sftp", func(u *url.URL) (straw.StreamStore, error) { + return NewSFTPStreamStore(u.String()) + }) +} type SFTPStreamStore struct { sshClient *ssh.Client @@ -74,7 +81,7 @@ func (s *SFTPStreamStore) Mkdir(path string, mode os.FileMode) error { return err } -func (s *SFTPStreamStore) OpenReadCloser(name string) (StrawReader, error) { +func (s *SFTPStreamStore) OpenReadCloser(name string) (straw.StrawReader, error) { sr, err := s.sftpClient.Open(name) if err != nil { return nil, err @@ -99,7 +106,7 @@ func (s *SFTPStreamStore) Remove(name string) error { return err } -func (s *SFTPStreamStore) CreateWriteCloser(name string) (StrawWriter, error) { +func (s *SFTPStreamStore) CreateWriteCloser(name string) (straw.StrawWriter, error) { fi, err := s.Stat(name) if err == nil && fi.IsDir() { return nil, fmt.Errorf("%s is a directory", name) diff --git a/strawurl.go b/strawurl.go index 1582d83..614f2d7 100644 --- a/strawurl.go +++ b/strawurl.go @@ -22,29 +22,8 @@ func Open(u string) (StreamStore, error) { } func init() { + // the only "built in" backend is "file" Register("file", func(u *url.URL) (StreamStore, error) { return &OsStreamStore{}, nil }) - Register("gs", func(u *url.URL) (StreamStore, error) { - creds := u.Query().Get("credentialsfile") - if creds == "" { - return nil, fmt.Errorf("gs URLs must provide a `credentialsfile` parameter") - } - return NewGCSStreamStore(creds, u.Host) - }) - Register("s3", func(u *url.URL) (StreamStore, error) { - sse := u.Query().Get("sse") - var opts []S3Option - switch sse { - case "": - case "AES256": - opts = append(opts, S3ServerSideEncoding(ServerSideEncryptionTypeAES256)) - default: - return nil, fmt.Errorf("unknown server side encryption type '%s'", sse) - } - return NewS3StreamStore(u.Host, opts...) - }) - Register("sftp", func(u *url.URL) (StreamStore, error) { - return NewSFTPStreamStore(u.String()) - }) }