Merge pull request #18 from ConductorOne/ggreer/cache-http-gets

Cache all http gets. Increase cache TTL to 5 minutes.

go.mod

@@ -3,12 +3,13 @@ module github.com/conductorone/baton-bitbucket-datacenter
 go 1.22.2
 
 require (
-	github.com/conductorone/baton-sdk v0.1.41
+	github.com/conductorone/baton-sdk v0.1.47
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
 	go.uber.org/zap v1.27.0
 )
 
 require (
+	github.com/deckarep/golang-set/v2 v2.6.0 // indirect
 	go.opentelemetry.io/otel v1.27.0 // indirect
 	go.opentelemetry.io/otel/metric v1.27.0 // indirect
 )

go.sum

@@ -50,13 +50,15 @@ github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/conductorone/baton-sdk v0.1.41 h1:M6U8QrG4mvI4UtlVyyCf0jv1DsHr0FQbV+8FXWi6qEE=
-github.com/conductorone/baton-sdk v0.1.41/go.mod h1:CxHwuTWhrX2w5yEuAxoRWIUofimnnnM9ancsZPvTko8=
+github.com/conductorone/baton-sdk v0.1.47 h1:sHUrhaFbzMb7GDAyQs8QIEZWAfm363IKNY0IUBOqdxw=
+github.com/conductorone/baton-sdk v0.1.47/go.mod h1:fq4It+11UqfEVOumA0IDQhpkMqJTAV31WaFnfkgGVYM=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM=
+github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
 github.com/denisenkom/go-mssqldb v0.10.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/doug-martin/goqu/v9 v9.19.0 h1:PD7t1X3tRcUiSdc5TEyOFKujZA5gs3VSA7wxSvBx7qo=
 github.com/doug-martin/goqu/v9 v9.19.0/go.mod h1:nf0Wc2/hV3gYK9LiyqIrzBEVGlI8qW3GuDCEobC4wBQ=
@@ -137,6 +139,7 @@ github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0V
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA=
 github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
@@ -217,8 +220,14 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg=
 go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ=
+go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.27.0 h1:/jlt1Y8gXWiHG9FBx6cJaIC5hYx5Fe64nC8w5Cylt/0=
+go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.27.0/go.mod h1:bmToOGOBZ4hA9ghphIc1PAf66VA8KOtsuy3+ScStG20=
 go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik=
 go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak=
+go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI=
+go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A=
+go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2NemcCrOL8gI=
+go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw=
 go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw=
 go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=

pkg/client/cache.go

@@ -17,8 +17,7 @@ type Cache interface {
 }
 
 func CreateCacheKey(req *http.Request) string {
-	s := req.URL.Scheme + "://" + req.URL.Host + req.URL.RequestURI()
-	return s
+	return req.URL.String()
 }
 
 func CopyResponse(resp *http.Response) *http.Response {

pkg/client/client.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/conductorone/baton-sdk/pkg/uhttp"
 	"github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap"
+	"go.uber.org/zap"
 )
 
 type DataCenterClient struct {
@@ -80,7 +81,7 @@ func NewClient() *DataCenterClient {
 			password:    "",
 			bearerToken: "",
 		},
-		bitbucketCache: NewGoCache(30, 30),
+		bitbucketCache: NewGoCache(300, 300),
 	}
 }
 
@@ -192,6 +193,33 @@ func New(ctx context.Context, baseUrl string, bitbucketClient *DataCenterClient)
 	return &dc, nil
 }
 
+func (d *DataCenterClient) CachedGet(ctx context.Context, req *http.Request, options ...uhttp.DoOption) (*http.Response, error) {
+	l := ctxzap.Extract(ctx)
+	cacheKey := CreateCacheKey(req)
+
+	if d.bitbucketCache.Has(cacheKey) {
+		l.Debug("cache hit", zap.String("cacheKey", cacheKey))
+		resp := d.bitbucketCache.Get(cacheKey)
+		return resp, nil
+	}
+
+	l.Debug("cache miss", zap.String("cacheKey", cacheKey))
+	resp, err := d.httpClient.Do(req, options...)
+	if err != nil {
+		return nil, &BitbucketError{
+			ErrorMessage:     err.Error(),
+			ErrorDescription: err.Error(),
+			ErrorCode:        resp.StatusCode,
+			ErrorSummary:     fmt.Sprint(resp.Body),
+			ErrorLink:        req.URL.String(),
+		}
+	}
+
+	d.bitbucketCache.Set(cacheKey, resp)
+	defer resp.Body.Close()
+	return resp, nil
+}
+
 // GetUsers
 // Get all users. Only authenticated users may call this resource.
 // https://developer.atlassian.com/server/bitbucket/rest/v819/api-group-system-maintenance/#api-api-latest-users-get
@@ -226,15 +254,13 @@ func (d *DataCenterClient) GetUsers(ctx context.Context, startPage, limit string
 		return nil, Page{}, err
 	}
 
-	resp, err := d.httpClient.Do(req, uhttp.WithJSONResponse(&userData))
+	resp, err := d.CachedGet(ctx, req, uhttp.WithJSONResponse(&userData))
 	if err != nil {
-		return nil, Page{}, &BitbucketError{
-			ErrorMessage:     err.Error(),
-			ErrorDescription: err.Error(),
-			ErrorCode:        resp.StatusCode,
-			ErrorSummary:     fmt.Sprint(resp.Body),
-			ErrorLink:        endpointUrl,
-		}
+		return nil, Page{}, err
+	}
+	err = WithResponse(resp, &userData)
+	if err != nil {
+		return nil, page, err
 	}
 
 	defer resp.Body.Close()
@@ -299,15 +325,13 @@ func (d *DataCenterClient) GetProjects(ctx context.Context, startPage, limit str
 		return nil, Page{}, err
 	}
 
-	resp, err := d.httpClient.Do(req, uhttp.WithJSONResponse(&projectData))
+	resp, err := d.CachedGet(ctx, req, uhttp.WithJSONResponse(&projectData))
 	if err != nil {
-		return nil, Page{}, &BitbucketError{
-			ErrorMessage:     err.Error(),
-			ErrorDescription: err.Error(),
-			ErrorCode:        resp.StatusCode,
-			ErrorSummary:     fmt.Sprint(resp.Body),
-			ErrorLink:        endpointUrl,
-		}
+		return nil, Page{}, err
+	}
+	err = WithResponse(resp, &projectData)
+	if err != nil {
+		return nil, page, err
 	}
 
 	defer resp.Body.Close()
@@ -369,15 +393,13 @@ func (d *DataCenterClient) GetRepos(ctx context.Context, startPage, limit string
 		return nil, Page{}, err
 	}
 
-	resp, err := d.httpClient.Do(req, uhttp.WithJSONResponse(&repoData))
+	resp, err := d.CachedGet(ctx, req, uhttp.WithJSONResponse(&repoData))
 	if err != nil {
-		return nil, Page{}, &BitbucketError{
-			ErrorMessage:     err.Error(),
-			ErrorDescription: err.Error(),
-			ErrorCode:        resp.StatusCode,
-			ErrorSummary:     fmt.Sprint(resp.Body),
-			ErrorLink:        endpointUrl,
-		}
+		return nil, Page{}, err
+	}
+	err = WithResponse(resp, &repoData)
+	if err != nil {
+		return nil, page, err
 	}
 
 	defer resp.Body.Close()
@@ -442,15 +464,13 @@ func (d *DataCenterClient) GetGroups(ctx context.Context, startPage, limit strin
 		return nil, Page{}, err
 	}
 
-	resp, err := d.httpClient.Do(req, uhttp.WithJSONResponse(&groupData))
+	resp, err := d.CachedGet(ctx, req, uhttp.WithJSONResponse(&groupData))
 	if err != nil {
-		return nil, Page{}, &BitbucketError{
-			ErrorMessage:     err.Error(),
-			ErrorDescription: err.Error(),
-			ErrorCode:        resp.StatusCode,
-			ErrorSummary:     fmt.Sprint(resp.Body),
-			ErrorLink:        endpointUrl,
-		}
+		return nil, Page{}, err
+	}
+	err = WithResponse(resp, &groupData)
+	if err != nil {
+		return nil, page, err
 	}
 
 	defer resp.Body.Close()
@@ -511,10 +531,14 @@ func (d *DataCenterClient) GetGroupMembers(ctx context.Context, startPage, limit
 		return nil, Page{}, err
 	}
 
-	resp, err := d.httpClient.Do(req, uhttp.WithJSONResponse(&memberData))
+	resp, err := d.CachedGet(ctx, req, uhttp.WithJSONResponse(&memberData))
 	if err != nil {
 		return nil, Page{}, err
 	}
+	err = WithResponse(resp, &memberData)
+	if err != nil {
+		return nil, page, err
+	}
 
 	defer resp.Body.Close()
 	sPage = strconv.Itoa(memberData.Start)
@@ -593,15 +617,13 @@ func (d *DataCenterClient) GetGlobalUserPermissions(ctx context.Context, startPa
 		return nil, Page{}, err
 	}
 
-	resp, err := d.httpClient.Do(req, uhttp.WithJSONResponse(&permissionsData))
+	resp, err := d.CachedGet(ctx, req, uhttp.WithJSONResponse(&permissionsData))
 	if err != nil {
-		return nil, Page{}, &BitbucketError{
-			ErrorMessage:     err.Error(),
-			ErrorDescription: err.Error(),
-			ErrorCode:        resp.StatusCode,
-			ErrorSummary:     fmt.Sprint(resp.Body),
-			ErrorLink:        endpointUrl,
-		}
+		return nil, Page{}, err
+	}
+	err = WithResponse(resp, &permissionsData)
+	if err != nil {
+		return nil, page, err
 	}
 
 	defer resp.Body.Close()
@@ -649,14 +671,13 @@ func (d *DataCenterClient) GetGlobalGroupPermissions(ctx context.Context, startP
 		return nil, Page{}, err
 	}
 
-	resp, err := d.httpClient.Do(req, uhttp.WithJSONResponse(&permissionsData))
+	resp, err := d.CachedGet(ctx, req, uhttp.WithJSONResponse(&permissionsData))
 	if err != nil {
-		return nil, Page{}, &BitbucketError{
-			ErrorMessage:     err.Error(),
-			ErrorDescription: err.Error(),
-			ErrorCode:        resp.StatusCode,
-			ErrorSummary:     fmt.Sprint(resp.Body),
-		}
+		return nil, Page{}, err
+	}
+	err = WithResponse(resp, &permissionsData)
+	if err != nil {
+		return nil, page, err
 	}
 
 	defer resp.Body.Close()
@@ -742,10 +763,14 @@ func (d *DataCenterClient) GetUserRepositoryPermissions(ctx context.Context, sta
 		return nil, Page{}, err
 	}
 
-	resp, err := d.httpClient.Do(req, uhttp.WithJSONResponse(&permissionData))
+	resp, err := d.CachedGet(ctx, req, uhttp.WithJSONResponse(&permissionData))
 	if err != nil {
 		return nil, Page{}, err
 	}
+	err = WithResponse(resp, &permissionData)
+	if err != nil {
+		return nil, page, err
+	}
 
 	defer resp.Body.Close()
 	sPage = strconv.Itoa(permissionData.Start)
@@ -811,10 +836,14 @@ func (d *DataCenterClient) GetUserProjectsPermissions(ctx context.Context, start
 		return nil, Page{}, err
 	}
 
-	resp, err := d.httpClient.Do(req, uhttp.WithJSONResponse(&permissionData))
+	resp, err := d.CachedGet(ctx, req, uhttp.WithJSONResponse(&permissionData))
 	if err != nil {
 		return nil, Page{}, err
 	}
+	err = WithResponse(resp, &permissionData)
+	if err != nil {
+		return nil, page, err
+	}
 
 	defer resp.Body.Close()
 	sPage = strconv.Itoa(permissionData.Start)
@@ -861,10 +890,14 @@ func (d *DataCenterClient) GetGroupProjectsPermissions(ctx context.Context, star
 		return nil, Page{}, err
 	}
 
-	resp, err := d.httpClient.Do(req, uhttp.WithJSONResponse(&permissionData))
+	resp, err := d.CachedGet(ctx, req, uhttp.WithJSONResponse(&permissionData))
 	if err != nil {
 		return nil, Page{}, err
 	}
+	err = WithResponse(resp, &permissionData)
+	if err != nil {
+		return nil, page, err
+	}
 
 	defer resp.Body.Close()
 	sPage = strconv.Itoa(permissionData.Start)
@@ -935,7 +968,6 @@ func (d *DataCenterClient) GetGroupRepositoryPermissions(ctx context.Context, st
 		permissionData GroupPermissionsAPIData
 		page           Page
 		sPage, nPage   = "0", "0"
-		resp           *http.Response
 	)
 	endpointUrl := fmt.Sprintf("%s/%s/%s/repos/%s/%s", d.baseUrl,
 		allProjectsEndpoint,
@@ -963,32 +995,16 @@ func (d *DataCenterClient) GetGroupRepositoryPermissions(ctx context.Context, st
 		return nil, Page{}, err
 	}
 
-	cacheKey := CreateCacheKey(req)
-	found := d.bitbucketCache.Has(cacheKey)
-	if found {
-		resp = d.bitbucketCache.Get(cacheKey)
-		err := WithResponse(resp, &permissionData)
-		if err != nil {
-			return nil, page, err
-		}
-
-		defer resp.Body.Close()
-	} else {
-		resp, err = d.httpClient.Do(req, uhttp.WithJSONResponse(&permissionData))
-		if err != nil {
-			return nil, Page{}, &BitbucketError{
-				ErrorMessage:     err.Error(),
-				ErrorDescription: err.Error(),
-				ErrorCode:        resp.StatusCode,
-				ErrorSummary:     fmt.Sprint(resp.Body),
-				ErrorLink:        endpointUrl,
-			}
-		}
-
-		d.bitbucketCache.Set(cacheKey, resp)
-		defer resp.Body.Close()
+	resp, err := d.CachedGet(ctx, req, uhttp.WithJSONResponse(&permissionData))
+	if err != nil {
+		return nil, Page{}, err
+	}
+	err = WithResponse(resp, &permissionData)
+	if err != nil {
+		return nil, page, err
 	}
 
+	defer resp.Body.Close()
 	sPage = strconv.Itoa(permissionData.Start)
 	nPage = strconv.Itoa(permissionData.NextPageStart)
 	if !permissionData.IsLastPage {