From 85e8a943d76d42b02c6b563fc9999b3be5f84236 Mon Sep 17 00:00:00 2001 From: andrei-tyk <97896463+andrei-tyk@users.noreply.github.com> Date: Fri, 1 Nov 2024 14:09:03 +0200 Subject: [PATCH 1/2] TT-13271, fix for token metadata not being cached (#6689) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### **User description** Currently when the token is cached into redis we are not cacheing the extra metadata so only the initial request gets populated with the requiered fields.
TT-13271
Summary Add support for custom OAuth server response fields
Type Story Story
Status Ready for Testing
Points N/A
Labels -
--- ## Description ## Related Issue ## Motivation and Context ## How This Has Been Tested ## Screenshots (if appropriate) ## Types of changes - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Refactoring or add test (improvements in base code or adds test coverage to functionality) ## Checklist - [ ] I ensured that the documentation is up to date - [ ] I explained why this PR updates go.mod in detail with reasoning why it's required - [ ] I would like a code coverage CI quality gate exception and have explained why ___ ### **PR Type** enhancement, bug fix ___ ### **Description** - Introduced a new `TokenData` struct to encapsulate token and extra metadata, improving the structure and readability of the code. - Added functions `createTokenDataBytes` and `unmarshalTokenData` to handle JSON marshaling and unmarshaling of token data, enhancing the robustness of data handling. - Modified the caching mechanism to store both token and metadata, ensuring that metadata is preserved and accessible when retrieving tokens from the cache. - Improved the handling of extra metadata by using a map, allowing for more flexible and efficient metadata management. ___ ### **Changes walkthrough** đź“ť
Relevant files
Enhancement
mw_oauth2_auth.go
Improve token caching and metadata handling in OAuth2       
gateway/mw_oauth2_auth.go
  • Introduced a new TokenData struct to handle token and metadata.
  • Added functions to marshal and unmarshal token data.
  • Modified caching logic to store and retrieve token data with metadata.
  • Enhanced metadata handling by using a map for extra metadata.
    		•  +85/-17 
    ___ > đź’ˇ **PR-Agent usage**: Comment `/help "your question"` on any pull request to receive relevant information --- gateway/mw_oauth2_auth.go | 84 ++++++++++++++++---- gateway/mw_oauth2_auth_test.go | 138 +++++++++++++++++++++++++++++++++ 2 files changed, 206 insertions(+), 16 deletions(-) diff --git a/gateway/mw_oauth2_auth.go b/gateway/mw_oauth2_auth.go index bc683edbdf1..6808a39ab45 100644 --- a/gateway/mw_oauth2_auth.go +++ b/gateway/mw_oauth2_auth.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "net/http" "strings" @@ -57,13 +58,18 @@ type upstreamOAuthPasswordCache struct { func (cache *upstreamOAuthPasswordCache) getToken(r *http.Request, OAuthSpec *UpstreamOAuth) (string, error) { cacheKey := generatePasswordOAuthCacheKey(OAuthSpec.Spec.UpstreamAuth.OAuth, OAuthSpec.Spec.APIID) - tokenString, err := retryGetKeyAndLock(cacheKey, &cache.RedisCluster) + tokenData, err := retryGetKeyAndLock(cacheKey, &cache.RedisCluster) if err != nil { return "", err } - if tokenString != "" { - decryptedToken := decrypt(getPaddedSecret(OAuthSpec.Gw.GetConfig().Secret), tokenString) + if tokenData != "" { + tokenContents, err := unmarshalTokenData(tokenData) + if err != nil { + return "", err + } + decryptedToken := decrypt(getPaddedSecret(OAuthSpec.Gw.GetConfig().Secret), tokenContents.Token) + setExtraMetadata(r, OAuthSpec.Spec.UpstreamAuth.OAuth.PasswordAuthentication.ExtraMetadata, tokenContents.ExtraMetadata) return decryptedToken, nil } @@ -73,10 +79,15 @@ func (cache *upstreamOAuthPasswordCache) getToken(r *http.Request, OAuthSpec *Up } encryptedToken := encrypt(getPaddedSecret(OAuthSpec.Gw.GetConfig().Secret), token.AccessToken) - setExtraMetadata(r, OAuthSpec.Spec.UpstreamAuth.OAuth.PasswordAuthentication.ExtraMetadata, token) + tokenDataBytes, err := createTokenDataBytes(encryptedToken, token, OAuthSpec.Spec.UpstreamAuth.OAuth.PasswordAuthentication.ExtraMetadata) + if err != nil { + return "", err + } + metadataMap := buildMetadataMap(token, OAuthSpec.Spec.UpstreamAuth.OAuth.PasswordAuthentication.ExtraMetadata) + setExtraMetadata(r, OAuthSpec.Spec.UpstreamAuth.OAuth.PasswordAuthentication.ExtraMetadata, metadataMap) ttl := time.Until(token.Expiry) - if err := setTokenInCache(cacheKey, encryptedToken, ttl, &cache.RedisCluster); err != nil { + if err := setTokenInCache(cacheKey, string(tokenDataBytes), ttl, &cache.RedisCluster); err != nil { return "", err } @@ -271,16 +282,26 @@ func generateClientCredentialsCacheKey(config apidef.UpstreamOAuth, apiId string return hex.EncodeToString(hash.Sum(nil)) } +type TokenData struct { + Token string `json:"token"` + ExtraMetadata map[string]interface{} `json:"extra_metadata"` +} + func (cache *upstreamOAuthClientCredentialsCache) getToken(r *http.Request, OAuthSpec *UpstreamOAuth) (string, error) { cacheKey := generateClientCredentialsCacheKey(OAuthSpec.Spec.UpstreamAuth.OAuth, OAuthSpec.Spec.APIID) - tokenString, err := retryGetKeyAndLock(cacheKey, &cache.RedisCluster) + tokenData, err := retryGetKeyAndLock(cacheKey, &cache.RedisCluster) if err != nil { return "", err } - if tokenString != "" { - decryptedToken := decrypt(getPaddedSecret(OAuthSpec.Gw.GetConfig().Secret), tokenString) + if tokenData != "" { + tokenContents, err := unmarshalTokenData(tokenData) + if err != nil { + return "", err + } + decryptedToken := decrypt(getPaddedSecret(OAuthSpec.Gw.GetConfig().Secret), tokenContents.Token) + setExtraMetadata(r, OAuthSpec.Spec.UpstreamAuth.OAuth.ClientCredentials.ExtraMetadata, tokenContents.ExtraMetadata) return decryptedToken, nil } @@ -290,24 +311,55 @@ func (cache *upstreamOAuthClientCredentialsCache) getToken(r *http.Request, OAut } encryptedToken := encrypt(getPaddedSecret(OAuthSpec.Gw.GetConfig().Secret), token.AccessToken) - setExtraMetadata(r, OAuthSpec.Spec.UpstreamAuth.OAuth.ClientCredentials.ExtraMetadata, token) + tokenDataBytes, err := createTokenDataBytes(encryptedToken, token, OAuthSpec.Spec.UpstreamAuth.OAuth.ClientCredentials.ExtraMetadata) + if err != nil { + return "", err + } + metadataMap := buildMetadataMap(token, OAuthSpec.Spec.UpstreamAuth.OAuth.ClientCredentials.ExtraMetadata) + setExtraMetadata(r, OAuthSpec.Spec.UpstreamAuth.OAuth.ClientCredentials.ExtraMetadata, metadataMap) ttl := time.Until(token.Expiry) - if err := setTokenInCache(cacheKey, encryptedToken, ttl, &cache.RedisCluster); err != nil { + if err := setTokenInCache(cacheKey, string(tokenDataBytes), ttl, &cache.RedisCluster); err != nil { return "", err } return token.AccessToken, nil } -func setExtraMetadata(r *http.Request, keyList []string, token *oauth2.Token) { +func createTokenDataBytes(encryptedToken string, token *oauth2.Token, extraMetadataKeys []string) ([]byte, error) { + td := TokenData{ + Token: encryptedToken, + ExtraMetadata: buildMetadataMap(token, extraMetadataKeys), + } + return json.Marshal(td) +} + +func unmarshalTokenData(tokenData string) (TokenData, error) { + var tokenContents TokenData + err := json.Unmarshal([]byte(tokenData), &tokenContents) + if err != nil { + return TokenData{}, fmt.Errorf("failed to unmarshal token data: %w", err) + } + return tokenContents, nil +} + +func buildMetadataMap(token *oauth2.Token, extraMetadataKeys []string) map[string]interface{} { + metadataMap := make(map[string]interface{}) + for _, key := range extraMetadataKeys { + if val := token.Extra(key); val != "" && val != nil { + metadataMap[key] = val + } + } + return metadataMap +} + +func setExtraMetadata(r *http.Request, keyList []string, token map[string]interface{}) { contextDataObject := ctxGetData(r) if contextDataObject == nil { contextDataObject = make(map[string]interface{}) } for _, key := range keyList { - val := token.Extra(key) - if val != "" { + if val, ok := token[key]; ok && val != "" { contextDataObject[key] = val } } @@ -318,13 +370,13 @@ func retryGetKeyAndLock(cacheKey string, cache *storage.RedisCluster) (string, e const maxRetries = 10 const retryDelay = 100 * time.Millisecond - var token string + var tokenData string var err error for i := 0; i < maxRetries; i++ { - token, err = cache.GetKey(cacheKey) + tokenData, err = cache.GetKey(cacheKey) if err == nil { - return token, nil + return tokenData, nil } lockKey := cacheKey + ":lock" diff --git a/gateway/mw_oauth2_auth_test.go b/gateway/mw_oauth2_auth_test.go index 736a6561f64..8f26b67039b 100644 --- a/gateway/mw_oauth2_auth_test.go +++ b/gateway/mw_oauth2_auth_test.go @@ -6,6 +6,9 @@ import ( "net/http" "net/http/httptest" "testing" + "time" + + "golang.org/x/oauth2" "github.com/stretchr/testify/assert" @@ -19,7 +22,13 @@ func TestUpstreamOauth2(t *testing.T) { tst := StartTest(nil) t.Cleanup(tst.Close) + var requestCount int ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + if requestCount > 0 { + assert.Fail(t, "Unexpected request received.") + } + requestCount++ if r.URL.String() != "/token" { assert.Fail(t, "authenticate client request URL = %q; want %q", r.URL, "/token") } @@ -90,6 +99,23 @@ func TestUpstreamOauth2(t *testing.T) { return true }, }, + { + Path: "/upstream-oauth-distributed/", + Code: http.StatusOK, + BodyMatchFunc: func(body []byte) bool { + resp := struct { + Headers map[string]string `json:"headers"` + }{} + err := json.Unmarshal(body, &resp) + assert.NoError(t, err) + + assert.Contains(t, resp.Headers, header.Authorization) + assert.NotEmpty(t, resp.Headers[header.Authorization]) + assert.Equal(t, "Bearer 90d64460d14870c08c81352a05dedd3465940a7c", resp.Headers[header.Authorization]) + + return true + }, + }, }...) } @@ -98,8 +124,13 @@ func TestPasswordCredentialsTokenRequest(t *testing.T) { tst := StartTest(nil) t.Cleanup(tst.Close) + var requestCount int ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() + if requestCount > 0 { + assert.Fail(t, "Unexpected request received.") + } + requestCount++ expected := "/token" if r.URL.String() != expected { assert.Fail(t, "URL = %q; want %q", r.URL, expected) @@ -174,5 +205,112 @@ func TestPasswordCredentialsTokenRequest(t *testing.T) { return true }, }, + { + Path: "/upstream-oauth-password/", + Code: http.StatusOK, + BodyMatchFunc: func(body []byte) bool { + resp := struct { + Headers map[string]string `json:"headers"` + }{} + err := json.Unmarshal(body, &resp) + assert.NoError(t, err) + + assert.Contains(t, resp.Headers, header.Authorization) + assert.NotEmpty(t, resp.Headers[header.Authorization]) + assert.Equal(t, "Bearer 90d64460d14870c08c81352a05dedd3465940a7c", resp.Headers[header.Authorization]) + + return true + }, + }, }...) } + +func TestSetExtraMetadata(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "http://tykxample.com", nil) + + keyList := []string{"key1", "key2"} + token := map[string]interface{}{ + "key1": "value1", + "key2": "value2", + "key3": "value3", + } + + setExtraMetadata(req, keyList, token) + + contextData := ctxGetData(req) + + assert.Equal(t, "value1", contextData["key1"]) + assert.Equal(t, "value2", contextData["key2"]) + assert.NotContains(t, contextData, "key3") +} + +func TestBuildMetadataMap(t *testing.T) { + token := &oauth2.Token{ + AccessToken: "tyk_upstream_oauth_access_token", + TokenType: "Bearer", + Expiry: time.Now().Add(time.Hour), + } + token = token.WithExtra(map[string]interface{}{ + "key1": "value1", + "key2": "value2", + "key3": "", + }) + extraMetadataKeys := []string{"key1", "key2", "key3", "key4"} + + metadataMap := buildMetadataMap(token, extraMetadataKeys) + + assert.Equal(t, "value1", metadataMap["key1"]) + assert.Equal(t, "value2", metadataMap["key2"]) + assert.NotContains(t, metadataMap, "key3") + assert.NotContains(t, metadataMap, "key4") +} + +func TestCreateTokenDataBytes(t *testing.T) { + token := &oauth2.Token{ + AccessToken: "tyk_upstream_oauth_access_token", + TokenType: "Bearer", + Expiry: time.Now().Add(time.Hour), + } + token = token.WithExtra(map[string]interface{}{ + "key1": "value1", + "key2": "value2", + "key3": "", + }) + + extraMetadataKeys := []string{"key1", "key2", "key3", "key4"} + + encryptedToken := "encrypted_tyk_upstream_oauth_access_token" + tokenDataBytes, err := createTokenDataBytes(encryptedToken, token, extraMetadataKeys) + + assert.NoError(t, err) + + var tokenData TokenData + err = json.Unmarshal(tokenDataBytes, &tokenData) + assert.NoError(t, err) + + assert.Equal(t, encryptedToken, tokenData.Token) + assert.Equal(t, "value1", tokenData.ExtraMetadata["key1"]) + assert.Equal(t, "value2", tokenData.ExtraMetadata["key2"]) + assert.NotContains(t, tokenData.ExtraMetadata, "key3") + assert.NotContains(t, tokenData.ExtraMetadata, "key4") +} + +func TestUnmarshalTokenData(t *testing.T) { + tokenData := TokenData{ + Token: "tyk_upstream_oauth_access_token", + ExtraMetadata: map[string]interface{}{ + "key1": "value1", + "key2": "value2", + }, + } + + tokenDataBytes, err := json.Marshal(tokenData) + assert.NoError(t, err) + + result, err := unmarshalTokenData(string(tokenDataBytes)) + + assert.NoError(t, err) + + assert.Equal(t, tokenData.Token, result.Token) + assert.Equal(t, tokenData.ExtraMetadata, result.ExtraMetadata) +} From 9335a5126ad920dda05e40908bd5d8d31f0c51eb Mon Sep 17 00:00:00 2001 From: Tit Petric Date: Sat, 2 Nov 2024 08:15:24 +0100 Subject: [PATCH 2/2] [TT-12885] Add plugin development guide for manual builds (#6598) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### **User description**
    TT-12885
    Summary [Plugin docs] Provide guidance on compiling plugins manually
    Type Story Story
    Status In Dev
    Points N/A
    Labels -
    --- https://tyktech.atlassian.net/browse/TT-12885 ___ ### **PR Type** Documentation, Enhancement ___ ### **Description** - Added a comprehensive guide for Go plugin development, detailing environment setup, workspace creation, and plugin testing. - Introduced Taskfiles for automating markdown formatting and plugin development tasks, enhancing workflow efficiency. - Fixed indentation issues in the swagger documentation to improve readability. ___ ### **Changes walkthrough** đź“ť
    Relevant files
    Configuration changes
    Taskfile.yml
    Add Taskfile for markdown formatting with mdox                     
    docs/Taskfile.yml
  • Added a Taskfile for managing markdown formatting tasks.
  • Introduced a default task to run mdox for formatting markdown files.
    		• +8/-0     
    Taskfile.yml
    Add Taskfile for automating plugin development tasks         
    docs/plugins/Taskfile.yml
  • Added a Taskfile for automating plugin development tasks.
  • Defined tasks for checking out repositories, creating workspaces, and
    testing plugins.
    		• +61/-0   
    Documentation
    README.md
    Add comprehensive Go plugin development guide                       
    docs/plugins/README.md
  • Added a comprehensive guide for Go plugin development.
  • Included steps for setting up the environment, creating workspaces,
    and testing plugins.
  • Provided troubleshooting tips for common issues and debugging.
    		• +245/-0 
    Formatting
    swagger.md
    Fix indentation in swagger generation documentation           
    docs/swagger.md
  • Fixed indentation issues in the documentation for generating swagger
    files.
    		• +2/-2     
    ___ > 💡 **PR-Agent usage**: Comment `/help "your question"` on any pull request to receive relevant information --------- Co-authored-by: Tit Petric --- .vale.ini | 6 + .../styles/config/vocabularies/Tyk/accept.txt | 130 ++++ .vale/styles/write-good/Cliches.yml | 702 ++++++++++++++++++ .vale/styles/write-good/E-Prime.yml | 32 + .vale/styles/write-good/Illusions.yml | 11 + .vale/styles/write-good/Passive.yml | 183 +++++ .vale/styles/write-good/README.md | 27 + .vale/styles/write-good/So.yml | 5 + .vale/styles/write-good/ThereIs.yml | 6 + .vale/styles/write-good/TooWordy.yml | 221 ++++++ .vale/styles/write-good/Weasel.yml | 207 ++++++ .vale/styles/write-good/meta.json | 4 + docs/Taskfile.yml | 11 + docs/plugins/go-development-flow.md | 271 +++++++ docs/plugins/tests/.gitignore | 3 + docs/plugins/tests/Taskfile.yml | 61 ++ docs/{ => schemas}/swagger.md | 4 +- 17 files changed, 1882 insertions(+), 2 deletions(-) create mode 100644 .vale.ini create mode 100644 .vale/styles/config/vocabularies/Tyk/accept.txt create mode 100644 .vale/styles/write-good/Cliches.yml create mode 100644 .vale/styles/write-good/E-Prime.yml create mode 100644 .vale/styles/write-good/Illusions.yml create mode 100644 .vale/styles/write-good/Passive.yml create mode 100644 .vale/styles/write-good/README.md create mode 100644 .vale/styles/write-good/So.yml create mode 100644 .vale/styles/write-good/ThereIs.yml create mode 100644 .vale/styles/write-good/TooWordy.yml create mode 100644 .vale/styles/write-good/Weasel.yml create mode 100644 .vale/styles/write-good/meta.json create mode 100644 docs/Taskfile.yml create mode 100644 docs/plugins/go-development-flow.md create mode 100644 docs/plugins/tests/.gitignore create mode 100644 docs/plugins/tests/Taskfile.yml rename docs/{ => schemas}/swagger.md (88%) diff --git a/.vale.ini b/.vale.ini new file mode 100644 index 00000000000..106eebbd45d --- /dev/null +++ b/.vale.ini @@ -0,0 +1,6 @@ +StylesPath = .vale/styles +MinAlertLevel = error +Vocab = Tyk + +[*.md] +BasedOnStyles = Tyk, write-good diff --git a/.vale/styles/config/vocabularies/Tyk/accept.txt b/.vale/styles/config/vocabularies/Tyk/accept.txt new file mode 100644 index 00000000000..678fe7adec9 --- /dev/null +++ b/.vale/styles/config/vocabularies/Tyk/accept.txt @@ -0,0 +1,130 @@ +analytics +API +api +apim +APIs +apis +AuthToken +AWS +balancer +blockprof +blockprofile +blockquote +CMS +CNAME +conf +config +CoProcess +coprocess +cpuprofile +crypto +CSRF +Datasource +DataSources +dereference +dev +DRL +elasticsearch +ELB +enum +Etcd +Github +Golang +golang +goroutine +goroutines +GraphQL +graphql +hmac +HMAC +http +HTTP +httpbin +httpprofile +https +HTTPS +IAM +IDP +init +img +io +IoT +JQ +jq +JSON +json +JSVM +JWK +JWKs +JWT +keyless +lastmod +MDCB +memprofile +microservice +middleware +MongoDB +MPL +mprof +mTLS +Mutex +mutex +mutexes +mutexprof +mutexprofile +namespace +natively +OAuth2.0 +OpenAPI +packagecloud +Param +param +pem +PEM +performant +pprof +profiler +Proxied +proxied +proxying +Pump +querystring +Quickstart +Rebase +rebase +Redis +runtime +schemas +SHA +sha +sharded +SSL +ssl +SSO +sso +stderr +stdout +sudo +Swagger +swagger +TCP +TIB +tls +TLS +Tyk +tyk +Tyk's +Tyk-Dashboard +Tyk-Gateway +Tyk-Pump +Tyk-Sync +TykMakeBatchRequest +TykMakeHttpRequest +TykTechnologies +UDG +UI +unmarshal +uptime +Uptime +uptime +url diff --git a/.vale/styles/write-good/Cliches.yml b/.vale/styles/write-good/Cliches.yml new file mode 100644 index 00000000000..c95314387ba --- /dev/null +++ b/.vale/styles/write-good/Cliches.yml @@ -0,0 +1,702 @@ +extends: existence +message: "Try to avoid using clichés like '%s'." +ignorecase: true +level: warning +tokens: + - a chip off the old block + - a clean slate + - a dark and stormy night + - a far cry + - a fine kettle of fish + - a loose cannon + - a penny saved is a penny earned + - a tough row to hoe + - a word to the wise + - ace in the hole + - acid test + - add insult to injury + - against all odds + - air your dirty laundry + - all fun and games + - all in a day's work + - all talk, no action + - all thumbs + - all your eggs in one basket + - all's fair in love and war + - all's well that ends well + - almighty dollar + - American as apple pie + - an axe to grind + - another day, another dollar + - armed to the teeth + - as luck would have it + - as old as time + - as the crow flies + - at loose ends + - at my wits end + - avoid like the plague + - babe in the woods + - back against the wall + - back in the saddle + - back to square one + - back to the drawing board + - bad to the bone + - badge of honor + - bald faced liar + - ballpark figure + - banging your head against a brick wall + - baptism by fire + - barking up the wrong tree + - bat out of hell + - be all and end all + - beat a dead horse + - beat around the bush + - been there, done that + - beggars can't be choosers + - behind the eight ball + - bend over backwards + - benefit of the doubt + - bent out of shape + - best thing since sliced bread + - bet your bottom dollar + - better half + - better late than never + - better mousetrap + - better safe than sorry + - between a rock and a hard place + - beyond the pale + - bide your time + - big as life + - big cheese + - big fish in a small pond + - big man on campus + - bigger they are the harder they fall + - bird in the hand + - bird's eye view + - birds and the bees + - birds of a feather flock together + - bit the hand that feeds you + - bite the bullet + - bite the dust + - bitten off more than he can chew + - black as coal + - black as pitch + - black as the ace of spades + - blast from the past + - bleeding heart + - blessing in disguise + - blind ambition + - blind as a bat + - blind leading the blind + - blood is thicker than water + - blood sweat and tears + - blow off steam + - blow your own horn + - blushing bride + - boils down to + - bolt from the blue + - bone to pick + - bored stiff + - bored to tears + - bottomless pit + - boys will be boys + - bright and early + - brings home the bacon + - broad across the beam + - broken record + - brought back to reality + - bull by the horns + - bull in a china shop + - burn the midnight oil + - burning question + - burning the candle at both ends + - burst your bubble + - bury the hatchet + - busy as a bee + - by hook or by crook + - call a spade a spade + - called onto the carpet + - calm before the storm + - can of worms + - can't cut the mustard + - can't hold a candle to + - case of mistaken identity + - cat got your tongue + - cat's meow + - caught in the crossfire + - caught red-handed + - checkered past + - chomping at the bit + - cleanliness is next to godliness + - clear as a bell + - clear as mud + - close to the vest + - cock and bull story + - cold shoulder + - come hell or high water + - cool as a cucumber + - cool, calm, and collected + - cost a king's ransom + - count your blessings + - crack of dawn + - crash course + - creature comforts + - cross that bridge when you come to it + - crushing blow + - cry like a baby + - cry me a river + - cry over spilt milk + - crystal clear + - curiosity killed the cat + - cut and dried + - cut through the red tape + - cut to the chase + - cute as a bugs ear + - cute as a button + - cute as a puppy + - cuts to the quick + - dark before the dawn + - day in, day out + - dead as a doornail + - devil is in the details + - dime a dozen + - divide and conquer + - dog and pony show + - dog days + - dog eat dog + - dog tired + - don't burn your bridges + - don't count your chickens + - don't look a gift horse in the mouth + - don't rock the boat + - don't step on anyone's toes + - don't take any wooden nickels + - down and out + - down at the heels + - down in the dumps + - down the hatch + - down to earth + - draw the line + - dressed to kill + - dressed to the nines + - drives me up the wall + - dull as dishwater + - dyed in the wool + - eagle eye + - ear to the ground + - early bird catches the worm + - easier said than done + - easy as pie + - eat your heart out + - eat your words + - eleventh hour + - even the playing field + - every dog has its day + - every fiber of my being + - everything but the kitchen sink + - eye for an eye + - face the music + - facts of life + - fair weather friend + - fall by the wayside + - fan the flames + - feast or famine + - feather your nest + - feathered friends + - few and far between + - fifteen minutes of fame + - filthy vermin + - fine kettle of fish + - fish out of water + - fishing for a compliment + - fit as a fiddle + - fit the bill + - fit to be tied + - flash in the pan + - flat as a pancake + - flip your lid + - flog a dead horse + - fly by night + - fly the coop + - follow your heart + - for all intents and purposes + - for the birds + - for what it's worth + - force of nature + - force to be reckoned with + - forgive and forget + - fox in the henhouse + - free and easy + - free as a bird + - fresh as a daisy + - full steam ahead + - fun in the sun + - garbage in, garbage out + - gentle as a lamb + - get a kick out of + - get a leg up + - get down and dirty + - get the lead out + - get to the bottom of + - get your feet wet + - gets my goat + - gilding the lily + - give and take + - go against the grain + - go at it tooth and nail + - go for broke + - go him one better + - go the extra mile + - go with the flow + - goes without saying + - good as gold + - good deed for the day + - good things come to those who wait + - good time was had by all + - good times were had by all + - greased lightning + - greek to me + - green thumb + - green-eyed monster + - grist for the mill + - growing like a weed + - hair of the dog + - hand to mouth + - happy as a clam + - happy as a lark + - hasn't a clue + - have a nice day + - have high hopes + - have the last laugh + - haven't got a row to hoe + - head honcho + - head over heels + - hear a pin drop + - heard it through the grapevine + - heart's content + - heavy as lead + - hem and haw + - high and dry + - high and mighty + - high as a kite + - hit paydirt + - hold your head up high + - hold your horses + - hold your own + - hold your tongue + - honest as the day is long + - horns of a dilemma + - horse of a different color + - hot under the collar + - hour of need + - I beg to differ + - icing on the cake + - if the shoe fits + - if the shoe were on the other foot + - in a jam + - in a jiffy + - in a nutshell + - in a pig's eye + - in a pinch + - in a word + - in hot water + - in the gutter + - in the nick of time + - in the thick of it + - in your dreams + - it ain't over till the fat lady sings + - it goes without saying + - it takes all kinds + - it takes one to know one + - it's a small world + - it's only a matter of time + - ivory tower + - Jack of all trades + - jockey for position + - jog your memory + - joined at the hip + - judge a book by its cover + - jump down your throat + - jump in with both feet + - jump on the bandwagon + - jump the gun + - jump to conclusions + - just a hop, skip, and a jump + - just the ticket + - justice is blind + - keep a stiff upper lip + - keep an eye on + - keep it simple, stupid + - keep the home fires burning + - keep up with the Joneses + - keep your chin up + - keep your fingers crossed + - kick the bucket + - kick up your heels + - kick your feet up + - kid in a candy store + - kill two birds with one stone + - kiss of death + - knock it out of the park + - knock on wood + - knock your socks off + - know him from Adam + - know the ropes + - know the score + - knuckle down + - knuckle sandwich + - knuckle under + - labor of love + - ladder of success + - land on your feet + - lap of luxury + - last but not least + - last hurrah + - last-ditch effort + - law of the jungle + - law of the land + - lay down the law + - leaps and bounds + - let sleeping dogs lie + - let the cat out of the bag + - let the good times roll + - let your hair down + - let's talk turkey + - letter perfect + - lick your wounds + - lies like a rug + - life's a bitch + - life's a grind + - light at the end of the tunnel + - lighter than a feather + - lighter than air + - like clockwork + - like father like son + - like taking candy from a baby + - like there's no tomorrow + - lion's share + - live and learn + - live and let live + - long and short of it + - long lost love + - look before you leap + - look down your nose + - look what the cat dragged in + - looking a gift horse in the mouth + - looks like death warmed over + - loose cannon + - lose your head + - lose your temper + - loud as a horn + - lounge lizard + - loved and lost + - low man on the totem pole + - luck of the draw + - luck of the Irish + - make hay while the sun shines + - make money hand over fist + - make my day + - make the best of a bad situation + - make the best of it + - make your blood boil + - man of few words + - man's best friend + - mark my words + - meaningful dialogue + - missed the boat on that one + - moment in the sun + - moment of glory + - moment of truth + - money to burn + - more power to you + - more than one way to skin a cat + - movers and shakers + - moving experience + - naked as a jaybird + - naked truth + - neat as a pin + - needle in a haystack + - needless to say + - neither here nor there + - never look back + - never say never + - nip and tuck + - nip it in the bud + - no guts, no glory + - no love lost + - no pain, no gain + - no skin off my back + - no stone unturned + - no time like the present + - no use crying over spilled milk + - nose to the grindstone + - not a hope in hell + - not a minute's peace + - not in my backyard + - not playing with a full deck + - not the end of the world + - not written in stone + - nothing to sneeze at + - nothing ventured nothing gained + - now we're cooking + - off the top of my head + - off the wagon + - off the wall + - old hat + - older and wiser + - older than dirt + - older than Methuselah + - on a roll + - on cloud nine + - on pins and needles + - on the bandwagon + - on the money + - on the nose + - on the rocks + - on the spot + - on the tip of my tongue + - on the wagon + - on thin ice + - once bitten, twice shy + - one bad apple doesn't spoil the bushel + - one born every minute + - one brick short + - one foot in the grave + - one in a million + - one red cent + - only game in town + - open a can of worms + - open and shut case + - open the flood gates + - opportunity doesn't knock twice + - out of pocket + - out of sight, out of mind + - out of the frying pan into the fire + - out of the woods + - out on a limb + - over a barrel + - over the hump + - pain and suffering + - pain in the + - panic button + - par for the course + - part and parcel + - party pooper + - pass the buck + - patience is a virtue + - pay through the nose + - penny pincher + - perfect storm + - pig in a poke + - pile it on + - pillar of the community + - pin your hopes on + - pitter patter of little feet + - plain as day + - plain as the nose on your face + - play by the rules + - play your cards right + - playing the field + - playing with fire + - pleased as punch + - plenty of fish in the sea + - point with pride + - poor as a church mouse + - pot calling the kettle black + - pretty as a picture + - pull a fast one + - pull your punches + - pulling your leg + - pure as the driven snow + - put it in a nutshell + - put one over on you + - put the cart before the horse + - put the pedal to the metal + - put your best foot forward + - put your foot down + - quick as a bunny + - quick as a lick + - quick as a wink + - quick as lightning + - quiet as a dormouse + - rags to riches + - raining buckets + - raining cats and dogs + - rank and file + - rat race + - reap what you sow + - red as a beet + - red herring + - reinvent the wheel + - rich and famous + - rings a bell + - ripe old age + - ripped me off + - rise and shine + - road to hell is paved with good intentions + - rob Peter to pay Paul + - roll over in the grave + - rub the wrong way + - ruled the roost + - running in circles + - sad but true + - sadder but wiser + - salt of the earth + - scared stiff + - scared to death + - sealed with a kiss + - second to none + - see eye to eye + - seen the light + - seize the day + - set the record straight + - set the world on fire + - set your teeth on edge + - sharp as a tack + - shoot for the moon + - shoot the breeze + - shot in the dark + - shoulder to the wheel + - sick as a dog + - sigh of relief + - signed, sealed, and delivered + - sink or swim + - six of one, half a dozen of another + - skating on thin ice + - slept like a log + - slinging mud + - slippery as an eel + - slow as molasses + - smart as a whip + - smooth as a baby's bottom + - sneaking suspicion + - snug as a bug in a rug + - sow wild oats + - spare the rod, spoil the child + - speak of the devil + - spilled the beans + - spinning your wheels + - spitting image of + - spoke with relish + - spread like wildfire + - spring to life + - squeaky wheel gets the grease + - stands out like a sore thumb + - start from scratch + - stick in the mud + - still waters run deep + - stitch in time + - stop and smell the roses + - straight as an arrow + - straw that broke the camel's back + - strong as an ox + - stubborn as a mule + - stuff that dreams are made of + - stuffed shirt + - sweating blood + - sweating bullets + - take a load off + - take one for the team + - take the bait + - take the bull by the horns + - take the plunge + - takes one to know one + - takes two to tango + - the more the merrier + - the real deal + - the real McCoy + - the red carpet treatment + - the same old story + - there is no accounting for taste + - thick as a brick + - thick as thieves + - thin as a rail + - think outside of the box + - third time's the charm + - this day and age + - this hurts me worse than it hurts you + - this point in time + - three sheets to the wind + - through thick and thin + - throw in the towel + - tie one on + - tighter than a drum + - time and time again + - time is of the essence + - tip of the iceberg + - tired but happy + - to coin a phrase + - to each his own + - to make a long story short + - to the best of my knowledge + - toe the line + - tongue in cheek + - too good to be true + - too hot to handle + - too numerous to mention + - touch with a ten foot pole + - tough as nails + - trial and error + - trials and tribulations + - tried and true + - trip down memory lane + - twist of fate + - two cents worth + - two peas in a pod + - ugly as sin + - under the counter + - under the gun + - under the same roof + - under the weather + - until the cows come home + - unvarnished truth + - up the creek + - uphill battle + - upper crust + - upset the applecart + - vain attempt + - vain effort + - vanquish the enemy + - vested interest + - waiting for the other shoe to drop + - wakeup call + - warm welcome + - watch your p's and q's + - watch your tongue + - watching the clock + - water under the bridge + - weather the storm + - weed them out + - week of Sundays + - went belly up + - wet behind the ears + - what goes around comes around + - what you see is what you get + - when it rains, it pours + - when push comes to shove + - when the cat's away + - when the going gets tough, the tough get going + - white as a sheet + - whole ball of wax + - whole hog + - whole nine yards + - wild goose chase + - will wonders never cease? + - wisdom of the ages + - wise as an owl + - wolf at the door + - words fail me + - work like a dog + - world weary + - worst nightmare + - worth its weight in gold + - wrong side of the bed + - yanking your chain + - yappy as a dog + - years young + - you are what you eat + - you can run but you can't hide + - you only live once + - you're the boss + - young and foolish + - young and vibrant diff --git a/.vale/styles/write-good/E-Prime.yml b/.vale/styles/write-good/E-Prime.yml new file mode 100644 index 00000000000..074a102b250 --- /dev/null +++ b/.vale/styles/write-good/E-Prime.yml @@ -0,0 +1,32 @@ +extends: existence +message: "Try to avoid using '%s'." +ignorecase: true +level: suggestion +tokens: + - am + - are + - aren't + - be + - been + - being + - he's + - here's + - here's + - how's + - i'm + - is + - isn't + - it's + - she's + - that's + - there's + - they're + - was + - wasn't + - we're + - were + - weren't + - what's + - where's + - who's + - you're diff --git a/.vale/styles/write-good/Illusions.yml b/.vale/styles/write-good/Illusions.yml new file mode 100644 index 00000000000..b4f13218592 --- /dev/null +++ b/.vale/styles/write-good/Illusions.yml @@ -0,0 +1,11 @@ +extends: repetition +message: "'%s' is repeated!" +level: warning +alpha: true +action: + name: edit + params: + - truncate + - " " +tokens: + - '[^\s]+' diff --git a/.vale/styles/write-good/Passive.yml b/.vale/styles/write-good/Passive.yml new file mode 100644 index 00000000000..f472cb9049f --- /dev/null +++ b/.vale/styles/write-good/Passive.yml @@ -0,0 +1,183 @@ +extends: existence +message: "'%s' may be passive voice. Use active voice if you can." +ignorecase: true +level: warning +raw: + - \b(am|are|were|being|is|been|was|be)\b\s* +tokens: + - '[\w]+ed' + - awoken + - beat + - become + - been + - begun + - bent + - beset + - bet + - bid + - bidden + - bitten + - bled + - blown + - born + - bought + - bound + - bred + - broadcast + - broken + - brought + - built + - burnt + - burst + - cast + - caught + - chosen + - clung + - come + - cost + - crept + - cut + - dealt + - dived + - done + - drawn + - dreamt + - driven + - drunk + - dug + - eaten + - fallen + - fed + - felt + - fit + - fled + - flown + - flung + - forbidden + - foregone + - forgiven + - forgotten + - forsaken + - fought + - found + - frozen + - given + - gone + - gotten + - ground + - grown + - heard + - held + - hidden + - hit + - hung + - hurt + - kept + - knelt + - knit + - known + - laid + - lain + - leapt + - learnt + - led + - left + - lent + - let + - lighted + - lost + - made + - meant + - met + - misspelt + - mistaken + - mown + - overcome + - overdone + - overtaken + - overthrown + - paid + - pled + - proven + - put + - quit + - read + - rid + - ridden + - risen + - run + - rung + - said + - sat + - sawn + - seen + - sent + - set + - sewn + - shaken + - shaven + - shed + - shod + - shone + - shorn + - shot + - shown + - shrunk + - shut + - slain + - slept + - slid + - slit + - slung + - smitten + - sold + - sought + - sown + - sped + - spent + - spilt + - spit + - split + - spoken + - spread + - sprung + - spun + - stolen + - stood + - stridden + - striven + - struck + - strung + - stuck + - stung + - stunk + - sung + - sunk + - swept + - swollen + - sworn + - swum + - swung + - taken + - taught + - thought + - thrived + - thrown + - thrust + - told + - torn + - trodden + - understood + - upheld + - upset + - wed + - wept + - withheld + - withstood + - woken + - won + - worn + - wound + - woven + - written + - wrung diff --git a/.vale/styles/write-good/README.md b/.vale/styles/write-good/README.md new file mode 100644 index 00000000000..3edcc9b3760 --- /dev/null +++ b/.vale/styles/write-good/README.md @@ -0,0 +1,27 @@ +Based on [write-good](https://github.com/btford/write-good). + +> Naive linter for English prose for developers who can't write good and wanna learn to do other stuff good too. + +``` +The MIT License (MIT) + +Copyright (c) 2014 Brian Ford + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` diff --git a/.vale/styles/write-good/So.yml b/.vale/styles/write-good/So.yml new file mode 100644 index 00000000000..e57f099dc0b --- /dev/null +++ b/.vale/styles/write-good/So.yml @@ -0,0 +1,5 @@ +extends: existence +message: "Don't start a sentence with '%s'." +level: error +raw: + - '(?:[;-]\s)so[\s,]|\bSo[\s,]' diff --git a/.vale/styles/write-good/ThereIs.yml b/.vale/styles/write-good/ThereIs.yml new file mode 100644 index 00000000000..8b82e8f6ccc --- /dev/null +++ b/.vale/styles/write-good/ThereIs.yml @@ -0,0 +1,6 @@ +extends: existence +message: "Don't start a sentence with '%s'." +ignorecase: false +level: error +raw: + - '(?:[;-]\s)There\s(is|are)|\bThere\s(is|are)\b' diff --git a/.vale/styles/write-good/TooWordy.yml b/.vale/styles/write-good/TooWordy.yml new file mode 100644 index 00000000000..275701b1962 --- /dev/null +++ b/.vale/styles/write-good/TooWordy.yml @@ -0,0 +1,221 @@ +extends: existence +message: "'%s' is too wordy." +ignorecase: true +level: warning +tokens: + - a number of + - abundance + - accede to + - accelerate + - accentuate + - accompany + - accomplish + - accorded + - accrue + - acquiesce + - acquire + - additional + - adjacent to + - adjustment + - admissible + - advantageous + - adversely impact + - advise + - aforementioned + - aggregate + - aircraft + - all of + - all things considered + - alleviate + - allocate + - along the lines of + - already existing + - alternatively + - amazing + - ameliorate + - anticipate + - apparent + - appreciable + - as a matter of fact + - as a means of + - as far as I'm concerned + - as of yet + - as to + - as yet + - ascertain + - assistance + - at the present time + - at this time + - attain + - attributable to + - authorize + - because of the fact that + - belated + - benefit from + - bestow + - by means of + - by virtue of + - by virtue of the fact that + - cease + - close proximity + - commence + - comply with + - concerning + - consequently + - consolidate + - constitutes + - demonstrate + - depart + - designate + - discontinue + - due to the fact that + - each and every + - economical + - eliminate + - elucidate + - employ + - endeavor + - enumerate + - equitable + - equivalent + - evaluate + - evidenced + - exclusively + - expedite + - expend + - expiration + - facilitate + - factual evidence + - feasible + - finalize + - first and foremost + - for all intents and purposes + - for the most part + - for the purpose of + - forfeit + - formulate + - have a tendency to + - honest truth + - however + - if and when + - impacted + - implement + - in a manner of speaking + - in a timely manner + - in a very real sense + - in accordance with + - in addition + - in all likelihood + - in an effort to + - in between + - in excess of + - in lieu of + - in light of the fact that + - in many cases + - in my opinion + - in order to + - in regard to + - in some instances + - in terms of + - in the case of + - in the event that + - in the final analysis + - in the nature of + - in the near future + - in the process of + - inception + - incumbent upon + - indicate + - indication + - initiate + - irregardless + - is applicable to + - is authorized to + - is responsible for + - it is + - it is essential + - it seems that + - it was + - magnitude + - maximum + - methodology + - minimize + - minimum + - modify + - monitor + - multiple + - necessitate + - nevertheless + - not certain + - not many + - not often + - not unless + - not unlike + - notwithstanding + - null and void + - numerous + - objective + - obligate + - obtain + - on the contrary + - on the other hand + - one particular + - optimum + - overall + - owing to the fact that + - participate + - particulars + - pass away + - pertaining to + - point in time + - portion + - possess + - preclude + - previously + - prior to + - prioritize + - procure + - proficiency + - provided that + - purchase + - put simply + - readily apparent + - refer back + - regarding + - relocate + - remainder + - remuneration + - requirement + - reside + - residence + - retain + - satisfy + - shall + - should you wish + - similar to + - solicit + - span across + - strategize + - subsequent + - substantial + - successfully complete + - sufficient + - terminate + - the month of + - the point I am trying to make + - therefore + - time period + - took advantage of + - transmit + - transpire + - type of + - until such time as + - utilization + - utilize + - validate + - various different + - what I mean to say is + - whether or not + - with respect to + - with the exception of + - witnessed diff --git a/.vale/styles/write-good/Weasel.yml b/.vale/styles/write-good/Weasel.yml new file mode 100644 index 00000000000..e29391444bc --- /dev/null +++ b/.vale/styles/write-good/Weasel.yml @@ -0,0 +1,207 @@ +extends: existence +message: "'%s' is a weasel word!" +ignorecase: true +level: warning +tokens: + - absolutely + - accidentally + - additionally + - allegedly + - alternatively + - angrily + - anxiously + - approximately + - awkwardly + - badly + - barely + - beautifully + - blindly + - boldly + - bravely + - brightly + - briskly + - bristly + - bubbly + - busily + - calmly + - carefully + - carelessly + - cautiously + - cheerfully + - clearly + - closely + - coldly + - completely + - consequently + - correctly + - courageously + - crinkly + - cruelly + - crumbly + - cuddly + - currently + - daily + - daringly + - deadly + - definitely + - deliberately + - doubtfully + - dumbly + - eagerly + - early + - easily + - elegantly + - enormously + - enthusiastically + - equally + - especially + - eventually + - exactly + - exceedingly + - exclusively + - extremely + - fairly + - faithfully + - fatally + - fiercely + - finally + - fondly + - few + - foolishly + - fortunately + - frankly + - frantically + - generously + - gently + - giggly + - gladly + - gracefully + - greedily + - happily + - hardly + - hastily + - healthily + - heartily + - helpfully + - honestly + - hourly + - hungrily + - hurriedly + - immediately + - impatiently + - inadequately + - ingeniously + - innocently + - inquisitively + - interestingly + - irritably + - jiggly + - joyously + - justly + - kindly + - largely + - lately + - lazily + - likely + - literally + - lonely + - loosely + - loudly + - loudly + - luckily + - madly + - many + - mentally + - mildly + - monthly + - mortally + - mostly + - mysteriously + - neatly + - nervously + - nightly + - noisily + - normally + - obediently + - occasionally + - only + - openly + - painfully + - particularly + - patiently + - perfectly + - politely + - poorly + - powerfully + - presumably + - previously + - promptly + - punctually + - quarterly + - quickly + - quietly + - rapidly + - rarely + - really + - recently + - recklessly + - regularly + - remarkably + - relatively + - reluctantly + - repeatedly + - rightfully + - roughly + - rudely + - sadly + - safely + - selfishly + - sensibly + - seriously + - sharply + - shortly + - shyly + - significantly + - silently + - simply + - sleepily + - slowly + - smartly + - smelly + - smoothly + - softly + - solemnly + - sparkly + - speedily + - stealthily + - sternly + - stupidly + - substantially + - successfully + - suddenly + - surprisingly + - suspiciously + - swiftly + - tenderly + - tensely + - thoughtfully + - tightly + - timely + - truthfully + - unexpectedly + - unfortunately + - usually + - very + - victoriously + - violently + - vivaciously + - warmly + - waverly + - weakly + - wearily + - weekly + - wildly + - wisely + - worldly + - wrinkly + - yearly diff --git a/.vale/styles/write-good/meta.json b/.vale/styles/write-good/meta.json new file mode 100644 index 00000000000..a115d2886ac --- /dev/null +++ b/.vale/styles/write-good/meta.json @@ -0,0 +1,4 @@ +{ + "feed": "https://github.com/errata-ai/write-good/releases.atom", + "vale_version": ">=1.0.0" +} diff --git a/docs/Taskfile.yml b/docs/Taskfile.yml new file mode 100644 index 00000000000..8f9cd383f48 --- /dev/null +++ b/docs/Taskfile.yml @@ -0,0 +1,11 @@ +--- +version: "3" + +# Running this taskfile ensures that markdowns are +# consistently formatted according to mdox rules. + +tasks: + default: + desc: "Run mdox to format markdowns" + cmds: + - mdox fmt --no-soft-wraps $(find -name '*.md') diff --git a/docs/plugins/go-development-flow.md b/docs/plugins/go-development-flow.md new file mode 100644 index 00000000000..9fb892daf89 --- /dev/null +++ b/docs/plugins/go-development-flow.md @@ -0,0 +1,271 @@ +--- +title: Custom Go plugin development flow +tags: + - custom plugin + - golang + - go plugin + - middleware + - debugging go plugins +description: Development flow working with Go Plugins +date: "2024-10-11" +--- + +We recommend that you familiarize yourself with the following official Go documentation to help you work effectively with Go plugins: + +- [The official plugin package documentation - Warnings](https://pkg.go.dev/plugin) +- [Tutorial: Getting started with multi-module workspaces](https://go.dev/doc/tutorial/workspaces) + +{{< note success >}} **Note** + +Plugins are currently supported only on Linux, FreeBSD, and macOS, making them unsuitable for applications intended to be portable. {{< /note >}} + +Plugins need to be compiled to native shared object code, which can then be loaded by Tyk Gateway. It's important to understand the need for plugins to be compiled using exactly the same environment and [build flags]({{< ref "product-stack/tyk-gateway/advanced-configurations/plugins/golang/go-development-flow#build-flags" >}}) as the Gateway. To simplify this and minimise the risk of compatibility problems, we recommend the use of [Go workspaces](https://go.dev/blog/get-familiar-with-workspaces), to provide a consistent environment. + +## Setting up your environment + +To develop plugins, you'll need: + +- Go (matching the version used in the Gateway, which you can determine using `go.mod`). +- Git to check out Tyk Gateway source code. +- A folder with the code that you want to build into plugins. + +We recommend that you set up a *Go workspace*, which, at the end, is going to contain: + +- `/tyk-release-x.y.z` - the Tyk Gateway source code +- `/plugins` - the plugins +- `/go.work` - the *Go workspace* file +- `/go.work.sum` - *Go workspace* package checksums + +Using the *Go workspace* ensures build compatibility between the plugins and Gateway. + +### 1. Checking out Tyk Gateway source code + +``` +git clone --branch release-5.3.6 https://github.com/TykTechnologies/tyk.git tyk-release-5.3.6 || true +``` + +This example uses a particular `release-5.3.6` branch, to match Tyk Gateway release 5.3.6. With newer `git` versions, you may pass `--branch v5.3.6` and it would use the tag. In case you want to use the tag it's also possible to navigate into the folder and issue `git checkout tags/v5.3.6`. + +### 2. Preparing the Go workspace + +Your Go workspace can be very simple: + +1. Create a `.go` file containing the code for your plugin. +2. Create a `go.mod` file for the plugin. +3. Ensure the correct Go version is in use. + +As an example, we can use the [CustomGoPlugin.go](https://github.com/TykTechnologies/custom-go-plugin/blob/master/go/src/CustomGoPlugin.go) sample as the source for our plugin as shown: + +``` +mkdir -p plugins +cd plugins +go mod init testplugin +go mod edit -go $(go mod edit -json go.mod | jq -r .Go) +wget -q https://raw.githubusercontent.com/TykTechnologies/custom-go-plugin/refs/heads/master/go/src/CustomGoPlugin.go +cd - +``` + +The following snippet provides you with a way to get the exact Go version used by Gateway from it's [go.mod](https://github.com/TykTechnologies/tyk/blob/release-5.3.6/go.mod#L3) file: + +- `go mod edit -json go.mod | jq -r .Go` (e.g. `1.22.7`) + +This should be used to ensure the version matches between gateway and the plugin. + +To summarize what was done: + +1. We created a plugins folder and initialzed a `go` project using `go mod` command. +2. Set the Go version of `go.mod` to match the one set in the Gateway. +3. Initialzied the project with sample plugin `go` code. + +At this point, we don't have a *Go workspace* but we will create one next so that we can effectively share the Gateway dependency across Go modules. + +### 3. Creating the Go workspace + +To set up the Go workspace, start in the directory that contains the Gateway and the Plugins folder. You'll first, create the `go.work` file to set up your Go workspace, and include the `tyk-release-5.3.6` and `plugins` folders. Then, navigate to the plugins folder to fetch the Gateway dependency at the exact commit hash and run `go mod tidy` to ensure dependencies are up to date. + +Follow these commands: + +``` +go work init ./tyk-release-5.3.6 +go work use ./plugins +commit_hash=$(cd tyk-release-5.3.6 && git rev-parse HEAD) +cd plugins && go get github.com/TykTechnologies/tyk@${commit_hash} && go mod tidy && cd - +``` + +The following snippet provides you to get the commit hash exactly, so it can be used with `go get`. + +- `git rev-parse HEAD` + +The Go workspace file (`go.work`) should look like this: + +``` +go 1.22.7 + +use ( + ./plugins + ./tyk-release-5.3.6 +) +``` + +### 4. Building and validating the plugin + +Now that your *Go workspace* is ready, you can build your plugin as follows: + +``` +cd tyk-release-5.3.6 && go build -tags=goplugin -trimpath . && cd - +cd plugins && go build -trimpath -buildmode=plugin . && cd - +``` + +These steps build both the Gateway and the plugin. + +You can use the Gateway binary that you just built to test that your new plugin loads into the Gateway without having to configure and then make a request to an API using this command: + +``` +./tyk-release-5.3.6/tyk plugin load -f plugins/testplugin.so -s AuthCheck +``` + +You should see an output similar to: + +``` +time="Oct 14 13:39:55" level=info msg="--- Go custom plugin init success! ---- " +[file=plugins/testplugin.so, symbol=AuthCheck] loaded ok, got 0x76e1aeb52140 +``` + +The log shows that the plugin has correctly loaded into the Gateway and that its `init` function has been successfully invoked. + +### 5. Summary + +In the preceding steps we have put together an end-to-end build environment for both the Gateway and the plugin. Bear in mind that runtime environments may have additional restrictions beyond Go version and build flags to which the plugin developer must pay attention. + +Compatibility in general is a big concern when working with Go plugins: as the plugins are tightly coupled to the Gateway, consideration must always be made for the build restrictions enforced by environment and configuration options. + +Continue with [Loading Go Plugins into Tyk](https://tyk.io/docs/product-stack/tyk-gateway/advanced-configurations/plugins/golang/loading-go-plugins/). + +## Debugging Golang Plugins + +Plugins are native Go code compiled to a binary shared object file. The code may depend on `cgo` and require libraries like `libc` provided by the runtime environment. The following are some debugging steps for diagnosing issues arising from using plugins. + +### Warnings + +The [Plugin package - Warnings](https://pkg.go.dev/plugin#hdr-Warnings) section in the Go documentation outlines several requirements which can't be ignored when working with plugins. The most important restriction is the following: + +> Runtime crashes are likely to occur unless all parts of the program (the application and all its plugins) are compiled using exactly the same version of the toolchain, the same build tags, and the same values of certain flags and environment variables. + +We provide the *Tyk Plugin Compiler* docker image, which we strongly recommend is used to build plugins compatible with the official Gateway releases. This tool provides the cross compilation toolchain, Go version used to build the release, and ensures that compatible flags are used when compiling plugins, like `-trimpath`, `CC`, `CGO_ENABLED`, `GOOS`, `GOARCH`. + +The *Plugin Compiler* also works around known Go issues such as: + +- https://github.com/golang/go/issues/19004 +- https://www.reddit.com/r/golang/comments/qxghjv/plugin_already_loaded_when_a_plugin_is_loaded/ + +Supplying the argument `build_id` to the *Plugin Compiler* ensures the same plugin can be rebuilt. The *Plugin Compiler* does this by replacing the plugin `go.mod` module path. + +Continue with [Tyk Plugin Compiler](https://tyk.io/docs/product-stack/tyk-gateway/advanced-configurations/plugins/golang/go-plugin-compiler/). + +### Using Incorrect Build Flags + +When working with Go plugins, it's easy to miss the restriction that the plugin at the very least must be built with the same Go version, and the same flags (notably `-trimpath`) as the Tyk Gateway on which it is to be used. + +If you miss an argument (for example `-trimpath`) when building the plugin, the Gateway will report an error when your API attempts to load the plugin, for example: + +``` +task: [test] cd tyk-release-5.3.6 && go build -tags=goplugin -trimpath . +task: [test] cd plugins && go build -buildmode=plugin . +task: [test] ./tyk-release-5.3.6/tyk plugin load -f plugins/testplugin.so -s AuthCheck +tyk: error: unexpected error: plugin.Open("plugins/testplugin"): plugin was built with a different version of package internal/goarch, try --help +``` + +Usually when the error hints at a standard library package, the build flags between the Gateway and plugin binaries don't match. + +Other error messages may be reported, depending on what triggered the issue. For example, if you omitted `-race` in the plugin but the gateway was built with `-race`, the following error will be reported: + +``` +plugin was built with a different version of package runtime/internal/sys, try --help +``` + +Strictly speaking: + +- Build flags like `-trimpath`, `-race` need to match. +- Go toolchain / build env needs to be exactly the same. +- For cross compilation you must use the same `CC` value for the build (CGO). +- `CGO_ENABLED=1`, `GOOS`, `GOARCH` must match with runtime. + +When something is off, you can check what is different by using the `go version -m` command for the Gateway (`go version -m tyk`) and plugin (`go version -m plugin.so`). Inspecting and comparing the output of `build` tokens usually yields the difference that caused the compatibility issue. + +### Plugin Compatibility Issues + +Below are some common situations where dependencies might cause issues: + +- The `Gateway` has a dependency without a `go.mod` file, but the plugin needs to use it. +- Both the `Gateway` and the plugin share a dependency. In this case, the plugin must use the exact same version as the `Gateway`. +- The plugin requires a different version of a shared dependency. + +Here’s how to handle each case: + +**Case 1: Gateway dependency lacks `go.mod`** + +- The plugin depends on the `Gateway`, which uses dependency *A*. +- *A* doesn’t have a `go.mod` file, so a pseudo version is generated during the build. +- Result: The build completes, but the plugin fails to load due to a version mismatch. + +**Solution:** Update the code to remove dependency *A*, or use a version of *A* that includes a `go.mod` file. + +**Case 2: Shared dependency with version matching** + +- The plugin and `Gateway` share a dependency, and this dependency includes a `go.mod` file. +- The version matches, and the dependency is promoted to *direct* in `go.mod`. +- Outcome: You’ll need to keep this dependency version in sync with the `Gateway`. + +**Case 3: Plugin requires a different version of a shared dependency** + +- The plugin and `Gateway` share a dependency, but the plugin needs a different version. +- If the other version is a major release (e.g., `/v4`), it’s treated as a separate package, allowing both versions to coexist. +- If it’s just a minor/patch difference, the plugin will likely fail to load due to a version conflict. + +**Recommendation:** For best results, use Go package versions that follow the Go module versioning (metaversion). However, keep in mind that many `Gateway` dependencies use basic `v1` semantic versioning, which doesn’t always enforce strict versioned import paths. + +### List plugin symbols + +Sometimes it's useful to list symbols from a plugin. For example, we can list the symbols as they are compiled into our testplugin: + +``` +# nm -gD testplugin.so | grep testplugin +00000000014db4b0 R go:link.pkghashbytes.testplugin +000000000170f7d0 D go:link.pkghash.testplugin +000000000130f5e0 T testplugin.AddFooBarHeader +000000000130f900 T testplugin.AddFooBarHeader.deferwrap1 +000000000130f980 T testplugin.AuthCheck +0000000001310100 T testplugin.AuthCheck.deferwrap1 +000000000130f540 T testplugin.init +0000000001310ce0 T testplugin.init.0 +0000000001ce9580 D testplugin..inittask +0000000001310480 T testplugin.InjectConfigData +0000000001310180 T testplugin.InjectMetadata +0000000001d2a3e0 B testplugin.logger +0000000001310cc0 T testplugin.main +0000000001310820 T testplugin.MakeOutboundCall +0000000001310c40 T testplugin.MakeOutboundCall.deferwrap1 +``` + +This command prints other symbols that are part of the binary. In the worst case, a build compatibility issue may cause a crash in the Gateway due to an unrecoverable error and this can be used to further debug the binaries produced. + +A very basic check to ensure Gateway/plugin compatibility is using the built in `go version -m `: + +``` +[output truncated] + build -buildmode=exe + build -compiler=gc + build -race=true + build -tags=goplugin + build -trimpath=true + build CGO_ENABLED=1 + build GOARCH=amd64 + build GOOS=linux + build GOAMD64=v1 + build vcs=git + build vcs.revision=1db1935d899296c91a55ba528e7b653aec02883b + build vcs.time=2024-09-24T12:54:26Z + build vcs.modified=false +``` + +These options should match between the Gateway binary and the plugin. You can use the command for both binaries and then compare the outputs. diff --git a/docs/plugins/tests/.gitignore b/docs/plugins/tests/.gitignore new file mode 100644 index 00000000000..cdd21ea7a76 --- /dev/null +++ b/docs/plugins/tests/.gitignore @@ -0,0 +1,3 @@ +/plugins +/tyk-release* +/go.work* diff --git a/docs/plugins/tests/Taskfile.yml b/docs/plugins/tests/Taskfile.yml new file mode 100644 index 00000000000..59d663387ae --- /dev/null +++ b/docs/plugins/tests/Taskfile.yml @@ -0,0 +1,61 @@ +--- +version: "3" + +tasks: + all: + desc: "Do everything" + cmds: + - task: checkout + vars: + version: release-5.3.6 + - task: clean + - task: workspace + - task: test + + checkout: + desc: "Checking out tyk" + cmds: + - git clone --branch {{.version}} https://github.com/TykTechnologies/tyk.git tyk-{{.version}} || true + + workspace: + desc: "Create workspace" + vars: + release: tyk-release-5.3.6 + commit: + sh: git rev-parse HEAD + go: + sh: go mod edit -json ./{{.release}}/go.mod | jq .Go -r + cmds: + - mkdir -p plugins + - task: workspace:plugins + vars: + go: '{{.go}}' + - go work init ./{{.release}} + - go work use ./plugins + - cd plugins && go get github.com/TykTechnologies/tyk@{{.commit}} + + workspace:plugins: + internal: true + desc: "Setup plugins" + dir: plugins + requires: + vars: [go] + cmds: + - rm -f go.mod go.sum + - go mod init testplugin + - go mod edit -go "{{.go}}" + - wget -q https://raw.githubusercontent.com/TykTechnologies/custom-go-plugin/refs/heads/master/go/src/CustomGoPlugin.go + + test: + desc: "Test plugin build" + vars: + args: -trimpath -race + cmds: + - cd tyk-release-5.3.6 && go build -tags=goplugin {{.args}} . + - cd plugins && go build {{.args}} -buildmode=plugin . + - ./tyk-release-5.3.6/tyk plugin load -f plugins/testplugin.so -s AuthCheck + + clean: + desc: "Clean workspace" + cmds: + - rm -rf plugins go.work* diff --git a/docs/swagger.md b/docs/schemas/swagger.md similarity index 88% rename from docs/swagger.md rename to docs/schemas/swagger.md index e7b85797b53..99e69558f96 100644 --- a/docs/swagger.md +++ b/docs/schemas/swagger.md @@ -25,8 +25,8 @@ We used the Go library [openapi-go](https://github.com/swaggest/openapi-go) beca 4. Run the command `make generate` in this directory. 5. After running this command, a `swagger.yml` file containing all the gateway endpoints will be generated in the same directory. 6. What the `make generate` command does: - - The command is defined as: `rm -f swagger.yml && go run main.go && redocly lint swagger.yml`. - - It first removes the existing `swagger.yml` file (if any), generates a new `swagger.yml` file, and finally lints the file using Redocly. + - The command is defined as: `rm -f swagger.yml && go run main.go && redocly lint swagger.yml`. + - It first removes the existing `swagger.yml` file (if any), generates a new `swagger.yml` file, and finally lints the file using Redocly. ## File Structure