Merge pull request #5 from yandex-cloud/feature/defaults-metadata-sup…

…port

defaults metadata support

README.md

@@ -10,16 +10,22 @@ for
 | Key | Description | 
 |:---|:---|
 | `group_id`        | (_optional_) [Log group](https://cloud.yandex.ru/docs/logging/concepts/log-group) ID. Has higher priority than `folder_id`. |
-| `folder_id`       | (_optional_) Folder ID. Has lower priority than `group_id`. Can be auto-detected via [metadata service](https://cloud.yandex.com/en/docs/compute/concepts/vm-metadata) if `group_id` and `folder_id` are not set. Metadata url can be configured through environment variable `YC_METADATA_URL`. By default it's `http://169.254.169.254`. |
+| `folder_id`       | (_optional_) Folder ID. Has lower priority than `group_id`. Can be auto-detected via [metadata service](#metadata) if `group_id` and `folder_id` are not set. |
 | `resource_type`   | (_optional_) Resource type of log entries | 
 | `resource_id`     | (_optional_) Resource id of log entries | 
 | `message_tag_key` | Key of the field to be assigned to the message tag. By default, will be skipped | 
 | `message_key`     | Key of the field, which will go to `message` attribute of LogEntry | 
 | `level_key`       | Key of the field, which contains log level, optional |
 | `default_level`   | (_optional_) Default level for messages, i.e., `INFO` |
-| `default_payload` | (_optional_) String with default JSON payload for entries (will be merged together with custom entry payload) |
+| `default_payload` | (_optional_) String with default JSON payload for entries (will be merged together with custom entry payload). You can define templated value as follows: `{{key:default}}` or `{{key}}`. All strings matched this template will be replaced by metadata values found via [metadata service](#metadata). You also can use this template as a JSON value (without quotes), so that JSON struct or array will be included in default payload. |
 | `authorization`   | see [Authorization](#authorization) section below |
 
+### Metadata
+
+[Metadata service documentation](https://cloud.yandex.com/en/docs/compute/concepts/vm-metadata).
+
+Metadata url can be configured through environment variable `YC_METADATA_URL`. By default it's `http://169.254.169.254`.
+
 ### Authorization
 
 Configuration parameter `authorization` may have one of the following values:
@@ -48,6 +54,6 @@ Example:
     message_key     text
     level_key       severity
     default_level   WARN
-    default_payload {"num":5, "str": "string", "bool": true}
+    default_payload {"num":5, "str": "string", "bool": true, "host":"{{instance/hostname}}", "not-found":"{{not/found:default}}", "struct": {{instance/disks/0}}, "array": {{instance/disks}} }
     authorization   instance-service-account
 ```

config.go

@@ -76,6 +76,11 @@ func getDefaults(plugin unsafe.Pointer) (*logging.LogEntryDefaults, error) {
 
 	defaultPayload := getConfigKey(plugin, keyDefaultPayload)
 	if len(defaultPayload) > 0 {
+		var err error
+		defaultPayload, err = parsePayload(defaultPayload)
+		if err != nil {
+			return nil, err
+		}
 		payload, err := payloadFromString(defaultPayload)
 		if err != nil {
 			return nil, err

go.mod

@@ -4,8 +4,10 @@ go 1.17
 
 require (
 	github.com/fluent/fluent-bit-go v0.0.0-20201210173045-3fd1e0486df2
+	github.com/startdusk/strnaming v0.7.0
 	github.com/yandex-cloud/go-genproto v0.0.0-20211202135052-789603780fb2
 	github.com/yandex-cloud/go-sdk v0.0.0-20211206101223-7c4e7926bf53
+	go.uber.org/multierr v1.7.0
 	google.golang.org/grpc v1.42.0
 	google.golang.org/protobuf v1.27.1
 )
@@ -17,6 +19,7 @@ require (
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/ugorji/go/codec v1.1.7 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
 	golang.org/x/net v0.0.0-20211205041911-012df41ee64c // indirect
 	golang.org/x/sys v0.0.0-20211205182925-97ca703d548d // indirect
 	golang.org/x/text v0.3.7 // indirect

go.sum

@@ -15,8 +15,11 @@ github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -68,21 +71,37 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/startdusk/strnaming v0.7.0 h1:b/jLgK+A+fu2cJ/hcMBLnUUgyuhlNZ3/WafQNr+4YFY=
+github.com/startdusk/strnaming v0.7.0/go.mod h1:5wBRJtNUqoKYH+KICeRKFcWBkOt9pRKrsqNzWInAGkw=
 github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/gjson v1.9.0/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk=
+github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
+github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 github.com/yandex-cloud/go-genproto v0.0.0-20211202135052-789603780fb2 h1:NOMh7Oy10aaqHWDajqN1whPrFD2RViJOqsWeAOtW7fQ=
 github.com/yandex-cloud/go-genproto v0.0.0-20211202135052-789603780fb2/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE=
 github.com/yandex-cloud/go-sdk v0.0.0-20211206101223-7c4e7926bf53 h1:SOTpRQ+mqmu3IbemdahJSev3Srp7CYNUzjDbFlBPeHU=
 github.com/yandex-cloud/go-sdk v0.0.0-20211206101223-7c4e7926bf53/go.mod h1:fjnP1VWobOVjfhIfeOZvq+zxhdTbqsddVXUm4psPkRU=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec=
+go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -169,7 +188,8 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

metadata.go

@@ -6,45 +6,130 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
+	"strconv"
+	"strings"
 	"time"
 
+	"google.golang.org/protobuf/types/known/structpb"
+
+	"github.com/startdusk/strnaming"
 	ycsdk "github.com/yandex-cloud/go-sdk"
 )
 
-func getMetadataValue(key string) (string, error) {
+func getMetadataUrl() string {
 	const (
 		keyMetadataUrlEnv = "YC_METADATA_URL"
 		urlSuffix         = "/computeMetadata/v1/"
-		requestTimeout    = 5 * time.Second
 	)
-
 	metadataEndpoint := os.Getenv(keyMetadataUrlEnv)
 	if len(metadataEndpoint) == 0 {
 		metadataEndpoint = "http://" + ycsdk.InstanceMetadataAddr
 	}
-	urlMetadata := metadataEndpoint + urlSuffix + key
+	return metadataEndpoint + urlSuffix
+}
+
+func getMetadataValue(key string) (string, error) {
+	const requestTimeout = 5 * time.Second
+
+	urlMetadata := getMetadataUrl() + key
 
 	client := http.Client{}
 	req, err := http.NewRequest(http.MethodGet, urlMetadata, nil)
 	if err != nil {
-		return "", fmt.Errorf("could not make request to autodetect folder ID: %s", err.Error())
+		return "", fmt.Errorf("could not make request to get metadata value by key %q: %s", key, err.Error())
 	}
 	req.Header.Set("Metadata-Flavor", "Google")
 
 	ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
 	defer cancel()
 	resp, err := client.Do(req.WithContext(ctx))
 	if err != nil {
-		return "", fmt.Errorf("could not get instance metadata to autodetect folder ID: %s", err.Error())
+		return "", fmt.Errorf("could not get instance metadata value by key %q: %s", key, err.Error())
 	}
 	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("request to autodetect folder ID returned status other than OK: %s", resp.Status)
+		return "", fmt.Errorf("request to get metadata value by key %q returned status other than OK: %s", key, resp.Status)
 	}
 
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return "", fmt.Errorf("response body returned by request to autodetect folder ID read failed: %s", err.Error())
+		return "", fmt.Errorf("response body returned by request to get metadata value by key %q read failed: %s", key, err.Error())
 	}
 
 	return string(body), nil
 }
+
+func getAllMetadata() (*structpb.Struct, error) {
+	const (
+		queryParam     = "?recursive=true"
+		requestTimeout = 5 * time.Second
+	)
+
+	urlMetadata := getMetadataUrl() + queryParam
+
+	client := http.Client{}
+	req, err := http.NewRequest(http.MethodGet, urlMetadata, nil)
+	if err != nil {
+		return nil, fmt.Errorf("could not make request to get all metadata: %s", err.Error())
+	}
+	req.Header.Set("Metadata-Flavor", "Google")
+
+	ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
+	defer cancel()
+	resp, err := client.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, fmt.Errorf("could not get all metadata: %s", err.Error())
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("request to get all metadata returned status other than OK: %s", resp.Status)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("response body returned by request to get all metadata read failed: %s", err.Error())
+	}
+
+	metadataStruct := new(structpb.Struct)
+	err = metadataStruct.UnmarshalJSON(body)
+	if err != nil {
+		return nil, fmt.Errorf("could not unmarshal response body returned by request to get all metadata: %s", err.Error())
+	}
+
+	return metadataStruct, nil
+}
+
+func getCachedMetadataValue(metadata *structpb.Struct, key string) (string, error) {
+	toCamel := strnaming.NewCamel()
+	toCamel.WithDelimiter('-')
+
+	path := strings.Split(key, "/")
+	cur := structpb.NewStructValue(metadata)
+	for _, p := range path {
+		p = toCamel.Convert(p)
+
+		switch cur.GetKind().(type) {
+		case *structpb.Value_StructValue:
+			cur = cur.GetStructValue().GetFields()[p]
+		case *structpb.Value_ListValue:
+			index, err := strconv.Atoi(p)
+			if err != nil {
+				return "", fmt.Errorf("incorrect metadata key: %q, expected number instead of %q", key, p)
+			}
+			cur = cur.GetListValue().GetValues()[index]
+		default:
+			return "", fmt.Errorf("incorrect metadata key: %q", key)
+		}
+	}
+	if cur == nil {
+		return "", fmt.Errorf("incorrect metadata key: %q", key)
+	}
+
+	if _, ok := cur.GetKind().(*structpb.Value_StringValue); ok {
+		return cur.GetStringValue(), nil
+	}
+
+	content, err := cur.MarshalJSON()
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal JSON from value by key %q: %s", key, err.Error())
+	}
+	return string(content), nil
+}

util.go

@@ -2,11 +2,13 @@ package main
 
 import (
 	"fmt"
+	"regexp"
 	"strings"
 	"time"
 	"unicode/utf8"
 
 	"github.com/fluent/fluent-bit-go/output"
+	"go.uber.org/multierr"
 	"google.golang.org/protobuf/types/known/structpb"
 
 	"github.com/yandex-cloud/go-genproto/yandex/cloud/logging/v1"
@@ -28,6 +30,42 @@ func payloadFromString(payload string) (*structpb.Struct, error) {
 	return result, nil
 }
 
+var templateReg = regexp.MustCompile(`{{[^{}]+}}`)
+
+func parsePayload(payload string) (string, error) {
+	metadataCache, err := getAllMetadata()
+	if err != nil {
+		return "", err
+	}
+	var multierror error
+	parsed := templateReg.ReplaceAllStringFunc(payload, func(t string) string {
+		res, err := replaceTemplate(t, metadataCache)
+		if err != nil {
+			multierror = multierr.Append(multierror, err)
+		}
+		return res
+	})
+	return parsed, multierror
+}
+
+func replaceTemplate(t string, metadata *structpb.Struct) (string, error) {
+	str := t[2 : len(t)-2]
+
+	fields := strings.Split(str, ":")
+	key := fields[0]
+	defaultValue := ""
+	if len(fields) >= 2 {
+		defaultValue = fields[1]
+	}
+
+	metadataValue, err := getCachedMetadataValue(metadata, key)
+	if err != nil {
+		fmt.Printf("yc-logging: using default value %q for template %q because of error: %s\n", defaultValue, t, err.Error())
+		return defaultValue, nil
+	}
+	return metadataValue, nil
+}
+
 func toString(raw interface{}) string {
 	switch typed := raw.(type) {
 	case string: