From ff4c3e57b8dff8b68526ebf76e509103062bc056 Mon Sep 17 00:00:00 2001 From: Andrei Kamarouski Date: Sat, 14 Sep 2024 14:47:13 +0200 Subject: [PATCH 01/12] start --- handlers/tasks.go | 10 +++++----- task-definitions-service/README.md | 1 + task-definitions-service/helper.go | 19 +++++++++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 task-definitions-service/README.md create mode 100644 task-definitions-service/helper.go diff --git a/handlers/tasks.go b/handlers/tasks.go index cc80d5e..0bcdb96 100644 --- a/handlers/tasks.go +++ b/handlers/tasks.go @@ -64,16 +64,16 @@ func Create(c *gin.Context) { // l = l.WithField("workspace", workspace) // } - reqCaps, configurationCaps, err := capabilities.ParseRequestCapabilities(c.Request.Body) + requestCapabilities, containerConfiguration, err := capabilities.ParseRequestCapabilities(c.Request.Body) if err != nil { l.WithError(err).Error("Failed to process capabilities") c.Error(utils.InvalidArgErr(fmt.Errorf("failed to process capabilities"), err.Error())).SetType(gin.ErrorTypePublic) return } - log.Trace("Request capabilitites: ", reqCaps.ToMap()) - log.Trace("Container configuration: ", configurationCaps.ToMap()) + log.Trace("Request capabilitites: ", requestCapabilities.ToMap()) + log.Trace("Container configuration: ", containerConfiguration.ToMap()) - env, routerUUID, err := environment.BuildEnvForTaskDefinitionOverride(workspace, configurationCaps) + env, routerUUID, err := environment.BuildEnvForTaskDefinitionOverride(workspace, containerConfiguration) if err != nil { log.WithError(err).Error("Failed to build execution environment") c.Error(utils.CreationErr(fmt.Errorf("failed to create executor"), err.Error())).SetType(gin.ErrorTypePublic) @@ -87,7 +87,7 @@ func Create(c *gin.Context) { env, workspace, routerUUID, - reqCaps, + requestCapabilities, c, l, ).StartService(startupTime) diff --git a/task-definitions-service/README.md b/task-definitions-service/README.md new file mode 100644 index 0000000..9c300b4 --- /dev/null +++ b/task-definitions-service/README.md @@ -0,0 +1 @@ +independent service that provides methods for getting/generating definitions udid \ No newline at end of file diff --git a/task-definitions-service/helper.go b/task-definitions-service/helper.go new file mode 100644 index 0000000..f2033c6 --- /dev/null +++ b/task-definitions-service/helper.go @@ -0,0 +1,19 @@ +package definitionsservice + +import ( + "github.com/zebrunner/esg/environment" + "github.com/zebrunner/esg/service" +) + +func getTaskDefinitionId(configuration *environment.ExecutionEnvironment) (id *int64, err error) { + taskDefinition, err := service.CreateTaskDefinition( + configuration.ContainerDefinitions(), + configuration.Volume(), + configuration.TaskDefinitionFamily, + configuration.TaskRoleArn, + ) + if err != nil { + return + } + return taskDefinition.Revision, nil +} From 3282f9f398665af480e0ec754ff46e77cbbb9559 Mon Sep 17 00:00:00 2001 From: Andrei Kamarouski Date: Sat, 14 Sep 2024 16:44:05 +0200 Subject: [PATCH 02/12] implement getDefinitionById --- go.mod | 19 +- go.sum | 14 ++ task-definitions-service/definitions.proto | 18 ++ .../definitions/definitions.pb.go | 214 ++++++++++++++++++ .../definitions/definitions_grpc.pb.go | 121 ++++++++++ .../definitions/helper.go | 37 +++ task-definitions-service/helper.go | 19 -- task-definitions-service/main.go | 94 ++++++++ task-definitions-service/proto.sh | 8 + 9 files changed, 518 insertions(+), 26 deletions(-) create mode 100644 task-definitions-service/definitions.proto create mode 100644 task-definitions-service/definitions/definitions.pb.go create mode 100644 task-definitions-service/definitions/definitions_grpc.pb.go create mode 100644 task-definitions-service/definitions/helper.go delete mode 100644 task-definitions-service/helper.go create mode 100644 task-definitions-service/main.go create mode 100644 task-definitions-service/proto.sh diff --git a/go.mod b/go.mod index 2d4323b..fc907a3 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/zebrunner/esg -go 1.18 +go 1.21 + +toolchain go1.22.3 require ( github.com/aerokube/util v1.0.1 @@ -13,16 +15,18 @@ require ( github.com/redis/go-redis/v9 v9.5.1 github.com/sethvargo/go-password v0.2.0 github.com/sirupsen/logrus v1.9.3 - golang.org/x/crypto v0.21.0 - golang.org/x/net v0.22.0 - golang.org/x/sys v0.18.0 // indirect + golang.org/x/crypto v0.24.0 + golang.org/x/net v0.26.0 + golang.org/x/sys v0.21.0 // indirect ) require github.com/davegardnerisme/deephash v0.0.0-20210406090112-6d072427d830 +require google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect + require ( github.com/bytedance/sonic v1.9.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect @@ -48,8 +52,9 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/text v0.14.0 // indirect - google.golang.org/protobuf v1.33.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/grpc v1.66.2 + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 8898493..5348673 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,7 @@ github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= @@ -212,6 +213,7 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -222,6 +224,8 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -240,6 +244,8 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -249,6 +255,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -263,8 +271,14 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= +google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/task-definitions-service/definitions.proto b/task-definitions-service/definitions.proto new file mode 100644 index 0000000..4129e3a --- /dev/null +++ b/task-definitions-service/definitions.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; +package definitions; + +option go_package = "./definitions"; + +service Service { + rpc getTaskDefinitionId(Configuration) returns (Definition); +} + +message Configuration { + // configuration should be json-marshalled environment.Environment object + //todo think about another options how send no-proto message + bytes configuration = 1; +} + +message Definition { + int64 revision = 1; +} diff --git a/task-definitions-service/definitions/definitions.pb.go b/task-definitions-service/definitions/definitions.pb.go new file mode 100644 index 0000000..33f81a6 --- /dev/null +++ b/task-definitions-service/definitions/definitions.pb.go @@ -0,0 +1,214 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.34.2 +// protoc v3.21.12 +// source: definitions.proto + +package definitions + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Configuration struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Configuration []byte `protobuf:"bytes,1,opt,name=configuration,proto3" json:"configuration,omitempty"` +} + +func (x *Configuration) Reset() { + *x = Configuration{} + if protoimpl.UnsafeEnabled { + mi := &file_definitions_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Configuration) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Configuration) ProtoMessage() {} + +func (x *Configuration) ProtoReflect() protoreflect.Message { + mi := &file_definitions_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Configuration.ProtoReflect.Descriptor instead. +func (*Configuration) Descriptor() ([]byte, []int) { + return file_definitions_proto_rawDescGZIP(), []int{0} +} + +func (x *Configuration) GetConfiguration() []byte { + if x != nil { + return x.Configuration + } + return nil +} + +type Definition struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Revision int64 `protobuf:"varint,1,opt,name=revision,proto3" json:"revision,omitempty"` +} + +func (x *Definition) Reset() { + *x = Definition{} + if protoimpl.UnsafeEnabled { + mi := &file_definitions_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Definition) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Definition) ProtoMessage() {} + +func (x *Definition) ProtoReflect() protoreflect.Message { + mi := &file_definitions_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Definition.ProtoReflect.Descriptor instead. +func (*Definition) Descriptor() ([]byte, []int) { + return file_definitions_proto_rawDescGZIP(), []int{1} +} + +func (x *Definition) GetRevision() int64 { + if x != nil { + return x.Revision + } + return 0 +} + +var File_definitions_proto protoreflect.FileDescriptor + +var file_definitions_proto_rawDesc = []byte{ + 0x0a, 0x11, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x12, 0x0b, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x22, 0x35, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x24, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x28, 0x0a, 0x0a, 0x44, 0x65, 0x66, 0x69, 0x6e, + 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, + 0x6e, 0x32, 0x55, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4a, 0x0a, 0x13, + 0x67, 0x65, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x49, 0x64, 0x12, 0x1a, 0x2e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, + 0x17, 0x2e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x44, 0x65, + 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x0f, 0x5a, 0x0d, 0x2e, 0x2f, 0x64, 0x65, + 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, +} + +var ( + file_definitions_proto_rawDescOnce sync.Once + file_definitions_proto_rawDescData = file_definitions_proto_rawDesc +) + +func file_definitions_proto_rawDescGZIP() []byte { + file_definitions_proto_rawDescOnce.Do(func() { + file_definitions_proto_rawDescData = protoimpl.X.CompressGZIP(file_definitions_proto_rawDescData) + }) + return file_definitions_proto_rawDescData +} + +var file_definitions_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_definitions_proto_goTypes = []any{ + (*Configuration)(nil), // 0: definitions.Configuration + (*Definition)(nil), // 1: definitions.Definition +} +var file_definitions_proto_depIdxs = []int32{ + 0, // 0: definitions.Service.getTaskDefinitionId:input_type -> definitions.Configuration + 1, // 1: definitions.Service.getTaskDefinitionId:output_type -> definitions.Definition + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_definitions_proto_init() } +func file_definitions_proto_init() { + if File_definitions_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_definitions_proto_msgTypes[0].Exporter = func(v any, i int) any { + switch v := v.(*Configuration); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_definitions_proto_msgTypes[1].Exporter = func(v any, i int) any { + switch v := v.(*Definition); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_definitions_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_definitions_proto_goTypes, + DependencyIndexes: file_definitions_proto_depIdxs, + MessageInfos: file_definitions_proto_msgTypes, + }.Build() + File_definitions_proto = out.File + file_definitions_proto_rawDesc = nil + file_definitions_proto_goTypes = nil + file_definitions_proto_depIdxs = nil +} diff --git a/task-definitions-service/definitions/definitions_grpc.pb.go b/task-definitions-service/definitions/definitions_grpc.pb.go new file mode 100644 index 0000000..ee708e2 --- /dev/null +++ b/task-definitions-service/definitions/definitions_grpc.pb.go @@ -0,0 +1,121 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc v3.21.12 +// source: definitions.proto + +package definitions + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + Service_GetTaskDefinitionId_FullMethodName = "/definitions.Service/getTaskDefinitionId" +) + +// ServiceClient is the client API for Service service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ServiceClient interface { + GetTaskDefinitionId(ctx context.Context, in *Configuration, opts ...grpc.CallOption) (*Definition, error) +} + +type serviceClient struct { + cc grpc.ClientConnInterface +} + +func NewServiceClient(cc grpc.ClientConnInterface) ServiceClient { + return &serviceClient{cc} +} + +func (c *serviceClient) GetTaskDefinitionId(ctx context.Context, in *Configuration, opts ...grpc.CallOption) (*Definition, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Definition) + err := c.cc.Invoke(ctx, Service_GetTaskDefinitionId_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ServiceServer is the server API for Service service. +// All implementations must embed UnimplementedServiceServer +// for forward compatibility. +type ServiceServer interface { + GetTaskDefinitionId(context.Context, *Configuration) (*Definition, error) + mustEmbedUnimplementedServiceServer() +} + +// UnimplementedServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedServiceServer struct{} + +func (UnimplementedServiceServer) GetTaskDefinitionId(context.Context, *Configuration) (*Definition, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetTaskDefinitionId not implemented") +} +func (UnimplementedServiceServer) mustEmbedUnimplementedServiceServer() {} +func (UnimplementedServiceServer) testEmbeddedByValue() {} + +// UnsafeServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ServiceServer will +// result in compilation errors. +type UnsafeServiceServer interface { + mustEmbedUnimplementedServiceServer() +} + +func RegisterServiceServer(s grpc.ServiceRegistrar, srv ServiceServer) { + // If the following call pancis, it indicates UnimplementedServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&Service_ServiceDesc, srv) +} + +func _Service_GetTaskDefinitionId_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Configuration) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ServiceServer).GetTaskDefinitionId(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Service_GetTaskDefinitionId_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ServiceServer).GetTaskDefinitionId(ctx, req.(*Configuration)) + } + return interceptor(ctx, in, info, handler) +} + +// Service_ServiceDesc is the grpc.ServiceDesc for Service service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Service_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "definitions.Service", + HandlerType: (*ServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "getTaskDefinitionId", + Handler: _Service_GetTaskDefinitionId_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "definitions.proto", +} diff --git a/task-definitions-service/definitions/helper.go b/task-definitions-service/definitions/helper.go new file mode 100644 index 0000000..bb9e48d --- /dev/null +++ b/task-definitions-service/definitions/helper.go @@ -0,0 +1,37 @@ +package definitions + +import ( + sync "sync" + + log "github.com/sirupsen/logrus" + "github.com/zebrunner/esg/config" + "github.com/zebrunner/esg/utils" + grpc "google.golang.org/grpc" +) + +type Client struct { + sync.Mutex + Client ServiceClient +} + +var ( + client = &Client{} +) + +// Get gRPC client for interacting with the task definition service +// if cannot connect to service cause os.Exit +func GetClient() ServiceClient { + defer func() { + client.Unlock() + }() + client.Lock() + if client.Client != nil { + return client.Client + } + c, err := grpc.NewClient(config.Conf.DefinitionsConnectionString) + if err != nil { + utils.ExitWithError(err, "failed to initialize gRPC definitions client", log.NewEntry(log.StandardLogger())) + } + client.Client = NewServiceClient(c) + return client.Client +} diff --git a/task-definitions-service/helper.go b/task-definitions-service/helper.go deleted file mode 100644 index f2033c6..0000000 --- a/task-definitions-service/helper.go +++ /dev/null @@ -1,19 +0,0 @@ -package definitionsservice - -import ( - "github.com/zebrunner/esg/environment" - "github.com/zebrunner/esg/service" -) - -func getTaskDefinitionId(configuration *environment.ExecutionEnvironment) (id *int64, err error) { - taskDefinition, err := service.CreateTaskDefinition( - configuration.ContainerDefinitions(), - configuration.Volume(), - configuration.TaskDefinitionFamily, - configuration.TaskRoleArn, - ) - if err != nil { - return - } - return taskDefinition.Revision, nil -} diff --git a/task-definitions-service/main.go b/task-definitions-service/main.go new file mode 100644 index 0000000..80fab6c --- /dev/null +++ b/task-definitions-service/main.go @@ -0,0 +1,94 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "net" + "os" + "os/signal" + "syscall" + + log "github.com/sirupsen/logrus" + "github.com/zebrunner/esg/config" + "github.com/zebrunner/esg/environment" + "github.com/zebrunner/esg/service" + "github.com/zebrunner/esg/task-definitions-service/definitions" + "github.com/zebrunner/esg/utils" + "google.golang.org/grpc" +) + +const ( + listen = ":5555" +) + +type ServiceServerImpl struct { + definitions.ServiceServer +} + +func (ServiceServerImpl) GetTaskDefinitionId(_ context.Context, configuration *definitions.Configuration) (*definitions.Definition, error) { + var env *environment.ExecutionEnvironment + err := json.Unmarshal(configuration.Configuration, env) + if err != nil { + log.WithError(err).Error("Unable to unmarshal environment configuration object") + return nil, err + } + + taskDefinition, err := service.CreateTaskDefinition( + env.ContainerDefinitions(), + env.Volume(), + env.TaskDefinitionFamily, + env.TaskRoleArn, + ) + if err != nil { + log.WithError(err).Error("Unable to create task definition") + return nil, err + } + return &definitions.Definition{Revision: *taskDefinition.Revision}, nil +} + +func main() { + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + + flag.Parse() + log.SetLevel(config.Conf.ParseLogLevel()) + + defer func() { + config.CloseConnections() + }() + + awsSession, err := service.InitAws() + if err != nil { + utils.ExitWithError(err, "Failed to init aws session", log.NewEntry(log.StandardLogger())) + } + service.AwsSess = awsSession + + if err = config.InitDBConnection(config.Conf.DbConnectionString); err != nil { + utils.ExitWithError(err, "Failed to init DB client", log.NewEntry(log.StandardLogger())) + } + + if err = config.InitRedisClusterConnection(); err != nil { + utils.ExitWithError(err, "Failed to init redis connection", log.NewEntry(log.StandardLogger())) + } + + listener, err := net.Listen("tcp", listen) + if err != nil { + log.WithError(err).Fatalf("failed to listen tcp port %s", listen) + } + + server := grpc.NewServer() + definitions.RegisterServiceServer(server, &ServiceServerImpl{}) + + go func() { + if err := server.Serve(listener); err != nil { + log.WithError(err).Fatal("Failed to start task-definitions server") + } + }() + log.Info("Service started") + <-quit + + log.Info("Shutdown task-definitions ...") + server.GracefulStop() + log.Info("task-definitions exited") +} diff --git a/task-definitions-service/proto.sh b/task-definitions-service/proto.sh new file mode 100644 index 0000000..62d0920 --- /dev/null +++ b/task-definitions-service/proto.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +export GO_PATH=~/go +export PATH="$PATH:$(go env GOPATH)/bin" + +protoc --go_out=./definitions --go_opt=paths=source_relative \ + --go-grpc_out=./definitions --go-grpc_opt=paths=source_relative \ + definitions.proto From a12ef44435cf666ca4dc54435e2f7bae02d892ec Mon Sep 17 00:00:00 2001 From: Andrei Kamarouski Date: Sat, 14 Sep 2024 17:03:22 +0200 Subject: [PATCH 03/12] reuse GetTaskDefinitionId for starter.go --- starter/starter.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/starter/starter.go b/starter/starter.go index 8d8d4fe..63676c1 100644 --- a/starter/starter.go +++ b/starter/starter.go @@ -2,6 +2,7 @@ package starter import ( "context" + "encoding/json" "fmt" "net/http" "strings" @@ -20,6 +21,7 @@ import ( envtype "github.com/zebrunner/esg/environment/envType" "github.com/zebrunner/esg/selenium" "github.com/zebrunner/esg/service" + "github.com/zebrunner/esg/task-definitions-service/definitions" "github.com/zebrunner/esg/utils" "github.com/zebrunner/esg/zebrunner" ) @@ -250,8 +252,17 @@ func (starter genericStarter) StartService(startupTime context.Context) (map[str // override request context, as after response is sent, request context is canceled starter.basis.Request = starter.basis.Request.WithContext(genericCtx) go func() { + packedEnvironment, err := json.Marshal(starter.basis.Env) + if err != nil { + log.WithError(err).Error("Failed to create task definition: could not marshal environment") + zebrunner.AbortLaunch(starter.basis.MapperEntity.RouterUUID, starter.basis.MapperEntity.Workspace, + starter.basis.Env.Capabilities.LaunchUUID.ToPrimitive(), fmt.Sprintf("failed to create task defenition for generic: %v", err.Error())) + + genericCtxCancel() + return + } // create new task definition for generic task - taskDefinition, err := service.CreateTaskDefinition(starter.basis.Env.ContainerDefinitions(), starter.basis.Env.Volume(), starter.basis.Env.TaskDefinitionFamily, starter.basis.Env.TaskRoleArn) + taskDefinitionRevision, err := definitions.GetClient().GetTaskDefinitionId(context.Background(), &definitions.Configuration{Configuration: packedEnvironment}) // abort launch if failed to create new task definition if err != nil { log.WithError(err).Error("Failed to create task definition") @@ -262,7 +273,7 @@ func (starter genericStarter) StartService(startupTime context.Context) (map[str return } // set revision of newly created task definition - starter.basis.Env.TaskDefinitionFamily = fmt.Sprintf("%s:%v", starter.basis.Env.TaskDefinitionFamily, *taskDefinition.Revision) + starter.basis.Env.TaskDefinitionFamily = fmt.Sprintf("%s:%v", starter.basis.Env.TaskDefinitionFamily, taskDefinitionRevision.Revision) _, startErr := basicStarter(starter).StartService(startupTime) From 8b003cba5246e733d4174154b31c3d22adf7d347 Mon Sep 17 00:00:00 2001 From: Andrei Kamarouski Date: Sat, 14 Sep 2024 17:04:19 +0200 Subject: [PATCH 04/12] fix gin issue go.mod --- go.mod | 19 ++++++++++++------- go.sum | 22 ++++++++++++++++++++++ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index fc907a3..23f9644 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.22.3 require ( github.com/aerokube/util v1.0.1 github.com/aws/aws-sdk-go v1.51.5 - github.com/gin-gonic/gin v1.9.1 + github.com/gin-gonic/gin v1.10.0 github.com/google/uuid v1.6.0 github.com/jackc/pgtype v1.14.2 github.com/jackc/pgx/v4 v4.18.3 @@ -22,10 +22,15 @@ require ( require github.com/davegardnerisme/deephash v0.0.0-20210406090112-6d072427d830 -require google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect +require ( + github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect +) require ( - github.com/bytedance/sonic v1.9.1 // indirect + github.com/bytedance/sonic v1.11.6 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect @@ -33,7 +38,7 @@ require ( github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.19.0 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgconn v1.14.3 // indirect @@ -43,15 +48,15 @@ require ( github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pelletier/go-toml/v2 v2.2.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect - golang.org/x/arch v0.3.0 // indirect + golang.org/x/arch v0.8.0 // indirect golang.org/x/text v0.16.0 // indirect google.golang.org/grpc v1.66.2 google.golang.org/protobuf v1.34.1 // indirect diff --git a/go.sum b/go.sum index 5348673..03f794a 100644 --- a/go.sum +++ b/go.sum @@ -11,12 +11,20 @@ github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -35,6 +43,8 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -44,6 +54,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4= github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -115,6 +127,9 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -146,6 +161,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pelletier/go-toml/v2 v2.2.0 h1:QLgLl2yMN7N+ruc31VynXs1vhMZa7CeHHejIeBAsoHo= github.com/pelletier/go-toml/v2 v2.2.0/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -202,6 +219,8 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -213,6 +232,7 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -241,6 +261,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -292,4 +313,5 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= From 7a1cedbc2fb9b44963cf20d8d6dba88eeae4a5bf Mon Sep 17 00:00:00 2001 From: Andrei Kamarouski Date: Mon, 16 Sep 2024 13:56:10 +0200 Subject: [PATCH 05/12] add method for getting revision by hash --- environment/environment.go | 10 +- starter/starter.go | 4 +- task-definitions-service/cache/cache.go | 18 +++ task-definitions-service/definitions.proto | 16 ++- .../definitions/definitions.pb.go | 130 ++++++++++++++---- .../definitions/definitions_grpc.pb.go | 72 ++++++++-- task-definitions-service/main.go | 20 ++- 7 files changed, 214 insertions(+), 56 deletions(-) create mode 100644 task-definitions-service/cache/cache.go diff --git a/environment/environment.go b/environment/environment.go index 1cae7e2..f92a43b 100644 --- a/environment/environment.go +++ b/environment/environment.go @@ -1,6 +1,7 @@ package environment import ( + "context" "fmt" "strings" @@ -8,12 +9,12 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ecs" - "github.com/zebrunner/esg/cachemaps/definitionmap" "github.com/zebrunner/esg/cachemaps/resourcesToAllocate" "github.com/zebrunner/esg/capabilities" "github.com/zebrunner/esg/config" envtype "github.com/zebrunner/esg/environment/envType" "github.com/zebrunner/esg/environment/network" + "github.com/zebrunner/esg/task-definitions-service/definitions" "github.com/zebrunner/esg/utils" ) @@ -285,10 +286,9 @@ func (env *ExecutionEnvironment) GetFamilyRevision() (string, error) { return env.TaskDefinitionFamily, nil } - revision, found := definitionmap.FindRevision(env.HashOvverideDefinition()) - if !found { + revision, err := definitions.GetClient().GetTaskDefinitionRevisionByHash(context.Background(), &definitions.Hash{Value: env.HashOvverideDefinition()}) + if err != nil { return "", fmt.Errorf("revision not found for '%s'", env.TaskDefinitionFamily) } - - return fmt.Sprint(env.TaskDefinitionFamily, ":", revision), nil + return fmt.Sprint(env.TaskDefinitionFamily, ":", revision.Value), nil } diff --git a/starter/starter.go b/starter/starter.go index 63676c1..acb5357 100644 --- a/starter/starter.go +++ b/starter/starter.go @@ -262,7 +262,7 @@ func (starter genericStarter) StartService(startupTime context.Context) (map[str return } // create new task definition for generic task - taskDefinitionRevision, err := definitions.GetClient().GetTaskDefinitionId(context.Background(), &definitions.Configuration{Configuration: packedEnvironment}) + taskDefinitionRevision, err := definitions.GetClient().GetTaskDefinitionRevision(context.Background(), &definitions.Configuration{Configuration: packedEnvironment}) // abort launch if failed to create new task definition if err != nil { log.WithError(err).Error("Failed to create task definition") @@ -273,7 +273,7 @@ func (starter genericStarter) StartService(startupTime context.Context) (map[str return } // set revision of newly created task definition - starter.basis.Env.TaskDefinitionFamily = fmt.Sprintf("%s:%v", starter.basis.Env.TaskDefinitionFamily, taskDefinitionRevision.Revision) + starter.basis.Env.TaskDefinitionFamily = fmt.Sprintf("%s:%v", starter.basis.Env.TaskDefinitionFamily, taskDefinitionRevision.Value) _, startErr := basicStarter(starter).StartService(startupTime) diff --git a/task-definitions-service/cache/cache.go b/task-definitions-service/cache/cache.go new file mode 100644 index 0000000..663ec37 --- /dev/null +++ b/task-definitions-service/cache/cache.go @@ -0,0 +1,18 @@ +package cache + +import ( + "sync" +) + +var ( + cache = &RevisionsCache{} +) + +type RevisionsCache struct { + sync.RWMutex + Cache map[string]int64 +} + +func GetCache() *RevisionsCache { + return cache +} diff --git a/task-definitions-service/definitions.proto b/task-definitions-service/definitions.proto index 4129e3a..5b2a371 100644 --- a/task-definitions-service/definitions.proto +++ b/task-definitions-service/definitions.proto @@ -4,7 +4,13 @@ package definitions; option go_package = "./definitions"; service Service { - rpc getTaskDefinitionId(Configuration) returns (Definition); + + // todo add cache support for this method + // create definition according to the environment.Environment and return revision id + // for generic tasks + rpc getTaskDefinitionRevision(Configuration) returns (Revision); + + rpc getTaskDefinitionRevisionByHash(Hash) returns (Revision); } message Configuration { @@ -13,6 +19,10 @@ message Configuration { bytes configuration = 1; } -message Definition { - int64 revision = 1; +message Revision { + int64 value = 1; +} + +message Hash { + string value = 1; } diff --git a/task-definitions-service/definitions/definitions.pb.go b/task-definitions-service/definitions/definitions.pb.go index 33f81a6..3cde68b 100644 --- a/task-definitions-service/definitions/definitions.pb.go +++ b/task-definitions-service/definitions/definitions.pb.go @@ -25,6 +25,8 @@ type Configuration struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + // configuration should be json-marshalled environment.Environment object + // todo think about another options how send no-proto message Configuration []byte `protobuf:"bytes,1,opt,name=configuration,proto3" json:"configuration,omitempty"` } @@ -67,16 +69,16 @@ func (x *Configuration) GetConfiguration() []byte { return nil } -type Definition struct { +type Revision struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Revision int64 `protobuf:"varint,1,opt,name=revision,proto3" json:"revision,omitempty"` + Value int64 `protobuf:"varint,1,opt,name=value,proto3" json:"value,omitempty"` } -func (x *Definition) Reset() { - *x = Definition{} +func (x *Revision) Reset() { + *x = Revision{} if protoimpl.UnsafeEnabled { mi := &file_definitions_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -84,13 +86,13 @@ func (x *Definition) Reset() { } } -func (x *Definition) String() string { +func (x *Revision) String() string { return protoimpl.X.MessageStringOf(x) } -func (*Definition) ProtoMessage() {} +func (*Revision) ProtoMessage() {} -func (x *Definition) ProtoReflect() protoreflect.Message { +func (x *Revision) ProtoReflect() protoreflect.Message { mi := &file_definitions_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -102,18 +104,65 @@ func (x *Definition) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use Definition.ProtoReflect.Descriptor instead. -func (*Definition) Descriptor() ([]byte, []int) { +// Deprecated: Use Revision.ProtoReflect.Descriptor instead. +func (*Revision) Descriptor() ([]byte, []int) { return file_definitions_proto_rawDescGZIP(), []int{1} } -func (x *Definition) GetRevision() int64 { +func (x *Revision) GetValue() int64 { if x != nil { - return x.Revision + return x.Value } return 0 } +type Hash struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Value string `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` +} + +func (x *Hash) Reset() { + *x = Hash{} + if protoimpl.UnsafeEnabled { + mi := &file_definitions_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Hash) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Hash) ProtoMessage() {} + +func (x *Hash) ProtoReflect() protoreflect.Message { + mi := &file_definitions_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Hash.ProtoReflect.Descriptor instead. +func (*Hash) Descriptor() ([]byte, []int) { + return file_definitions_proto_rawDescGZIP(), []int{2} +} + +func (x *Hash) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + var File_definitions_proto protoreflect.FileDescriptor var file_definitions_proto_rawDesc = []byte{ @@ -122,17 +171,23 @@ var file_definitions_proto_rawDesc = []byte{ 0x22, 0x35, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x28, 0x0a, 0x0a, 0x44, 0x65, 0x66, 0x69, 0x6e, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, - 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, - 0x6e, 0x32, 0x55, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4a, 0x0a, 0x13, - 0x67, 0x65, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x49, 0x64, 0x12, 0x1a, 0x2e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, - 0x17, 0x2e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x44, 0x65, - 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x0f, 0x5a, 0x0d, 0x2e, 0x2f, 0x64, 0x65, - 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x20, 0x0a, 0x08, 0x52, 0x65, 0x76, 0x69, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x1c, 0x0a, 0x04, 0x48, 0x61, 0x73, + 0x68, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x32, 0xa6, 0x01, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x12, 0x4e, 0x0a, 0x19, 0x67, 0x65, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x44, 0x65, + 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, + 0x12, 0x1a, 0x2e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x15, 0x2e, 0x64, + 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x52, 0x65, 0x76, 0x69, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x4b, 0x0a, 0x1f, 0x67, 0x65, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x44, 0x65, + 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, + 0x42, 0x79, 0x48, 0x61, 0x73, 0x68, 0x12, 0x11, 0x2e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x48, 0x61, 0x73, 0x68, 0x1a, 0x15, 0x2e, 0x64, 0x65, 0x66, 0x69, + 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x52, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, + 0x42, 0x0f, 0x5a, 0x0d, 0x2e, 0x2f, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -147,16 +202,19 @@ func file_definitions_proto_rawDescGZIP() []byte { return file_definitions_proto_rawDescData } -var file_definitions_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_definitions_proto_msgTypes = make([]protoimpl.MessageInfo, 3) var file_definitions_proto_goTypes = []any{ (*Configuration)(nil), // 0: definitions.Configuration - (*Definition)(nil), // 1: definitions.Definition + (*Revision)(nil), // 1: definitions.Revision + (*Hash)(nil), // 2: definitions.Hash } var file_definitions_proto_depIdxs = []int32{ - 0, // 0: definitions.Service.getTaskDefinitionId:input_type -> definitions.Configuration - 1, // 1: definitions.Service.getTaskDefinitionId:output_type -> definitions.Definition - 1, // [1:2] is the sub-list for method output_type - 0, // [0:1] is the sub-list for method input_type + 0, // 0: definitions.Service.getTaskDefinitionRevision:input_type -> definitions.Configuration + 2, // 1: definitions.Service.getTaskDefinitionRevisionByHash:input_type -> definitions.Hash + 1, // 2: definitions.Service.getTaskDefinitionRevision:output_type -> definitions.Revision + 1, // 3: definitions.Service.getTaskDefinitionRevisionByHash:output_type -> definitions.Revision + 2, // [2:4] is the sub-list for method output_type + 0, // [0:2] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name 0, // [0:0] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name @@ -181,7 +239,19 @@ func file_definitions_proto_init() { } } file_definitions_proto_msgTypes[1].Exporter = func(v any, i int) any { - switch v := v.(*Definition); i { + switch v := v.(*Revision); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_definitions_proto_msgTypes[2].Exporter = func(v any, i int) any { + switch v := v.(*Hash); i { case 0: return &v.state case 1: @@ -199,7 +269,7 @@ func file_definitions_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_definitions_proto_rawDesc, NumEnums: 0, - NumMessages: 2, + NumMessages: 3, NumExtensions: 0, NumServices: 1, }, diff --git a/task-definitions-service/definitions/definitions_grpc.pb.go b/task-definitions-service/definitions/definitions_grpc.pb.go index ee708e2..22e268a 100644 --- a/task-definitions-service/definitions/definitions_grpc.pb.go +++ b/task-definitions-service/definitions/definitions_grpc.pb.go @@ -19,14 +19,19 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - Service_GetTaskDefinitionId_FullMethodName = "/definitions.Service/getTaskDefinitionId" + Service_GetTaskDefinitionRevision_FullMethodName = "/definitions.Service/getTaskDefinitionRevision" + Service_GetTaskDefinitionRevisionByHash_FullMethodName = "/definitions.Service/getTaskDefinitionRevisionByHash" ) // ServiceClient is the client API for Service service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type ServiceClient interface { - GetTaskDefinitionId(ctx context.Context, in *Configuration, opts ...grpc.CallOption) (*Definition, error) + // todo add cache support for this method + // create definition according to the environment.Environment and return revision id + // for generic tasks + GetTaskDefinitionRevision(ctx context.Context, in *Configuration, opts ...grpc.CallOption) (*Revision, error) + GetTaskDefinitionRevisionByHash(ctx context.Context, in *Hash, opts ...grpc.CallOption) (*Revision, error) } type serviceClient struct { @@ -37,10 +42,20 @@ func NewServiceClient(cc grpc.ClientConnInterface) ServiceClient { return &serviceClient{cc} } -func (c *serviceClient) GetTaskDefinitionId(ctx context.Context, in *Configuration, opts ...grpc.CallOption) (*Definition, error) { +func (c *serviceClient) GetTaskDefinitionRevision(ctx context.Context, in *Configuration, opts ...grpc.CallOption) (*Revision, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - out := new(Definition) - err := c.cc.Invoke(ctx, Service_GetTaskDefinitionId_FullMethodName, in, out, cOpts...) + out := new(Revision) + err := c.cc.Invoke(ctx, Service_GetTaskDefinitionRevision_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *serviceClient) GetTaskDefinitionRevisionByHash(ctx context.Context, in *Hash, opts ...grpc.CallOption) (*Revision, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Revision) + err := c.cc.Invoke(ctx, Service_GetTaskDefinitionRevisionByHash_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -51,7 +66,11 @@ func (c *serviceClient) GetTaskDefinitionId(ctx context.Context, in *Configurati // All implementations must embed UnimplementedServiceServer // for forward compatibility. type ServiceServer interface { - GetTaskDefinitionId(context.Context, *Configuration) (*Definition, error) + // todo add cache support for this method + // create definition according to the environment.Environment and return revision id + // for generic tasks + GetTaskDefinitionRevision(context.Context, *Configuration) (*Revision, error) + GetTaskDefinitionRevisionByHash(context.Context, *Hash) (*Revision, error) mustEmbedUnimplementedServiceServer() } @@ -62,8 +81,11 @@ type ServiceServer interface { // pointer dereference when methods are called. type UnimplementedServiceServer struct{} -func (UnimplementedServiceServer) GetTaskDefinitionId(context.Context, *Configuration) (*Definition, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetTaskDefinitionId not implemented") +func (UnimplementedServiceServer) GetTaskDefinitionRevision(context.Context, *Configuration) (*Revision, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetTaskDefinitionRevision not implemented") +} +func (UnimplementedServiceServer) GetTaskDefinitionRevisionByHash(context.Context, *Hash) (*Revision, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetTaskDefinitionRevisionByHash not implemented") } func (UnimplementedServiceServer) mustEmbedUnimplementedServiceServer() {} func (UnimplementedServiceServer) testEmbeddedByValue() {} @@ -86,20 +108,38 @@ func RegisterServiceServer(s grpc.ServiceRegistrar, srv ServiceServer) { s.RegisterService(&Service_ServiceDesc, srv) } -func _Service_GetTaskDefinitionId_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { +func _Service_GetTaskDefinitionRevision_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(Configuration) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(ServiceServer).GetTaskDefinitionId(ctx, in) + return srv.(ServiceServer).GetTaskDefinitionRevision(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: Service_GetTaskDefinitionId_FullMethodName, + FullMethod: Service_GetTaskDefinitionRevision_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ServiceServer).GetTaskDefinitionId(ctx, req.(*Configuration)) + return srv.(ServiceServer).GetTaskDefinitionRevision(ctx, req.(*Configuration)) + } + return interceptor(ctx, in, info, handler) +} + +func _Service_GetTaskDefinitionRevisionByHash_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Hash) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ServiceServer).GetTaskDefinitionRevisionByHash(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Service_GetTaskDefinitionRevisionByHash_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ServiceServer).GetTaskDefinitionRevisionByHash(ctx, req.(*Hash)) } return interceptor(ctx, in, info, handler) } @@ -112,8 +152,12 @@ var Service_ServiceDesc = grpc.ServiceDesc{ HandlerType: (*ServiceServer)(nil), Methods: []grpc.MethodDesc{ { - MethodName: "getTaskDefinitionId", - Handler: _Service_GetTaskDefinitionId_Handler, + MethodName: "getTaskDefinitionRevision", + Handler: _Service_GetTaskDefinitionRevision_Handler, + }, + { + MethodName: "getTaskDefinitionRevisionByHash", + Handler: _Service_GetTaskDefinitionRevisionByHash_Handler, }, }, Streams: []grpc.StreamDesc{}, diff --git a/task-definitions-service/main.go b/task-definitions-service/main.go index 80fab6c..47835de 100644 --- a/task-definitions-service/main.go +++ b/task-definitions-service/main.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "flag" + "fmt" "net" "os" "os/signal" @@ -13,6 +14,7 @@ import ( "github.com/zebrunner/esg/config" "github.com/zebrunner/esg/environment" "github.com/zebrunner/esg/service" + "github.com/zebrunner/esg/task-definitions-service/cache" "github.com/zebrunner/esg/task-definitions-service/definitions" "github.com/zebrunner/esg/utils" "google.golang.org/grpc" @@ -26,7 +28,7 @@ type ServiceServerImpl struct { definitions.ServiceServer } -func (ServiceServerImpl) GetTaskDefinitionId(_ context.Context, configuration *definitions.Configuration) (*definitions.Definition, error) { +func (ServiceServerImpl) GetTaskDefinitionRevision(_ context.Context, configuration *definitions.Configuration) (*definitions.Revision, error) { var env *environment.ExecutionEnvironment err := json.Unmarshal(configuration.Configuration, env) if err != nil { @@ -44,7 +46,21 @@ func (ServiceServerImpl) GetTaskDefinitionId(_ context.Context, configuration *d log.WithError(err).Error("Unable to create task definition") return nil, err } - return &definitions.Definition{Revision: *taskDefinition.Revision}, nil + return &definitions.Revision{Value: *taskDefinition.Revision}, nil +} + +func (ServiceServerImpl) GetTaskDefinitionRevisionByHash(_ context.Context, hash *definitions.Hash) (*definitions.Revision, error) { + cache := cache.GetCache() + if cache.Cache == nil { + return nil, fmt.Errorf("cache is not initialized yet") + } + cache.RLock() + revision, ok := cache.Cache[hash.Value] + cache.RUnlock() + if !ok { + return nil, fmt.Errorf("there are no revision with such hash") + } + return &definitions.Revision{Value: revision}, nil } func main() { From b98d15f0dbc248e4dec0afc68d8deafaefc297fd Mon Sep 17 00:00:00 2001 From: Andrei Kamarouski Date: Mon, 16 Sep 2024 14:32:04 +0200 Subject: [PATCH 06/12] code cleanup --- task-definitions-service/definitions.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/task-definitions-service/definitions.proto b/task-definitions-service/definitions.proto index 5b2a371..6a677df 100644 --- a/task-definitions-service/definitions.proto +++ b/task-definitions-service/definitions.proto @@ -10,6 +10,7 @@ service Service { // for generic tasks rpc getTaskDefinitionRevision(Configuration) returns (Revision); + // get revision by hash (without explicit reference to redis) rpc getTaskDefinitionRevisionByHash(Hash) returns (Revision); } From ace12bb95db839ea8598d75899f35473d8b615b4 Mon Sep 17 00:00:00 2001 From: Andrei Kamarouski Date: Mon, 16 Sep 2024 14:34:18 +0200 Subject: [PATCH 07/12] code cleanup --- cachemaps/definitionmap/definitionmap.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/cachemaps/definitionmap/definitionmap.go b/cachemaps/definitionmap/definitionmap.go index b7ebbdf..b905e03 100644 --- a/cachemaps/definitionmap/definitionmap.go +++ b/cachemaps/definitionmap/definitionmap.go @@ -68,19 +68,6 @@ func getDefinitions() (map[string]int64, error) { return hashRevisionMap, nil } -// Find revision in definitionsMap (without redis usage). -func FindRevision(hash string) (int64, bool) { - if definitionsMap == nil { - return -1, false - } - - mutex.RLock() - revision, ok := definitionsMap[hash] - mutex.RUnlock() - - return revision, ok -} - // every `interval` in minutes we update local definitionsMap syncing it with redis cache to minimize redis calls at run-time func ActualizeDefinitionsMap(interval time.Duration) { for { From 9967270c804ff943824d37c92681a7256057cc2d Mon Sep 17 00:00:00 2001 From: Andrei Kamarouski Date: Tue, 17 Sep 2024 10:27:40 +0200 Subject: [PATCH 08/12] get images from task definitions --- handlers/definitions.go | 38 ++-- task-definitions-service/definitions.proto | 15 ++ .../definitions/definitions.pb.go | 174 +++++++++++++++--- .../definitions/definitions_grpc.pb.go | 49 ++++- task-definitions-service/main.go | 33 ++++ 5 files changed, 261 insertions(+), 48 deletions(-) diff --git a/handlers/definitions.go b/handlers/definitions.go index 453a9f6..886dee5 100644 --- a/handlers/definitions.go +++ b/handlers/definitions.go @@ -1,13 +1,15 @@ package handlers import ( + "context" + "io" "net/http" + "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" "github.com/zebrunner/esg/config" - "github.com/zebrunner/esg/definitions" - envtype "github.com/zebrunner/esg/environment/envType" "github.com/zebrunner/esg/images" + "github.com/zebrunner/esg/task-definitions-service/definitions" ) var ( @@ -30,33 +32,27 @@ type refreshDefinitionsModel struct { } func GetImages(c *gin.Context) { - images, err := images.ListImages(config.Conf.ImageRepositories, config.Conf.ExcludeBrowsers) + stream, err := definitions.GetClient().GetImages(context.Background(), nil) if err != nil { log.WithError(err).Error("Failed to list images") c.Status(http.StatusInternalServerError) return } - imagesDataResponse := make([]imageDataModel, 0, len(images)) - for _, image := range images { - imgData := imageDataModel{ - Name: image.BrowserName, - Version: image.Tag, - Platform: image.Platform.String(), - } - - if image.Platform == envtype.ANDROID { - imgData.BrowserName = "chrome" - imgData.BrowserVersion = "107.0" + imagesDataResponse := make([]definitions.Image, 0, 0) +CYCLE: + for true { + image, err := stream.Recv() + if err != nil { + if err == io.EOF { + break CYCLE + } + log.WithError(err).Error("Failed to receive image") + c.Status(http.StatusInternalServerError) + return } - - if image.Platform == envtype.CYPRESS { - imgData.ImageUrl = image.GetUrl() - } - - imagesDataResponse = append(imagesDataResponse, imgData) + imagesDataResponse = append(imagesDataResponse, *image) } - c.JSON(http.StatusOK, imagesDataResponse) } diff --git a/task-definitions-service/definitions.proto b/task-definitions-service/definitions.proto index 6a677df..0e8c4da 100644 --- a/task-definitions-service/definitions.proto +++ b/task-definitions-service/definitions.proto @@ -3,6 +3,8 @@ package definitions; option go_package = "./definitions"; +import "google/protobuf/empty.proto"; + service Service { // todo add cache support for this method @@ -12,6 +14,9 @@ service Service { // get revision by hash (without explicit reference to redis) rpc getTaskDefinitionRevisionByHash(Hash) returns (Revision); + + // get images list from ecr (public and private) + rpc getImages(google.protobuf.Empty) returns (stream Image); } message Configuration { @@ -27,3 +32,13 @@ message Revision { message Hash { string value = 1; } + +message Image { + string name = 1; + string version = 2; + string platform =3; + string browserName = 4; + string browserVersion = 5; + // TODO: investigate possibility of 'ImageUrl' field removal + string imageUrl = 6 [json_name="image"]; +} diff --git a/task-definitions-service/definitions/definitions.pb.go b/task-definitions-service/definitions/definitions.pb.go index 3cde68b..c30e3e9 100644 --- a/task-definitions-service/definitions/definitions.pb.go +++ b/task-definitions-service/definitions/definitions.pb.go @@ -9,6 +9,7 @@ package definitions import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + emptypb "google.golang.org/protobuf/types/known/emptypb" reflect "reflect" sync "sync" ) @@ -163,31 +164,136 @@ func (x *Hash) GetValue() string { return "" } +type Image struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` + Platform string `protobuf:"bytes,3,opt,name=platform,proto3" json:"platform,omitempty"` + BrowserName string `protobuf:"bytes,4,opt,name=browserName,proto3" json:"browserName,omitempty"` + BrowserVersion string `protobuf:"bytes,5,opt,name=browserVersion,proto3" json:"browserVersion,omitempty"` + // TODO: investigate possibility of 'ImageUrl' field removal + ImageUrl string `protobuf:"bytes,6,opt,name=imageUrl,json=image,proto3" json:"imageUrl,omitempty"` +} + +func (x *Image) Reset() { + *x = Image{} + if protoimpl.UnsafeEnabled { + mi := &file_definitions_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Image) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Image) ProtoMessage() {} + +func (x *Image) ProtoReflect() protoreflect.Message { + mi := &file_definitions_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Image.ProtoReflect.Descriptor instead. +func (*Image) Descriptor() ([]byte, []int) { + return file_definitions_proto_rawDescGZIP(), []int{3} +} + +func (x *Image) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Image) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +func (x *Image) GetPlatform() string { + if x != nil { + return x.Platform + } + return "" +} + +func (x *Image) GetBrowserName() string { + if x != nil { + return x.BrowserName + } + return "" +} + +func (x *Image) GetBrowserVersion() string { + if x != nil { + return x.BrowserVersion + } + return "" +} + +func (x *Image) GetImageUrl() string { + if x != nil { + return x.ImageUrl + } + return "" +} + var File_definitions_proto protoreflect.FileDescriptor var file_definitions_proto_rawDesc = []byte{ 0x0a, 0x11, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0b, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x22, 0x35, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x24, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x20, 0x0a, 0x08, 0x52, 0x65, 0x76, 0x69, 0x73, - 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x1c, 0x0a, 0x04, 0x48, 0x61, 0x73, - 0x68, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x32, 0xa6, 0x01, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x12, 0x4e, 0x0a, 0x19, 0x67, 0x65, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x44, 0x65, - 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x1a, 0x2e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x15, 0x2e, 0x64, - 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x52, 0x65, 0x76, 0x69, 0x73, - 0x69, 0x6f, 0x6e, 0x12, 0x4b, 0x0a, 0x1f, 0x67, 0x65, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x44, 0x65, - 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, - 0x42, 0x79, 0x48, 0x61, 0x73, 0x68, 0x12, 0x11, 0x2e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x48, 0x61, 0x73, 0x68, 0x1a, 0x15, 0x2e, 0x64, 0x65, 0x66, 0x69, - 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x52, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, - 0x42, 0x0f, 0x5a, 0x0d, 0x2e, 0x2f, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x35, 0x0a, + 0x0d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, + 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x20, 0x0a, 0x08, 0x52, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x1c, 0x0a, 0x04, 0x48, 0x61, 0x73, 0x68, 0x12, 0x14, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x22, 0xb4, 0x01, 0x0a, 0x05, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1a, 0x0a, 0x08, + 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x20, 0x0a, 0x0b, 0x62, 0x72, 0x6f, 0x77, + 0x73, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x62, + 0x72, 0x6f, 0x77, 0x73, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x26, 0x0a, 0x0e, 0x62, 0x72, + 0x6f, 0x77, 0x73, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0e, 0x62, 0x72, 0x6f, 0x77, 0x73, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x17, 0x0a, 0x08, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x55, 0x72, 0x6c, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x32, 0xe1, 0x01, 0x0a, 0x07, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4e, 0x0a, 0x19, 0x67, 0x65, 0x74, 0x54, 0x61, + 0x73, 0x6b, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x76, 0x69, + 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1a, 0x2e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x1a, 0x15, 0x2e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x52, + 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x4b, 0x0a, 0x1f, 0x67, 0x65, 0x74, 0x54, 0x61, + 0x73, 0x6b, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x76, 0x69, + 0x73, 0x69, 0x6f, 0x6e, 0x42, 0x79, 0x48, 0x61, 0x73, 0x68, 0x12, 0x11, 0x2e, 0x64, 0x65, 0x66, + 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x48, 0x61, 0x73, 0x68, 0x1a, 0x15, 0x2e, + 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x52, 0x65, 0x76, 0x69, + 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x39, 0x0a, 0x09, 0x67, 0x65, 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, + 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x12, 0x2e, 0x64, 0x65, 0x66, 0x69, + 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x30, 0x01, 0x42, + 0x0f, 0x5a, 0x0d, 0x2e, 0x2f, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -202,19 +308,23 @@ func file_definitions_proto_rawDescGZIP() []byte { return file_definitions_proto_rawDescData } -var file_definitions_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_definitions_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_definitions_proto_goTypes = []any{ (*Configuration)(nil), // 0: definitions.Configuration (*Revision)(nil), // 1: definitions.Revision (*Hash)(nil), // 2: definitions.Hash + (*Image)(nil), // 3: definitions.Image + (*emptypb.Empty)(nil), // 4: google.protobuf.Empty } var file_definitions_proto_depIdxs = []int32{ 0, // 0: definitions.Service.getTaskDefinitionRevision:input_type -> definitions.Configuration 2, // 1: definitions.Service.getTaskDefinitionRevisionByHash:input_type -> definitions.Hash - 1, // 2: definitions.Service.getTaskDefinitionRevision:output_type -> definitions.Revision - 1, // 3: definitions.Service.getTaskDefinitionRevisionByHash:output_type -> definitions.Revision - 2, // [2:4] is the sub-list for method output_type - 0, // [0:2] is the sub-list for method input_type + 4, // 2: definitions.Service.getImages:input_type -> google.protobuf.Empty + 1, // 3: definitions.Service.getTaskDefinitionRevision:output_type -> definitions.Revision + 1, // 4: definitions.Service.getTaskDefinitionRevisionByHash:output_type -> definitions.Revision + 3, // 5: definitions.Service.getImages:output_type -> definitions.Image + 3, // [3:6] is the sub-list for method output_type + 0, // [0:3] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name 0, // [0:0] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name @@ -262,6 +372,18 @@ func file_definitions_proto_init() { return nil } } + file_definitions_proto_msgTypes[3].Exporter = func(v any, i int) any { + switch v := v.(*Image); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -269,7 +391,7 @@ func file_definitions_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_definitions_proto_rawDesc, NumEnums: 0, - NumMessages: 3, + NumMessages: 4, NumExtensions: 0, NumServices: 1, }, diff --git a/task-definitions-service/definitions/definitions_grpc.pb.go b/task-definitions-service/definitions/definitions_grpc.pb.go index 22e268a..8db1a85 100644 --- a/task-definitions-service/definitions/definitions_grpc.pb.go +++ b/task-definitions-service/definitions/definitions_grpc.pb.go @@ -11,6 +11,7 @@ import ( grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" + emptypb "google.golang.org/protobuf/types/known/emptypb" ) // This is a compile-time assertion to ensure that this generated file @@ -21,6 +22,7 @@ const _ = grpc.SupportPackageIsVersion9 const ( Service_GetTaskDefinitionRevision_FullMethodName = "/definitions.Service/getTaskDefinitionRevision" Service_GetTaskDefinitionRevisionByHash_FullMethodName = "/definitions.Service/getTaskDefinitionRevisionByHash" + Service_GetImages_FullMethodName = "/definitions.Service/getImages" ) // ServiceClient is the client API for Service service. @@ -31,7 +33,10 @@ type ServiceClient interface { // create definition according to the environment.Environment and return revision id // for generic tasks GetTaskDefinitionRevision(ctx context.Context, in *Configuration, opts ...grpc.CallOption) (*Revision, error) + // get revision by hash (without explicit reference to redis) GetTaskDefinitionRevisionByHash(ctx context.Context, in *Hash, opts ...grpc.CallOption) (*Revision, error) + // get images list from ecr (public and private) + GetImages(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Image], error) } type serviceClient struct { @@ -62,6 +67,25 @@ func (c *serviceClient) GetTaskDefinitionRevisionByHash(ctx context.Context, in return out, nil } +func (c *serviceClient) GetImages(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Image], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Service_ServiceDesc.Streams[0], Service_GetImages_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[emptypb.Empty, Image]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Service_GetImagesClient = grpc.ServerStreamingClient[Image] + // ServiceServer is the server API for Service service. // All implementations must embed UnimplementedServiceServer // for forward compatibility. @@ -70,7 +94,10 @@ type ServiceServer interface { // create definition according to the environment.Environment and return revision id // for generic tasks GetTaskDefinitionRevision(context.Context, *Configuration) (*Revision, error) + // get revision by hash (without explicit reference to redis) GetTaskDefinitionRevisionByHash(context.Context, *Hash) (*Revision, error) + // get images list from ecr (public and private) + GetImages(*emptypb.Empty, grpc.ServerStreamingServer[Image]) error mustEmbedUnimplementedServiceServer() } @@ -87,6 +114,9 @@ func (UnimplementedServiceServer) GetTaskDefinitionRevision(context.Context, *Co func (UnimplementedServiceServer) GetTaskDefinitionRevisionByHash(context.Context, *Hash) (*Revision, error) { return nil, status.Errorf(codes.Unimplemented, "method GetTaskDefinitionRevisionByHash not implemented") } +func (UnimplementedServiceServer) GetImages(*emptypb.Empty, grpc.ServerStreamingServer[Image]) error { + return status.Errorf(codes.Unimplemented, "method GetImages not implemented") +} func (UnimplementedServiceServer) mustEmbedUnimplementedServiceServer() {} func (UnimplementedServiceServer) testEmbeddedByValue() {} @@ -144,6 +174,17 @@ func _Service_GetTaskDefinitionRevisionByHash_Handler(srv interface{}, ctx conte return interceptor(ctx, in, info, handler) } +func _Service_GetImages_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(emptypb.Empty) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(ServiceServer).GetImages(m, &grpc.GenericServerStream[emptypb.Empty, Image]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Service_GetImagesServer = grpc.ServerStreamingServer[Image] + // Service_ServiceDesc is the grpc.ServiceDesc for Service service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -160,6 +201,12 @@ var Service_ServiceDesc = grpc.ServiceDesc{ Handler: _Service_GetTaskDefinitionRevisionByHash_Handler, }, }, - Streams: []grpc.StreamDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "getImages", + Handler: _Service_GetImages_Handler, + ServerStreams: true, + }, + }, Metadata: "definitions.proto", } diff --git a/task-definitions-service/main.go b/task-definitions-service/main.go index 47835de..6ab55e5 100644 --- a/task-definitions-service/main.go +++ b/task-definitions-service/main.go @@ -5,6 +5,7 @@ import ( "encoding/json" "flag" "fmt" + "io" "net" "os" "os/signal" @@ -13,11 +14,14 @@ import ( log "github.com/sirupsen/logrus" "github.com/zebrunner/esg/config" "github.com/zebrunner/esg/environment" + envtype "github.com/zebrunner/esg/environment/envType" + "github.com/zebrunner/esg/images" "github.com/zebrunner/esg/service" "github.com/zebrunner/esg/task-definitions-service/cache" "github.com/zebrunner/esg/task-definitions-service/definitions" "github.com/zebrunner/esg/utils" "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/emptypb" ) const ( @@ -63,6 +67,35 @@ func (ServiceServerImpl) GetTaskDefinitionRevisionByHash(_ context.Context, hash return &definitions.Revision{Value: revision}, nil } +func (ServiceServerImpl) GetImages(_ *emptypb.Empty, stream grpc.ServerStreamingServer[definitions.Image]) error { + images, err := images.ListImages(config.Conf.ImageRepositories, config.Conf.ExcludeBrowsers) + if err != nil { + log.WithError(err).Error("Failed to list images") + return fmt.Errorf("failed to list images") + } + + for _, image := range images { + imageData := &definitions.Image{ + Name: image.BrowserName, + Version: image.Tag, + Platform: image.Platform.String(), + } + if image.Platform == envtype.ANDROID { + imageData.BrowserName = "chrome" + imageData.BrowserVersion = "107.0" + } + if image.Platform == envtype.CYPRESS { + imageData.ImageUrl = image.GetUrl() + } + + err = stream.Send(imageData) + if err != nil { + return io.EOF + } + } + return io.EOF +} + func main() { quit := make(chan os.Signal, 1) signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) From baa27f2e7802e54705007914515bffcd703acbb1 Mon Sep 17 00:00:00 2001 From: Andrei Kamarouski Date: Tue, 17 Sep 2024 10:42:16 +0200 Subject: [PATCH 09/12] reuse GetImages --- handlers/reporting.go | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/handlers/reporting.go b/handlers/reporting.go index cd0e90b..0b58893 100644 --- a/handlers/reporting.go +++ b/handlers/reporting.go @@ -1,8 +1,9 @@ package handlers import ( - "encoding/json" + "context" "fmt" + "io" "net/http" "os" "runtime" @@ -12,7 +13,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/zebrunner/esg/cachemaps/utilsmap" "github.com/zebrunner/esg/config" - "github.com/zebrunner/esg/definitions" + "github.com/zebrunner/esg/task-definitions-service/definitions" "github.com/gin-gonic/gin" ) @@ -53,28 +54,32 @@ func ClusterStatus(c *gin.Context) { } func ListDrivers(c *gin.Context) { - resBody, err := definitions.ListImages() + stream, err := definitions.GetClient().GetImages(context.Background(), nil) if err != nil { c.Status(http.StatusInternalServerError) log.WithError(err).Error("Failed to list images from task-definitions server") return } - originalImages := make([]imageDataModel, 0) - if err := json.Unmarshal(resBody, &originalImages); err != nil { - log.WithError(err).Error("Failed to unmarshal list of the images from task-definitions server") - return - } - - filteredImages := make([]imageDataModel, 0) - for _, image := range originalImages { + images := make([]definitions.Image, 0, 0) +CYCLE: + for true { + image, err := stream.Recv() + if err != nil { + if err == io.EOF { + break CYCLE + } + log.WithError(err).Error("Failed to receive image") + c.Status(http.StatusInternalServerError) + return + } // -debug images should not be added to the reporting, it should be available silently if strings.Contains(image.Version, "-debug") { continue } - filteredImages = append(filteredImages, image) + images = append(images, *image) } - c.JSON(http.StatusOK, filteredImages) + c.JSON(http.StatusOK, images) } func Welcome(c *gin.Context) { From 6c7496f8c081894e76b7199a9ad79ee80e6447bb Mon Sep 17 00:00:00 2001 From: Andrei Kamarouski Date: Tue, 17 Sep 2024 10:44:08 +0200 Subject: [PATCH 10/12] code cleanup --- cmd/task-definitions/main.go | 1 - definitions/requests.go | 21 --------------------- handlers/definitions.go | 25 ------------------------- 3 files changed, 47 deletions(-) diff --git a/cmd/task-definitions/main.go b/cmd/task-definitions/main.go index 0c9f344..366743c 100644 --- a/cmd/task-definitions/main.go +++ b/cmd/task-definitions/main.go @@ -95,7 +95,6 @@ func CreateRouter() *gin.Engine { r.GET("/", handlers.Ready) r.GET(definitions.IsReadyPath.String(), handlers.IsTaskDefinitionRefreshDone) - r.GET(definitions.GetImagesPath.String(), handlers.GetImages) r.POST(definitions.RefreshDefinitionsPath.String(), handlers.RefreshDefinitions) return r diff --git a/definitions/requests.go b/definitions/requests.go index 4b15168..58e2757 100644 --- a/definitions/requests.go +++ b/definitions/requests.go @@ -2,7 +2,6 @@ package definitions import ( "fmt" - "io" "net/http" "github.com/zebrunner/esg/config" @@ -10,7 +9,6 @@ import ( const ( IsReadyPath ApiPath = "/refresh-complete" - GetImagesPath ApiPath = "/images" RefreshDefinitionsPath ApiPath = "/refresh-definitions" ) @@ -43,22 +41,3 @@ func IsTaskDefinitionRefreshDone() (bool, error) { return false, fmt.Errorf("wrong status code: %v", res.StatusCode) } } - -func ListImages() ([]byte, error) { - req, err := http.NewRequest(http.MethodGet, GetImagesPath.StringUrl(), nil) - if err != nil { - return nil, err - } - - res, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - - resBody, err := io.ReadAll(res.Body) - if err != nil { - return nil, err - } - - return resBody, nil -} diff --git a/handlers/definitions.go b/handlers/definitions.go index 886dee5..b8d0878 100644 --- a/handlers/definitions.go +++ b/handlers/definitions.go @@ -31,31 +31,6 @@ type refreshDefinitionsModel struct { ExcludeBrowsers *string `json:"excludeBrowsers,omitempty"` } -func GetImages(c *gin.Context) { - stream, err := definitions.GetClient().GetImages(context.Background(), nil) - if err != nil { - log.WithError(err).Error("Failed to list images") - c.Status(http.StatusInternalServerError) - return - } - - imagesDataResponse := make([]definitions.Image, 0, 0) -CYCLE: - for true { - image, err := stream.Recv() - if err != nil { - if err == io.EOF { - break CYCLE - } - log.WithError(err).Error("Failed to receive image") - c.Status(http.StatusInternalServerError) - return - } - imagesDataResponse = append(imagesDataResponse, *image) - } - c.JSON(http.StatusOK, imagesDataResponse) -} - func Ready(c *gin.Context) { c.String(http.StatusOK, "ready to accept requests") } From 966dfe3da4b32f99fc976b94c2b65b0d28031f34 Mon Sep 17 00:00:00 2001 From: Andrei Kamarouski Date: Tue, 17 Sep 2024 11:12:08 +0200 Subject: [PATCH 11/12] changes --- task-definitions-service/main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/task-definitions-service/main.go b/task-definitions-service/main.go index 6ab55e5..5a7393b 100644 --- a/task-definitions-service/main.go +++ b/task-definitions-service/main.go @@ -12,6 +12,7 @@ import ( "syscall" log "github.com/sirupsen/logrus" + "github.com/zebrunner/esg/cachemaps/utilsmap" "github.com/zebrunner/esg/config" "github.com/zebrunner/esg/environment" envtype "github.com/zebrunner/esg/environment/envType" @@ -121,6 +122,10 @@ func main() { utils.ExitWithError(err, "Failed to init redis connection", log.NewEntry(log.StandardLogger())) } + if err = utilsmap.TaskDefinitionsVersion.Set(config.Version); err != nil { + log.WithError(err).Error("Failed to set task-definitions version in cache") + } + listener, err := net.Listen("tcp", listen) if err != nil { log.WithError(err).Fatalf("failed to listen tcp port %s", listen) From 048c8c05169926f13a003b73ba44faa54660c3dc Mon Sep 17 00:00:00 2001 From: Andrei Kamarouski Date: Tue, 24 Sep 2024 11:10:08 +0200 Subject: [PATCH 12/12] changes --- cachemaps/definitionmap/definitionmap.go | 1 - cmd/task-definitions/main.go | 44 +------ handlers/definitions.go | 18 --- task-definitions-service/cache/cache.go | 139 ++++++++++++++++++++- task-definitions-service/definitions.proto | 2 + task-definitions-service/main.go | 28 +++++ 6 files changed, 169 insertions(+), 63 deletions(-) diff --git a/cachemaps/definitionmap/definitionmap.go b/cachemaps/definitionmap/definitionmap.go index b905e03..65b2993 100644 --- a/cachemaps/definitionmap/definitionmap.go +++ b/cachemaps/definitionmap/definitionmap.go @@ -10,7 +10,6 @@ import ( ) var ( - definitionsMap map[string]int64 mutex = &sync.RWMutex{} ) diff --git a/cmd/task-definitions/main.go b/cmd/task-definitions/main.go index 366743c..9999788 100644 --- a/cmd/task-definitions/main.go +++ b/cmd/task-definitions/main.go @@ -2,21 +2,15 @@ package main import ( "context" - "flag" "net/http" - "os" - "os/signal" - "syscall" "time" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" - "github.com/zebrunner/esg/cachemaps/utilsmap" "github.com/zebrunner/esg/config" "github.com/zebrunner/esg/definitions" "github.com/zebrunner/esg/handlers" "github.com/zebrunner/esg/images" - "github.com/zebrunner/esg/service" "github.com/zebrunner/esg/utils" ) @@ -101,44 +95,8 @@ func CreateRouter() *gin.Engine { } func main() { - defer func() { - config.CloseConnections() - }() - - flag.Parse() - - log.SetLevel(config.Conf.ParseLogLevel()) - awsSess, err := service.InitAws() - if err != nil { - utils.ExitWithError(err, "Failed to init aws session", log.NewEntry(log.StandardLogger())) - } - service.AwsSess = awsSess - - err = config.InitDBConnection(config.Conf.DbConnectionString) - if err != nil { - utils.ExitWithError(err, "Failed to init DB client", log.NewEntry(log.StandardLogger())) - } - - err = config.InitRedisClusterConnection() - if err != nil { - utils.ExitWithError(err, "Failed to init redis connection", log.NewEntry(log.StandardLogger())) - } - - err = utilsmap.TaskDefinitionsVersion.Set(config.Version) - if err != nil { - log.WithError(err).Error("Failed to set task-definitions version in cache") - } - - // create sigterm listener chan - quit := make(chan os.Signal, 1) - signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) - - // wrapping router by http.Server object and starting it in new thread to wait for quit chan signal - srv := &http.Server{ - Addr: listen, - Handler: CreateRouter(), - } + go func() { log.Infof("Listening on %s", listen) if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { diff --git a/handlers/definitions.go b/handlers/definitions.go index b8d0878..e738ebd 100644 --- a/handlers/definitions.go +++ b/handlers/definitions.go @@ -12,29 +12,11 @@ import ( "github.com/zebrunner/esg/task-definitions-service/definitions" ) -var ( - DefinitionRefreshDone = false -) - -type imageDataModel struct { - Name string `json:"name"` - Version string `json:"version"` - Platform string `json:"platform"` - BrowserName string `json:"browserName,omitempty"` - BrowserVersion string `json:"browserVersion,omitempty"` - // TODO: investigate possibility of 'ImageUrl' field removal - ImageUrl string `json:"image,omitempty"` -} - type refreshDefinitionsModel struct { ImageRepositories *string `json:"imageRespositories,omitempty"` ExcludeBrowsers *string `json:"excludeBrowsers,omitempty"` } -func Ready(c *gin.Context) { - c.String(http.StatusOK, "ready to accept requests") -} - func IsTaskDefinitionRefreshDone(c *gin.Context) { if DefinitionRefreshDone { c.Status(http.StatusOK) diff --git a/task-definitions-service/cache/cache.go b/task-definitions-service/cache/cache.go index 663ec37..b88bd7c 100644 --- a/task-definitions-service/cache/cache.go +++ b/task-definitions-service/cache/cache.go @@ -1,11 +1,22 @@ package cache import ( + "database/sql" "sync" + "time" + + log "github.com/sirupsen/logrus" + "github.com/zebrunner/esg/cachemaps" + "github.com/zebrunner/esg/config" + "github.com/zebrunner/esg/db" + "github.com/zebrunner/esg/environment" + "github.com/zebrunner/esg/images" + "github.com/zebrunner/esg/service" ) var ( - cache = &RevisionsCache{} + cache = &RevisionsCache{} + refreshDone = &RefreshDone{} ) type RevisionsCache struct { @@ -13,6 +24,132 @@ type RevisionsCache struct { Cache map[string]int64 } +type RefreshDone struct { + sync.Mutex + DefinitionRefreshDone bool +} + +type hashRevision struct { + Hash string + Revision int64 +} + func GetCache() *RevisionsCache { return cache } + +func Refresh() error { + refreshDone.Lock() + refreshDone.DefinitionRefreshDone = false + refreshDone.Unlock() + + images, err := images.ListImages(config.Conf.ImageRepositories, config.Conf.ExcludeBrowsers) + if err != nil { + log.WithError(err).Error("failed to generate images") + return err + } + + revisions := make(map[string]int64) + for _, img := range images { + envs, err := buildEnvs(img) + if err != nil { + log.WithError(err).Error("Failed to build execution environments from images list") + return err + } + for _, env := range envs { + dbTaskDefinition, err := compareWithStoredTaskDefinition(env) + if err != nil { + log.WithError(err).WithField("family", env.TaskDefinitionFamily).Error("Couldn't create task defenition") + return err + } + revisions[dbTaskDefinition.OverrideDefinitionHash] = dbTaskDefinition.RevisionTag + } + } + + if err := WriteAll(revisions); err != nil { + log.WithError(err).Error("Failed to add hashRevision map to redis") + return err + } + + refreshDone.Lock() + refreshDone.DefinitionRefreshDone = true + refreshDone.Unlock() + return nil +} + +// Add new revisions +func WriteAll(definitions map[string]int64) error { + hashRevisionMap := make(map[string]hashRevision, len(definitions)) + for k, v := range definitions { + hashRevisionMap[k] = hashRevision{Hash: k, Revision: v} + } + + return cachemaps.WriteAll(config.RedisCluster.Pipeline(), cachemaps.DEFINITION, hashRevisionMap) +} + +func compareWithStoredTaskDefinition(env *environment.ExecutionEnvironment) (*db.TaskDefinition, error) { + l := log.WithField("schema", env.Schema).WithField("family", env.TaskDefinitionFamily) + + newDbDefinititon := db.CreateTaskDefinitionEntity(env) + savedDbDefinition, err := db.GetDefinition(env.TaskDefinitionFamily, env.Schema) + if err != nil { + if err != sql.ErrNoRows { + return nil, err + } + + l.Info("Creating new record") + taskDef, err := service.CreateTaskDefinition(env.ContainerDefinitions(), env.Volume(), env.TaskDefinitionFamily, env.TaskRoleArn) + if err != nil { + return nil, err + } + // pause after aws call + time.Sleep(1 * time.Second) + newDbDefinititon.RevisionTag = *taskDef.Revision + + err = db.InsertDefinition(newDbDefinititon) + if err != nil { + return nil, err + } + } else if newDbDefinititon.RegisterDefinitionHash != savedDbDefinition.RegisterDefinitionHash { + l.Debug("Updating definition record") + taskDef, err := service.CreateTaskDefinition(env.ContainerDefinitions(), env.Volume(), env.TaskDefinitionFamily, env.TaskRoleArn) + if err != nil { + return nil, err + } + // pause after aws call + time.Sleep(1 * time.Second) + newDbDefinititon.RevisionTag = *taskDef.Revision + + err = db.RefreshTag(savedDbDefinition.RegisterDefinitionHash, newDbDefinititon) + if err != nil { + return nil, err + } + } else { + l.Trace("Definition record is up-to-date") + newDbDefinititon.RevisionTag = savedDbDefinition.RevisionTag + } + + return newDbDefinititon, nil +} + +func buildEnvs(image images.Image) ([]*environment.ExecutionEnvironment, error) { + l := log.WithField("image", image.String()) + + capsList, err := image.GetMockCapabilities() + if err != nil { + l.WithError(err).Error("Failed to build capabilitites from image!") + return nil, err + } + + envsList := make([]*environment.ExecutionEnvironment, 0) + for _, caps := range capsList { + env, err := environment.BuildEnvForTaskDefinitionGeneration(image, caps) + if err != nil { + l.WithError(err).Error("Failed to build execution environment") + return nil, err + } + + envsList = append(envsList, env) + } + return envsList, nil +} diff --git a/task-definitions-service/definitions.proto b/task-definitions-service/definitions.proto index 0e8c4da..fa794cb 100644 --- a/task-definitions-service/definitions.proto +++ b/task-definitions-service/definitions.proto @@ -17,6 +17,8 @@ service Service { // get images list from ecr (public and private) rpc getImages(google.protobuf.Empty) returns (stream Image); + + r } message Configuration { diff --git a/task-definitions-service/main.go b/task-definitions-service/main.go index 5a7393b..2326a9e 100644 --- a/task-definitions-service/main.go +++ b/task-definitions-service/main.go @@ -10,6 +10,7 @@ import ( "os" "os/signal" "syscall" + "time" log "github.com/sirupsen/logrus" "github.com/zebrunner/esg/cachemaps/utilsmap" @@ -122,6 +123,7 @@ func main() { utils.ExitWithError(err, "Failed to init redis connection", log.NewEntry(log.StandardLogger())) } + // todo move TaskDefinitionsVersions variable to the current service if err = utilsmap.TaskDefinitionsVersion.Set(config.Version); err != nil { log.WithError(err).Error("Failed to set task-definitions version in cache") } @@ -139,6 +141,32 @@ func main() { log.WithError(err).Fatal("Failed to start task-definitions server") } }() + + // update cache + go func() { + err := environmentRefresh() + if err != nil { + return err + } + + envUpdateInterval := time.Hour * 12 + go func() { + time.Sleep(envUpdateInterval) + + for { + err := environmentUpdate() + if err != nil { + log.WithError(err).Warn("Failed to update task definitions. Retrying...") + time.Sleep(time.Second * 15) + } else { + time.Sleep(envUpdateInterval) + } + } + }() + + return nil + }() + log.Info("Service started") <-quit