Cherry-pick #4767 #4815 #4821 #4826 #4825 #4834 #4819 #4857 (#4860)

* BUGFIX: Added cancel method to fix context leak (#4767)

Signed-off-by: fazledyn-or <[email protected]>

* Define piped pluggin api (#4815)

Signed-off-by: khanhtc1202 <[email protected]>

* Update BuldPlan API for piped pluggin (#4821)

Signed-off-by: khanhtc1202 <[email protected]>

* Relocate plugin proto (#4826)

Signed-off-by: khanhtc1202 <[email protected]>

* Update controller to use new planner logic (#4825)

* Update controller to use new planner logic

Signed-off-by: khanhtc1202 <[email protected]>

* Update proto path

Signed-off-by: khanhtc1202 <[email protected]>

* Fix typo

Signed-off-by: khanhtc1202 <[email protected]>

* Fix typo

Signed-off-by: khanhtc1202 <[email protected]>

* Update planner logic to call proto instead of self executing

Signed-off-by: khanhtc1202 <[email protected]>

---------

Signed-off-by: khanhtc1202 <[email protected]>

* Update plugin proto for ExecutorService and add piped pluginservice (#4834)

* Add plugin planner for k8s (#4819)

* [WIP] Add planner

Signed-off-by: Yoshiki Fujikane <[email protected]>

* Not to use out.Version

Signed-off-by: Yoshiki Fujikane <[email protected]>

* Use last_successful_commit_hash and last_successful_config_file_name

Signed-off-by: Yoshiki Fujikane <[email protected]>

* Use in.WorkingDir

Signed-off-by: Yoshiki Fujikane <[email protected]>

* Use in.PipedConfig

Signed-off-by: Yoshiki Fujikane <[email protected]>

* Create git client

Signed-off-by: Yoshiki Fujikane <[email protected]>

* Create secret encryptor

Signed-off-by: Yoshiki Fujikane <[email protected]>

* Add startup server implementation

Signed-off-by: Yoshiki Fujikane <[email protected]>

* Fix for relocation of proto api

Signed-off-by: Yoshiki Fujikane <[email protected]>

* Add roughly implementation for planner plugin

Signed-off-by: Yoshiki Fujikane <[email protected]>

* Rename pkg name

Signed-off-by: Yoshiki Fujikane <[email protected]>

* Add licence

Signed-off-by: Yoshiki Fujikane <[email protected]>

* Comment out for the testing code

Signed-off-by: Yoshiki Fujikane <[email protected]>

---------

Signed-off-by: Yoshiki Fujikane <[email protected]>

* Truncate `deploymentStatus` metrics after reporting stats (#4857)

* Truncate deploymentStatus metrics after reporting to avoid excess message size

Signed-off-by: t-kikuc <[email protected]>

* Rename func to Flush() for clarity

Signed-off-by: t-kikuc <[email protected]>

* Add comment of what's included in statsreporter's body

Signed-off-by: t-kikuc <[email protected]>

* Fix indent in the comment

Signed-off-by: t-kikuc <[email protected]>

* Copy change of metrics.go to pipedv1

Signed-off-by: t-kikuc <[email protected]>

* Copy change of reporter.go to pipedv1

Signed-off-by: t-kikuc <[email protected]>

---------

Signed-off-by: t-kikuc <[email protected]>

---------

Signed-off-by: fazledyn-or <[email protected]>
Signed-off-by: khanhtc1202 <[email protected]>
Signed-off-by: Yoshiki Fujikane <[email protected]>
Signed-off-by: t-kikuc <[email protected]>
Co-authored-by: Ataf Fazledin Ahamed <[email protected]>
Co-authored-by: Khanh Tran <[email protected]>
Co-authored-by: Yoshiki Fujikane <[email protected]>

Makefile

@@ -131,8 +131,11 @@ run/piped: CONFIG_FILE ?=
 run/piped: INSECURE ?= false
 run/piped: LAUNCHER ?= false
 run/piped: LOG_ENCODING ?= humanize
+run/piped: EXPERIMENTAL ?= false
 run/piped:
-ifeq ($(LAUNCHER),true)
+ifeq ($(EXPERIMENTAL), true)
+	go run cmd/pipedv1/main.go piped --tools-dir=/tmp/piped-bin --config-file=$(CONFIG_FILE) --insecure=$(INSECURE) --log-encoding=$(LOG_ENCODING)
+else ifeq ($(LAUNCHER),true)
 	go run cmd/launcher/main.go launcher --config-file=$(CONFIG_FILE) --insecure=$(INSECURE) --log-encoding=$(LOG_ENCODING)
 else
 	go run cmd/piped/main.go piped --tools-dir=/tmp/piped-bin --config-file=$(CONFIG_FILE) --insecure=$(INSECURE) --log-encoding=$(LOG_ENCODING)

cmd/pipecd/server.go

@@ -402,7 +402,8 @@ func runHTTPServer(ctx context.Context, httpServer *http.Server, gracePeriod tim
 
 	<-ctx.Done()
 
-	ctx, _ = context.WithTimeout(context.Background(), gracePeriod)
+	ctx, cancel = context.WithTimeout(context.Background(), gracePeriod)
+	defer cancel()
 	logger.Info("stopping http server")
 	if err := httpServer.Shutdown(ctx); err != nil {
 		logger.Error("failed to shutdown http server", zap.Error(err))

cmd/pipedv1/main.go

@@ -0,0 +1,35 @@
+// Copyright 2024 The PipeCD Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"log"
+
+	"github.com/pipe-cd/pipecd/pkg/app/pipedv1/cmd/piped"
+	"github.com/pipe-cd/pipecd/pkg/cli"
+)
+
+func main() {
+	app := cli.NewApp(
+		"piped",
+		"A component that runs inside target environment to execute deployment and report its state.",
+	)
+	app.AddCommands(
+		piped.NewCommand(),
+	)
+	if err := app.Run(); err != nil {
+		log.Fatal(err)
+	}
+}

pkg/app/piped/controller/controllermetrics/metrics.go

@@ -54,3 +54,8 @@ func Register(r prometheus.Registerer) {
 		deploymentStatus,
 	)
 }
+
+// Flush deletes all deploymentStatus metrics to remove old unnecessary metrics.
+func Flush() {
+	deploymentStatus.Reset()
+}

pkg/app/piped/statsreporter/reporter.go

@@ -25,6 +25,7 @@ import (
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
 
+	"github.com/pipe-cd/pipecd/pkg/app/piped/controller/controllermetrics"
 	"github.com/pipe-cd/pipecd/pkg/app/server/service/pipedservice"
 )
 
@@ -92,11 +93,24 @@ func (r *reporter) report(ctx context.Context) error {
 	}
 
 	req := &pipedservice.ReportStatRequest{
+		// PipedStats includes the following metrics in addition to Go metrics:
+		//  - cloudprovider_kubernetes_tool_calls_total
+		//  - deployment_status
+		//  - livestatestore_kubernetes_api_requests_total
+		//  - livestatestore_kubernetes_resource_events_total
+		//  - plan_preview_command_handled_total
+		//  - plan_preview_command_handling_seconds
+		//  - plan_preview_command_received_total
 		PipedStats: b,
 	}
 	if _, err := r.apiClient.ReportStat(ctx, req); err != nil {
 		r.logger.Error("failed to report stats", zap.Error(err))
 		return err
 	}
+
+	// Delete deployment metrics which are already reported
+	// in order to avoid error of excess message size.
+	controllermetrics.Flush()
+
 	return nil
 }

pkg/app/pipedv1/analysisprovider/http/http.go

@@ -0,0 +1,80 @@
+// Copyright 2024 The PipeCD Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package http provides a way to analyze with http requests.
+// This allows you to do smoke tests, load tests and so on, at your leisure.
+package http
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/pipe-cd/pipecd/pkg/config"
+)
+
+const (
+	ProviderType   = "HTTP"
+	defaultTimeout = 30 * time.Second
+)
+
+type Provider struct {
+	client *http.Client
+}
+
+func (p *Provider) Type() string {
+	return ProviderType
+}
+
+func NewProvider(timeout time.Duration) *Provider {
+	if timeout == 0 {
+		timeout = defaultTimeout
+	}
+	return &Provider{
+		client: &http.Client{Timeout: timeout},
+	}
+}
+
+// Run sends an HTTP request and then evaluate whether the response is expected one.
+func (p *Provider) Run(ctx context.Context, cfg *config.AnalysisHTTP) (bool, string, error) {
+	req, err := p.makeRequest(ctx, cfg)
+	if err != nil {
+		return false, "", err
+	}
+
+	res, err := p.client.Do(req)
+	if err != nil {
+		return false, "", err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != cfg.ExpectedCode {
+		return false, "", fmt.Errorf("unexpected status code %d", res.StatusCode)
+	}
+	// TODO: Decide how to check if the body is expected one.
+	return true, "", nil
+}
+
+func (p *Provider) makeRequest(ctx context.Context, cfg *config.AnalysisHTTP) (*http.Request, error) {
+	req, err := http.NewRequestWithContext(ctx, cfg.Method, cfg.URL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header = make(http.Header, len(cfg.Headers))
+	for _, h := range cfg.Headers {
+		req.Header.Set(h.Key, h.Value)
+	}
+	return req, nil
+}

pkg/app/pipedv1/analysisprovider/log/factory/factory.go

@@ -0,0 +1,47 @@
+// Copyright 2024 The PipeCD Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package factory
+
+import (
+	"fmt"
+	"os"
+
+	"go.uber.org/zap"
+
+	"github.com/pipe-cd/pipecd/pkg/app/piped/analysisprovider/log"
+	"github.com/pipe-cd/pipecd/pkg/app/piped/analysisprovider/log/stackdriver"
+	"github.com/pipe-cd/pipecd/pkg/config"
+	"github.com/pipe-cd/pipecd/pkg/model"
+)
+
+// NewProvider generates an appropriate provider according to analysis provider config.
+func NewProvider(providerCfg *config.PipedAnalysisProvider, logger *zap.Logger) (provider log.Provider, err error) {
+	switch providerCfg.Type {
+	case model.AnalysisProviderStackdriver:
+		cfg := providerCfg.StackdriverConfig
+		sa, err := os.ReadFile(cfg.ServiceAccountFile)
+		if err != nil {
+			return nil, err
+		}
+		provider, err = stackdriver.NewProvider(sa)
+		if err != nil {
+			return nil, err
+		}
+
+	default:
+		return nil, fmt.Errorf("any of providers config not found")
+	}
+	return provider, nil
+}

pkg/app/pipedv1/analysisprovider/log/provider.go

@@ -0,0 +1,28 @@
+// Copyright 2024 The PipeCD Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package log
+
+import (
+	"context"
+)
+
+// Provider represents a client for log provider which provides logs for analysis.
+type Provider interface {
+	Type() string
+	// Evaluate runs the given query against the log provider,
+	// and then checks if there is at least one error log.
+	// Returns the result reason if non-error occurred.
+	Evaluate(ctx context.Context, query string) (result bool, reason string, err error)
+}

pkg/app/pipedv1/analysisprovider/log/stackdriver/stackdriver.go

@@ -0,0 +1,43 @@
+// Copyright 2024 The PipeCD Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stackdriver
+
+import (
+	"context"
+	"time"
+)
+
+const ProviderType = "StackdriverLogging"
+
+// Provider is a client for stackdriver.
+type Provider struct {
+	serviceAccount []byte
+
+	timeout time.Duration
+}
+
+func NewProvider(serviceAccount []byte) (*Provider, error) {
+	return &Provider{
+		serviceAccount: serviceAccount,
+	}, nil
+}
+
+func (p *Provider) Type() string {
+	return ProviderType
+}
+
+func (p *Provider) Evaluate(ctx context.Context, query string) (bool, string, error) {
+	return false, "", nil
+}