From 62883db0fe9fe015c1fd6cc6cfb942b3c3970064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Ma=C5=82achowski?= Date: Fri, 31 Jan 2025 09:44:45 +0100 Subject: [PATCH 1/5] Strip timestamps before searching report string --- pkg/imagebuilder/report.go | 5 +++ pkg/imagebuilder/report_test.go | 66 ++++++++++++++++----------------- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/pkg/imagebuilder/report.go b/pkg/imagebuilder/report.go index c251e8dba591..7bcd35d09015 100644 --- a/pkg/imagebuilder/report.go +++ b/pkg/imagebuilder/report.go @@ -44,11 +44,16 @@ func (br *BuildReport) GetImages() []string { } func NewBuildReportFromLogs(log string) (*BuildReport, error) { + // Strip all timestamps from log + log = regexp.MustCompile(`\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z\s+`).ReplaceAllString(log, "") + + // Find the report in the log matches := reportRegex.FindStringSubmatch(log) if len(matches) < 2 { return nil, nil } + // Parse the report data var report BuildReport if err := json.Unmarshal([]byte(matches[1]), &report); err != nil { return nil, err diff --git a/pkg/imagebuilder/report_test.go b/pkg/imagebuilder/report_test.go index 7aaeefa36b69..be99455821d6 100644 --- a/pkg/imagebuilder/report_test.go +++ b/pkg/imagebuilder/report_test.go @@ -7,45 +7,41 @@ import ( var _ = Describe("Report", func() { Describe("NewReportFromLogs", func() { - logs := `Starting: prepare_image_build_report -============================================================================== -Task : Python script -Description : Run a Python file or inline script -Version : 0.248.1 -Author : Microsoft Corporation -Help : https://docs.microsoft.com/azure/devops/pipelines/tasks/utility/python-script -============================================================================== -/usr/bin/python /home/vsts/work/1/s/scripts/prepare_image_build_report.py --image-build-report-file /home/vsts/work/1/s/image-report.json --image-name ginkgo-test-image/ginkgo --sign-step-succeeded true --job-status Succeeded --image-build-report-file /home/vsts/work/_temp/generated-tags.json --images-to-sign=europe-docker.pkg.dev/kyma-project/prod/ginkgo-test-image/ginkgo:1.23.0-50049457 --images-to-sign=europe-docker.pkg.dev/kyma-project/prod/ginkgo-test-image/ginkgo:wartosc --images-to-sign=europe-docker.pkg.dev/kyma-project/prod/ginkgo-test-image/ginkgo:innytag --images-to-sign=europe-docker.pkg.dev/kyma-project/prod/ginkgo-test-image/ginkgo:v20250129-50049457 --images-to-sign=europe-docker.pkg.dev/kyma-project/prod/ginkgo-test-image/ginkgo:1.23.0 ----IMAGE BUILD REPORT--- -{ - "status": "Succeeded", - "pushed": true, - "signed": true, - "is_production": true, - "image_spec": { - "image_name": "ginkgo-test-image/ginkgo", - "tags": [ - "1.23.0-50049457", - "wartosc", - "innytag", - "v20250129-50049457", - "1.23.0" - ], - "repository_path": "europe-docker.pkg.dev/kyma-project/prod/" - } -} ----END OF IMAGE BUILD REPORT--- - -Finishing: prepare_image_build_report` + logs := `2025-01-31T08:32:23.5327056Z ##[section]Starting: prepare_image_build_report +2025-01-31T08:32:23.5434336Z ============================================================================== +2025-01-31T08:32:23.5434499Z Task : Python script +2025-01-31T08:32:23.5434594Z Description : Run a Python file or inline script +2025-01-31T08:32:23.5434703Z Version : 0.248.1 +2025-01-31T08:32:23.5434803Z Author : Microsoft Corporation +2025-01-31T08:32:23.5434910Z Help : https://docs.microsoft.com/azure/devops/pipelines/tasks/utility/python-script +2025-01-31T08:32:23.5435059Z ============================================================================== +2025-01-31T08:32:23.6965198Z [command]/opt/hostedtoolcache/Python/3.13.1/x64/bin/python /home/vsts/work/1/s/scripts/prepare_image_build_report.py --image-name github-tools-sap/conduit-cli --image-build-succeeded true --sign-step-succeeded $(sign_images.signing_success) --job-status Succeeded --images-to-sign=europe-docker.pkg.dev/kyma-project/dev/github-tools-sap/conduit-cli:PR-477 +2025-01-31T08:32:23.7344251Z ---IMAGE BUILD REPORT--- +2025-01-31T08:32:23.7345746Z { +2025-01-31T08:32:23.7346062Z "status": "Succeeded", +2025-01-31T08:32:23.7357582Z "pushed": true, +2025-01-31T08:32:23.7358184Z "signed": false, +2025-01-31T08:32:23.7358759Z "is_production": false, +2025-01-31T08:32:23.7359525Z "image_spec": { +2025-01-31T08:32:23.7360295Z "image_name": "github-tools-sap/conduit-cli", +2025-01-31T08:32:23.7360618Z "tags": [ +2025-01-31T08:32:23.7361207Z "PR-477" +2025-01-31T08:32:23.7361687Z ], +2025-01-31T08:32:23.7362370Z "repository_path": "europe-docker.pkg.dev/kyma-project/dev/" +2025-01-31T08:32:23.7362690Z } +2025-01-31T08:32:23.7363276Z } +2025-01-31T08:32:23.7363903Z ---END OF IMAGE BUILD REPORT--- +2025-01-31T08:32:23.7416532Z +2025-01-31T08:32:23.7530550Z ##[section]Finishing: prepare_image_build_report` expectedReport := &BuildReport{ Status: "Succeeded", IsPushed: true, - IsSigned: true, - IsProduction: true, + IsSigned: false, + IsProduction: false, ImageSpec: ImageSpec{ - Name: "ginkgo-test-image/ginkgo", - Tags: []string{"1.23.0-50049457", "wartosc", "innytag", "v20250129-50049457", "1.23.0"}, - RepositoryPath: "europe-docker.pkg.dev/kyma-project/prod/", + Name: "github-tools-sap/conduit-cli", + Tags: []string{"PR-477"}, + RepositoryPath: "europe-docker.pkg.dev/kyma-project/dev/", }, } From 1eccbbb554522c70e528c075a5e5cdbfe72fccd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Ma=C5=82achowski?= Date: Fri, 31 Jan 2025 09:52:55 +0100 Subject: [PATCH 2/5] Move regex up, to match reprotRegex --- pkg/imagebuilder/report.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/imagebuilder/report.go b/pkg/imagebuilder/report.go index 7bcd35d09015..8af1cac90442 100644 --- a/pkg/imagebuilder/report.go +++ b/pkg/imagebuilder/report.go @@ -8,7 +8,11 @@ import ( ) // reportRegex is a regular expression that matches the image build report -var reportRegex = regexp.MustCompile(`(?s)---IMAGE BUILD REPORT---\n(.*)\n---END OF IMAGE BUILD REPORT---`) +var ( + reportRegex = regexp.MustCompile(`(?s)---IMAGE BUILD REPORT---\n(.*)\n---END OF IMAGE BUILD REPORT---`) + + timestampRegex = regexp.MustCompile(`\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z\s+`) +) type BuildReport struct { // Status is the overall status of the build including signing and pushing @@ -45,7 +49,7 @@ func (br *BuildReport) GetImages() []string { func NewBuildReportFromLogs(log string) (*BuildReport, error) { // Strip all timestamps from log - log = regexp.MustCompile(`\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z\s+`).ReplaceAllString(log, "") + log = timestampRegex.ReplaceAllString(log, "") // Find the report in the log matches := reportRegex.FindStringSubmatch(log) From 8adc533fd0b8813c68fd411d6f69e087ac74488d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Ma=C5=82achowski?= Date: Fri, 31 Jan 2025 10:36:38 +0100 Subject: [PATCH 3/5] Fix parsing build report --- cmd/image-builder/main.go | 9 ++++--- pkg/imagebuilder/report.go | 7 +---- pkg/imagebuilder/report_test.go | 48 ++++++++++++++++++++++++++------- 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index bc37e09edbbd..7ef250855315 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -383,6 +383,8 @@ func buildInADO(o options) error { if err != nil { return fmt.Errorf("build in ADO failed, failed parsing build report from ADO pipeline run logs, err: %s", err) } + + o.logger.Debugw("Parsed build report from ADO logs", "buildReport", buildReport) } else { dryRunPipelineRunResult := pipelines.RunResult("Succeeded") pipelineRunResult = &dryRunPipelineRunResult @@ -394,9 +396,10 @@ func buildInADO(o options) error { if o.ciSystem == GithubActions { fmt.Println("Setting GitHub outputs.") images := buildReport.GetImages() - if !o.dryRun { - fmt.Printf("Extracted built images from ADO logs: %v\n", images) - } else { + + o.logger.Debugw("Extracted built images from ADO logs", "images", images) + + if o.dryRun { fmt.Println("Running in dry-run mode. Skipping extracting images and results from ADO.") images = []string{"registry/repo/image1:tag1", "registry/repo/image2:tag2"} } diff --git a/pkg/imagebuilder/report.go b/pkg/imagebuilder/report.go index 8af1cac90442..52972c86a4b8 100644 --- a/pkg/imagebuilder/report.go +++ b/pkg/imagebuilder/report.go @@ -72,13 +72,8 @@ func WriteReportToFile(report *BuildReport, path string) error { return fmt.Errorf("failed to marshal report: %w", err) } - file, err := os.Open(path) + err = os.WriteFile(path, data, os.ModePerm) if err != nil { - return fmt.Errorf("failed to open file: %w", err) - } - defer file.Close() - - if _, err := file.Write(data); err != nil { return fmt.Errorf("failed to write report to file: %w", err) } diff --git a/pkg/imagebuilder/report_test.go b/pkg/imagebuilder/report_test.go index be99455821d6..e10577a25284 100644 --- a/pkg/imagebuilder/report_test.go +++ b/pkg/imagebuilder/report_test.go @@ -54,15 +54,16 @@ var _ = Describe("Report", func() { }) Describe("GetImages", func() { - report := &BuildReport{ - ImageSpec: ImageSpec{ - Name: "ginkgo-test-image/ginkgo", - Tags: []string{"1.23.0-50049457", "wartosc", "innytag", "v20250129-50049457", "1.23.0"}, - RepositoryPath: "europe-docker.pkg.dev/kyma-project/prod/", - }, - } - It("returns the list of images", func() { + It("returns the list of images from build report", func() { + report := &BuildReport{ + ImageSpec: ImageSpec{ + Name: "ginkgo-test-image/ginkgo", + Tags: []string{"1.23.0-50049457", "wartosc", "innytag", "v20250129-50049457", "1.23.0"}, + RepositoryPath: "europe-docker.pkg.dev/kyma-project/prod/", + }, + } + expectedImages := []string{ "europe-docker.pkg.dev/kyma-project/prod/ginkgo-test-image/ginkgo:1.23.0-50049457", "europe-docker.pkg.dev/kyma-project/prod/ginkgo-test-image/ginkgo:wartosc", @@ -75,7 +76,14 @@ var _ = Describe("Report", func() { }) It("returns an empty list if there are no tags", func() { - report.ImageSpec.Tags = []string{} + report := &BuildReport{ + ImageSpec: ImageSpec{ + Name: "ginkgo-test-image/ginkgo", + Tags: []string{}, + RepositoryPath: "europe-docker.pkg.dev/kyma-project/prod/", + }, + } + Expect(report.GetImages()).To(BeEmpty()) }) @@ -84,4 +92,26 @@ var _ = Describe("Report", func() { Expect(nilReport.GetImages()).To(BeEmpty()) }) }) + + Describe("WriteReportToFile", func() { + report := &BuildReport{ + Status: "Succeeded", + IsPushed: true, + IsSigned: false, + IsProduction: false, + ImageSpec: ImageSpec{ + Name: "github-tools-sap/conduit-cli", + Tags: []string{"PR-477"}, + RepositoryPath: "europe-docker.pkg.dev/kyma-project/dev/", + }, + } + + It("writes the report to a file", func() { + path := "/tmp/report.json" + err := WriteReportToFile(report, path) + Expect(err).ToNot(HaveOccurred()) + + Expect(path).To(BeAnExistingFile()) + }) + }) }) From c4701121a042fe2314fa0b64d96d848d67b9a7fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Ma=C5=82achowski?= Date: Fri, 31 Jan 2025 11:09:53 +0100 Subject: [PATCH 4/5] Add debug logs, cean up code --- cmd/image-builder/main.go | 9 +++------ pkg/imagebuilder/report.go | 8 ++------ pkg/imagebuilder/report_test.go | 12 +++++++----- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index 7ef250855315..0f468502b50b 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -399,25 +399,22 @@ func buildInADO(o options) error { o.logger.Debugw("Extracted built images from ADO logs", "images", images) - if o.dryRun { - fmt.Println("Running in dry-run mode. Skipping extracting images and results from ADO.") - images = []string{"registry/repo/image1:tag1", "registry/repo/image2:tag2"} - } data, err := json.Marshal(images) if err != nil { return fmt.Errorf("cannot marshal list of images: %w", err) } + o.logger.Debugw("Set GitHub outputs", "images", string(data), "adoResult", string(*pipelineRunResult)) + err = actions.SetOutput("images", string(data)) if err != nil { return fmt.Errorf("cannot set images GitHub output: %w", err) } - fmt.Println("images GitHub output set") + err = actions.SetOutput("adoResult", string(*pipelineRunResult)) if err != nil { return fmt.Errorf("cannot set adoResult GitHub output: %w", err) } - fmt.Println("adoResult GitHub output set") } if o.buildReportPath != "" { diff --git a/pkg/imagebuilder/report.go b/pkg/imagebuilder/report.go index 52972c86a4b8..6f51d5ca6a4d 100644 --- a/pkg/imagebuilder/report.go +++ b/pkg/imagebuilder/report.go @@ -11,7 +11,7 @@ import ( var ( reportRegex = regexp.MustCompile(`(?s)---IMAGE BUILD REPORT---\n(.*)\n---END OF IMAGE BUILD REPORT---`) - timestampRegex = regexp.MustCompile(`\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z\s+`) + timestampRegex = regexp.MustCompile(`\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z\s`) ) type BuildReport struct { @@ -36,10 +36,6 @@ type ImageSpec struct { func (br *BuildReport) GetImages() []string { var images []string - if br == nil { - return images - } - for _, tag := range br.ImageSpec.Tags { images = append(images, fmt.Sprintf("%s%s:%s", br.ImageSpec.RepositoryPath, br.ImageSpec.Name, tag)) } @@ -54,7 +50,7 @@ func NewBuildReportFromLogs(log string) (*BuildReport, error) { // Find the report in the log matches := reportRegex.FindStringSubmatch(log) if len(matches) < 2 { - return nil, nil + return nil, fmt.Errorf("no image build report found in log") } // Parse the report data diff --git a/pkg/imagebuilder/report_test.go b/pkg/imagebuilder/report_test.go index e10577a25284..5bb7e085de11 100644 --- a/pkg/imagebuilder/report_test.go +++ b/pkg/imagebuilder/report_test.go @@ -51,6 +51,13 @@ var _ = Describe("Report", func() { Expect(actual).To(Equal(expectedReport)) }) + + It("returns an error if the log does not contain the image build report", func() { + logs := `2025-01-31T08:32:23.5327056Z ##[section]Starting: prepare_image_build_report` + + _, err := NewBuildReportFromLogs(logs) + Expect(err).To(HaveOccurred()) + }) }) Describe("GetImages", func() { @@ -86,11 +93,6 @@ var _ = Describe("Report", func() { Expect(report.GetImages()).To(BeEmpty()) }) - - It("returns an empty list if build report is nil", func() { - var nilReport *BuildReport - Expect(nilReport.GetImages()).To(BeEmpty()) - }) }) Describe("WriteReportToFile", func() { From 6a2b4390bbc8a4346c2a388b78e3538d7bdd62e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Ma=C5=82achowski?= Date: Fri, 31 Jan 2025 13:16:37 +0100 Subject: [PATCH 5/5] Fix regex that search for builder report --- cmd/image-builder/main.go | 1 + pkg/imagebuilder/report.go | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index 0f468502b50b..1dae3f5ea71e 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -378,6 +378,7 @@ func buildInADO(o options) error { fmt.Printf("ADO pipeline image build logs:\n%s", logs) } + fmt.Println("Getting build report.") // Parse the build report from the ADO pipeline run logs. buildReport, err = imagebuilder.NewBuildReportFromLogs(logs) if err != nil { diff --git a/pkg/imagebuilder/report.go b/pkg/imagebuilder/report.go index 6f51d5ca6a4d..41e6952a15b2 100644 --- a/pkg/imagebuilder/report.go +++ b/pkg/imagebuilder/report.go @@ -9,7 +9,7 @@ import ( // reportRegex is a regular expression that matches the image build report var ( - reportRegex = regexp.MustCompile(`(?s)---IMAGE BUILD REPORT---\n(.*)\n---END OF IMAGE BUILD REPORT---`) + reportRegex = regexp.MustCompile(`(?s)---IMAGE BUILD REPORT---(.*)---END OF IMAGE BUILD REPORT---`) timestampRegex = regexp.MustCompile(`\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z\s`) ) @@ -36,6 +36,10 @@ type ImageSpec struct { func (br *BuildReport) GetImages() []string { var images []string + if br == nil { + return images + } + for _, tag := range br.ImageSpec.Tags { images = append(images, fmt.Sprintf("%s%s:%s", br.ImageSpec.RepositoryPath, br.ImageSpec.Name, tag)) } @@ -50,7 +54,7 @@ func NewBuildReportFromLogs(log string) (*BuildReport, error) { // Find the report in the log matches := reportRegex.FindStringSubmatch(log) if len(matches) < 2 { - return nil, fmt.Errorf("no image build report found in log") + return nil, fmt.Errorf("failed to find image build report in log") } // Parse the report data