From 944bee34af4044eed8c3b5f3e7257b946edfa2e3 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Fri, 1 Nov 2024 13:41:50 -0600 Subject: [PATCH 1/4] refactored error handlingl; refined documentation --- README.md | 4 ++-- cmd/root.go | 7 ++++-- internal/engine/config.go | 6 ++--- internal/engine/engine.go | 19 ++++++++++----- internal/engine/test.go | 24 +++++++++---------- ...g-headers.yml => fail-missing-headers.yml} | 0 6 files changed, 34 insertions(+), 26 deletions(-) rename tests/{pass-missing-headers.yml => fail-missing-headers.yml} (100%) diff --git a/README.md b/README.md index ff20617..ee86293 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,10 @@ **HTTP smoke testing made easy.** -Simply declare a list of URLs and their expected response values, and Emberfall will test, compare, and report which URLs fail along with details of what expectations were not met. +Simply declare a list of URLs and their expected response values, and Emberfall will test, compare, and report which URLs fail along with details of what expectations were not met. Tests are merely a list of request objects, each with a url, method, headers to be sent, and an expects field. With the `expects` field you can define the status code, body contents (as a string), and any headers (as strings) that should be present in the response. If anything expected is not present or not equal to the defined value `emberfall` will exit with a non-zero code. + ## Configuring Tests -Tests are merely a list of request objects, each with a url, method, headers to be sent, and an expects field. With the `expects` field you can define the status code, body contents (as a string), and any headers (as strings) that should be present in the response. If anything expected is not present or not equal to the defined value `emberfall` will exit with a non-zero code. The YAML tests config can be provided in two ways: - as a file: `emberfall --config path/to/config.yaml` diff --git a/cmd/root.go b/cmd/root.go index 086d37e..f73aa14 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -16,7 +16,7 @@ import ( var configPath string var rootCmd = &cobra.Command{ - Use: "gatling", + Use: "emberfall", Short: "Declarative API Testing", Run: func(cmd *cobra.Command, args []string) { configPath = strings.TrimSpace(configPath) @@ -25,7 +25,10 @@ var rootCmd = &cobra.Command{ fmt.Println(err) os.Exit(1) } - engine.Run(conf) + + if !engine.Run(conf) { + os.Exit(2) + } }, } diff --git a/internal/engine/config.go b/internal/engine/config.go index 6adf6bd..8934de6 100644 --- a/internal/engine/config.go +++ b/internal/engine/config.go @@ -25,13 +25,11 @@ func LoadConfig(configPath string) (*config, error) { if configPath == "-" { stat, err = os.Stdin.Stat() if err != nil { - fmt.Println(err) - os.Exit(1) + return nil, err } if stat.Size() < 1 { - fmt.Println("no config provided") - os.Exit(1) + return nil, fmt.Errorf("no config provided") } b, err = io.ReadAll(os.Stdin) diff --git a/internal/engine/engine.go b/internal/engine/engine.go index fcf900a..fe8eb50 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -5,12 +5,13 @@ import ( "net/http" ) -func Run(cfg *config) { +func Run(cfg *config) (success bool) { var ( - client *http.Client = &http.Client{} - req *http.Request - res *http.Response - err error + client *http.Client = &http.Client{} + req *http.Request + res *http.Response + err error + failures int ) for _, test := range cfg.Tests { @@ -38,8 +39,14 @@ func Run(cfg *config) { continue } - test.report(res) + success = test.report(res) + if !success { + failures++ + } } + + fmt.Printf("\n Ran %d tests with %d failures\n", len(cfg.Tests), failures) + return } func noRedirect(req *http.Request, via []*http.Request) error { diff --git a/internal/engine/test.go b/internal/engine/test.go index 3dcdae9..7124257 100644 --- a/internal/engine/test.go +++ b/internal/engine/test.go @@ -4,7 +4,6 @@ import ( "fmt" "io" "net/http" - "os" "strings" ) @@ -20,9 +19,13 @@ type test struct { } } -func (t *test) report(res *http.Response) { - var errors []string - result := "PASS" +func (t *test) report(res *http.Response) (success bool) { + errors := []string{} + + result := map[bool]string{ + true: "PASS", + false: "FAIL", + } if t.Expect.Status != res.StatusCode { errors = append(errors, fmt.Sprintf("expected status == %d got %d", t.Expect.Status, res.StatusCode)) @@ -48,19 +51,16 @@ func (t *test) report(res *http.Response) { } } - if len(errors) > 0 { - result = "FAIL" + if len(errors) == 0 { + success = true } - fmt.Printf("%s : %s %s\n", result, t.Method, t.Url) + fmt.Printf("%s : %s %s\n", result[success], t.Method, t.Url) if len(errors) > 0 { for _, e := range errors { fmt.Printf(" %s\n", e) } - code := len(errors) - if code > 125 { - code = 125 - } - os.Exit(code) } + + return } diff --git a/tests/pass-missing-headers.yml b/tests/fail-missing-headers.yml similarity index 100% rename from tests/pass-missing-headers.yml rename to tests/fail-missing-headers.yml From bd6570d1541e938dc07b42c13db6ed4df8dd2113 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Fri, 1 Nov 2024 13:55:56 -0600 Subject: [PATCH 2/4] refining error handling to account for http request errors --- internal/engine/engine.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/engine/engine.go b/internal/engine/engine.go index fe8eb50..a52e18d 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -20,6 +20,7 @@ func Run(cfg *config) (success bool) { if err != nil { fmt.Println(err) + failures++ continue } @@ -36,15 +37,19 @@ func Run(cfg *config) (success bool) { res, err = client.Do(req) if err != nil { fmt.Println(err) + failures++ continue } - success = test.report(res) - if !success { + if !test.report(res) { failures++ } } + if failures == 0 { + success = true + } + fmt.Printf("\n Ran %d tests with %d failures\n", len(cfg.Tests), failures) return } From 3a37d331370ce45c970498d4253c7b8e34ee8051 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Fri, 1 Nov 2024 13:56:44 -0600 Subject: [PATCH 3/4] added test --- tests/cli.bats | 11 +++++++++-- tests/fail-bad-url.yml | 5 +++++ 2 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 tests/fail-bad-url.yml diff --git a/tests/cli.bats b/tests/cli.bats index 382be7a..dc8c878 100644 --- a/tests/cli.bats +++ b/tests/cli.bats @@ -29,11 +29,18 @@ setup() { assert_output --partial 'PASS' } - @test "SHOULD FAIL with missing headers" { - run ./emberfall --config ./tests/pass-missing-headers.yml + run ./emberfall --config ./tests/fail-missing-headers.yml assert_failure assert_output --partial 'FAIL' assert_output --partial 'expected header x-no-exist was missing' } + +@test "SHOULD FAIL with bad url" { + run ./emberfall --config ./tests/fail-bad-url.yml + assert_failure + assert_output --partial 'FAIL' + assert_output --partial 'no such host' +} + diff --git a/tests/fail-bad-url.yml b/tests/fail-bad-url.yml new file mode 100644 index 0000000..c441a47 --- /dev/null +++ b/tests/fail-bad-url.yml @@ -0,0 +1,5 @@ +tests: + - url: https://dfsdfkshfawsladfkhj.com + method: GET + expect: + status: 200 # should fail with != 301 From 95de5b202b8ea3ea82bf4e1fbfa582683e766838 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Fri, 1 Nov 2024 14:05:38 -0600 Subject: [PATCH 4/4] fix broken test --- tests/cli.bats | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/cli.bats b/tests/cli.bats index dc8c878..646166d 100644 --- a/tests/cli.bats +++ b/tests/cli.bats @@ -40,7 +40,6 @@ setup() { @test "SHOULD FAIL with bad url" { run ./emberfall --config ./tests/fail-bad-url.yml assert_failure - assert_output --partial 'FAIL' assert_output --partial 'no such host' }