From c6cb840563b550493739aaf25a515ac15477503d Mon Sep 17 00:00:00 2001 From: arbulu89 Date: Mon, 26 Feb 2024 10:48:23 +0100 Subject: [PATCH] Add tests to the evaluation code --- test/support/factory.ex | 2 +- test/wanda/catalog_test.exs | 33 +++ test/wanda/executions/evaluation_test.exs | 278 ++++++++++++++++++++++ 3 files changed, 312 insertions(+), 1 deletion(-) diff --git a/test/support/factory.ex b/test/support/factory.ex index 818fedd12..70159241b 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -57,7 +57,7 @@ defmodule Wanda.Factory do def catalog_expectation_factory do %Catalog.Expectation{ name: Faker.StarWars.character(), - type: Enum.random([:expect, :expect_same]), + type: Enum.random([:expect, :expect_same, :expect_enum]), expression: Faker.StarWars.quote() } end diff --git a/test/wanda/catalog_test.exs b/test/wanda/catalog_test.exs index b3d7267a6..4dabf7f02 100644 --- a/test/wanda/catalog_test.exs +++ b/test/wanda/catalog_test.exs @@ -140,6 +140,39 @@ defmodule Wanda.CatalogTest do }} = Catalog.get_check("expect_same_check") end + test "should load a expect_enum expectation type" do + assert {:ok, + %Check{ + values: [ + %Value{ + default: 5, + name: "expected_passing_value" + }, + %Value{ + default: 3, + name: "expected_warning_value" + } + ], + expectations: [ + %Expectation{ + name: "some_expectation", + type: :expect_enum, + expression: """ + if facts.jedi == values.expected_passing_value { + "passing" + } else if facts.jedi == values.expected_warning_value { + "warning" + } else { + "critical" + } + """, + failure_message: "some failure message ${facts.jedi}", + warning_message: "some warning message ${facts.jedi}" + } + ] + }} = Catalog.get_check("expect_enum_check") + end + test "should load a warning severity" do assert {:ok, %Check{severity: :warning}} = Catalog.get_check("warning_severity_check") end diff --git a/test/wanda/executions/evaluation_test.exs b/test/wanda/executions/evaluation_test.exs index 5bb9cb33c..d60cc5a06 100644 --- a/test/wanda/executions/evaluation_test.exs +++ b/test/wanda/executions/evaluation_test.exs @@ -686,6 +686,284 @@ defmodule Wanda.Executions.EvaluationTest do end end + describe "expect_enum" do + test "should weight properly the return values", %{engine: engine} do + [ + %Catalog.Fact{name: fact_1_name}, + %Catalog.Fact{name: fact_2_name} + ] = catalog_facts = build_list(2, :catalog_fact) + + expression_1 = """ + if facts.#{fact_1_name} == 10 { + "passing" + } else if facts.#{fact_1_name} == 5 { + "warning" + } + """ + + expression_2 = """ + if facts.#{fact_2_name} == 20 { + "passing" + } else if facts.#{fact_2_name} == 15 { + "warning" + } + """ + + scenarios = [ + %{ + facts: [ + %{agent_1: 10, agent_2: 10}, + %{agent_1: 20, agent_2: 20} + ], + expectation_1_result: :passing, + expectation_2_result: :passing, + result: :passing + }, + %{ + facts: [ + %{agent_1: 5, agent_2: 10}, + %{agent_1: 20, agent_2: 20} + ], + expectation_1_result: :warning, + expectation_2_result: :passing, + result: :warning + }, + %{ + facts: [ + %{agent_1: 3, agent_2: 10}, + %{agent_1: 20, agent_2: 20} + ], + expectation_1_result: :critical, + expectation_2_result: :passing, + result: :critical + }, + %{ + facts: [ + %{agent_1: 3, agent_2: 10}, + %{agent_1: 20, agent_2: 15} + ], + expectation_1_result: :critical, + expectation_2_result: :warning, + result: :critical + } + ] + + Enum.each(scenarios, fn %{ + facts: [ + %{ + agent_1: agent_1_fact_1_value, + agent_2: agent_2_fact_1_value + }, + %{ + agent_1: agent_1_fact_2_value, + agent_2: agent_2_fact_2_value + } + ], + expectation_1_result: expectation_1_result, + expectation_2_result: expectation_2_result, + result: result + } -> + expectations = [ + build(:catalog_expectation, + name: "expectation_1", + type: :expect_enum, + expression: expresion_1 + ), + build(:catalog_expectation, + name: "expectation_2", + type: :expect_enum, + expression: expression_2 + ) + ] + + [%Catalog.Check{id: check_id}] = + checks = + build_list(1, :check, facts: catalog_facts, values: [], expectations: expectations) + + facts_agent_1 = [ + build(:fact, name: fact_1_name, check_id: check_id, value: agent_1_fact_1_value), + build(:fact, name: fact_2_name, check_id: check_id, value: agent_1_fact_2_value) + ] + + facts_agent_2 = [ + build(:fact, name: fact_1_name, check_id: check_id, value: agent_2_fact_1_value), + build(:fact, name: fact_2_name, check_id: check_id, value: agent_2_fact_2_value) + ] + + gathered_facts = %{ + check_id => %{ + "agent_1" => facts_agent_1, + "agent_2" => facts_agent_2 + } + } + + assert %Result{ + result: ^result, + check_results: [ + %CheckResult{ + result: ^result, + expectation_results: [ + %ExpectationResult{ + name: "expectation_1", + result: ^expectation_1_result, + type: :expect_enum + }, + %ExpectationResult{ + name: "expectation_2", + result: ^expectation_2_result, + type: :expect_enum + } + ] + } + ] + } = + Evaluation.execute( + UUID.uuid4(), + UUID.uuid4(), + checks, + gathered_facts, + %{}, + engine + ) + end) + end + + test "should set to critical unknown return values", %{engine: engine} do + [%Catalog.Fact{name: fact_name}] = catalog_facts = build_list(1, :catalog_fact) + + expectations = + build_list(1, :catalog_expectation, + name: "some_expectation", + type: :expect_enum, + expression: """ + if facts.#{fact_name} == "unknown" { + "unknown" + } + """ + ) + + [%Catalog.Check{id: check_id}] = + checks = + build_list(1, :check, facts: catalog_facts, values: [], expectations: expectations) + + facts = build_list(1, :fact, name: fact_name, check_id: check_id, value: "unknown") + + gathered_facts = %{ + check_id => %{ + "agent" => facts + } + } + + assert %Result{ + result: :critical, + check_results: [ + %CheckResult{ + result: :critical, + agents_check_results: [ + %AgentCheckResult{ + expectation_evaluations: [ + %ExpectationEvaluation{ + name: "some_expectation", + return_value: :critical, + type: :expect_enum, + failure_message: "Expectation not met" + } + ] + } + ], + expectation_results: [ + %ExpectationResult{ + name: "some_expectation", + result: :critical, + type: :expect_enum + } + ] + } + ] + } = + Evaluation.execute(UUID.uuid4(), UUID.uuid4(), checks, gathered_facts, %{}, engine) + end + + test "should interpolate the correct failure message", %{engine: engine} do + [%Catalog.Fact{name: fact_name}] = catalog_facts = build_list(1, :catalog_fact) + fact_value = 10 + + scenarios = [ + %{ + return_value: :warning, + message: "warning message: fact value #{fact_value}" + }, + %{ + return_value: :critical, + message: "failure message: fact value #{fact_value}" + } + ] + + Enum.each(scenarios, fn %{return_value: return_value, message: message} -> + expectations = + build_list(1, :catalog_expectation, + name: "some_expectation", + type: :expect_enum, + expression: """ + if facts.#{fact_name} == #{fact_value} { + "#{Atom.to_string(return_value)}" + } + """, + warning_message: "warning message: fact value ${facts.#{fact_name}}", + failure_message: "failure message: fact value ${facts.#{fact_name}}" + ) + + [%Catalog.Check{id: check_id}] = + checks = + build_list(1, :check, facts: catalog_facts, values: [], expectations: expectations) + + facts = build_list(1, :fact, name: fact_name, check_id: check_id, value: fact_value) + + gathered_facts = %{ + check_id => %{ + "agent" => facts + } + } + + assert %Result{ + result: ^return_value, + check_results: [ + %CheckResult{ + result: ^return_value, + agents_check_results: [ + %AgentCheckResult{ + expectation_evaluations: [ + %ExpectationEvaluation{ + name: "some_expectation", + return_value: ^return_value, + type: :expect_enum, + failure_message: ^message + } + ] + } + ], + expectation_results: [ + %ExpectationResult{ + name: "some_expectation", + result: ^return_value, + type: :expect_enum + } + ] + } + ] + } = + Evaluation.execute( + UUID.uuid4(), + UUID.uuid4(), + checks, + gathered_facts, + %{}, + engine + ) + end) + end + end + describe "expressions with arrays" do test "should return a passing result", %{engine: engine} do [value | _] =