Merge pull request #144 from ddosify/develop

Change CSV Read Location
getanteon · Feb 22, 2023 · ab8fa74 · ab8fa74
2 parents c1b086c + 5152a24
commit ab8fa74
Show file tree

Hide file tree

Showing 10 changed files with 194 additions and 75 deletions.
diff --git a/README.md b/README.md
@@ -604,7 +604,7 @@ If Ddosify can't receive the response for a request, that step is marked as Fail
 | `not`   | ( param `bool` ) | returns converse of given param |
 | `range`   | ( param `int`, low `int`,high `int` ) | returns param is in range of [low,high): low is included, high is not included. |
 | `json_path`   | ( json_path `string`) | extracts from response body using given json path |
-| `xml_path`   | ( xpath `string` ) | extracts from response body using given xml path |
+| `xpath`   | ( xpath `string` ) | extracts from response body using given xml path |
 | `regexp` | ( param `any`, regexp `string`, matchNo `int` ) | extracts from given value in the first parameter using given regular expression |
 
 ### Operators

diff --git a/config/json.go b/config/json.go
@@ -189,30 +189,9 @@ func (j *JsonReader) Init(jsonByte []byte) (err error) {
 }
 
 func (j *JsonReader) CreateHammer() (h types.Hammer, err error) {
-	// Read Data
-	var readData map[string]types.CsvData
-	if len(j.Data) > 0 {
-		readData = make(map[string]types.CsvData, len(j.Data))
-	}
-	for k, conf := range j.Data {
-		var rows []map[string]interface{}
-		rows, err = readCsv(conf)
-		if err != nil {
-			return
-		}
-		var csvData types.CsvData
-		csvData.Rows = rows
-
-		if conf.Order == "random" {
-			csvData.Random = true
-		}
-		readData[k] = csvData
-	}
-
 	// Scenario
 	s := types.Scenario{
 		Envs: j.Envs,
-		Data: readData,
 	}
 	var si types.ScenarioStep
 	for _, step := range j.Steps {
@@ -264,6 +243,26 @@ func (j *JsonReader) CreateHammer() (h types.Hammer, err error) {
 		samplingRate = types.DefaultSamplingCount
 	}
 
+	testDataConf := make(map[string]types.CsvConf)
+	for key, val := range j.Data {
+		vars := make(map[string]types.Tag)
+		for k, v := range val.Vars {
+			vars[k] = types.Tag{
+				Tag:  v.Tag,
+				Type: v.Type,
+			}
+		}
+		testDataConf[key] = types.CsvConf{
+			Path:          val.Path,
+			Delimiter:     val.Delimiter,
+			SkipFirstLine: val.SkipFirstLine,
+			Vars:          vars,
+			SkipEmptyLine: val.SkipEmptyLine,
+			AllowQuota:    val.AllowQuota,
+			Order:         val.Order,
+		}
+	}
+
 	// Hammer
 	h = types.Hammer{
 		IterationCount:    *j.IterCount,
@@ -275,6 +274,7 @@ func (j *JsonReader) CreateHammer() (h types.Hammer, err error) {
 		ReportDestination: j.Output,
 		Debug:             j.Debug,
 		SamplingRate:      samplingRate,
+		TestDataConf:      testDataConf,
 	}
 	return
 }

diff --git a/config/json_test.go b/config/json_test.go
@@ -56,6 +56,7 @@ func TestCreateHammerDefaultValues(t *testing.T) {
 			Strategy: proxy.ProxyTypeSingle,
 		},
 		SamplingRate: types.DefaultSamplingCount,
+		TestDataConf: make(map[string]types.CsvConf),
 	}
 
 	h, err := jsonReader.CreateHammer()
@@ -111,6 +112,7 @@ func TestCreateHammer(t *testing.T) {
 			Addr:     addr,
 		},
 		SamplingRate: types.DefaultSamplingCount,
+		TestDataConf: make(map[string]types.CsvConf),
 	}
 
 	h, err := jsonReader.CreateHammer()
@@ -166,6 +168,7 @@ func TestCreateHammerWithIterationCountInsteadOfReqCount(t *testing.T) {
 			Addr:     addr,
 		},
 		SamplingRate: types.DefaultSamplingCount,
+		TestDataConf: make(map[string]types.CsvConf),
 	}
 
 	h, err := jsonReader.CreateHammer()
@@ -224,6 +227,7 @@ func TestCreateHammerWithIterationCountOverridesReqCount(t *testing.T) {
 			Addr:     addr,
 		},
 		SamplingRate: types.DefaultSamplingCount,
+		TestDataConf: make(map[string]types.CsvConf),
 	}
 
 	h, err := jsonReader.CreateHammer()
@@ -259,6 +263,7 @@ func TestCreateHammerManualLoad(t *testing.T) {
 			Strategy: proxy.ProxyTypeSingle,
 		},
 		SamplingRate: types.DefaultSamplingCount,
+		TestDataConf: make(map[string]types.CsvConf),
 	}
 
 	h, err := jsonReader.CreateHammer()
@@ -294,6 +299,7 @@ func TestCreateHammerManualLoadOverrideOthers(t *testing.T) {
 			Strategy: proxy.ProxyTypeSingle,
 		},
 		SamplingRate: types.DefaultSamplingCount,
+		TestDataConf: make(map[string]types.CsvConf),
 	}
 
 	h, err := jsonReader.CreateHammer()
@@ -440,43 +446,6 @@ func TestCreateHammerCaptureEnvs(t *testing.T) {
 	}
 }
 
-func TestCreateHammerDataCsv(t *testing.T) {
-	t.Parallel()
-	jsonReader, _ := NewConfigReader(readConfigFile("config_testdata/config_data_csv.json"), ConfigTypeJson)
-
-	expectedRandom := true
-
-	h, err := jsonReader.CreateHammer()
-	if err != nil {
-		t.Errorf("TestCreateHammerDataCsv error occurred: %v", err)
-	}
-
-	csvData := h.Scenario.Data["info"]
-
-	if !reflect.DeepEqual(csvData.Random, expectedRandom) {
-		t.Errorf("TestCreateHammerDataCsv got: %t expected: %t", csvData.Random, expectedRandom)
-	}
-
-	expectedRow := map[string]interface{}{
-		"name": "Kenan",
-		"city": "Tokat",
-		"team": "Galatasaray",
-		"payload": map[string]interface{}{
-			"data": map[string]interface{}{
-				"profile": map[string]interface{}{
-					"name": "Kenan",
-				},
-			},
-		},
-		"age": 25,
-	}
-
-	if !reflect.DeepEqual(expectedRow, csvData.Rows[0]) {
-		t.Errorf("TestCreateHammerDataCsv got: %#v expected: %#v", csvData.Rows[0], expectedRow)
-	}
-
-}
-
 func TestCreateHammerInvalidTarget(t *testing.T) {
 	t.Parallel()
 	jsonReader, _ := NewConfigReader(readConfigFile("config_testdata/config_invalid_target.json"), ConfigTypeJson)
@@ -552,6 +521,7 @@ func TestCreateHammerTLSWithOnlyCertPath(t *testing.T) {
 			Strategy: proxy.ProxyTypeSingle,
 		},
 		SamplingRate: types.DefaultSamplingCount,
+		TestDataConf: make(map[string]types.CsvConf),
 	}
 
 	h, err := jsonReader.CreateHammer()
@@ -597,6 +567,7 @@ func TestCreateHammerTLSWithOnlyKeyPath(t *testing.T) {
 			Strategy: proxy.ProxyTypeSingle,
 		},
 		SamplingRate: types.DefaultSamplingCount,
+		TestDataConf: make(map[string]types.CsvConf),
 	}
 
 	h, err := jsonReader.CreateHammer()
@@ -633,6 +604,7 @@ func TestCreateHammerTLSWithWithEmptyPath(t *testing.T) {
 			Strategy: proxy.ProxyTypeSingle,
 		},
 		SamplingRate: types.DefaultSamplingCount,
+		TestDataConf: make(map[string]types.CsvConf),
 	}
 
 	h, err := jsonReader.CreateHammer()

diff --git a/config_examples/assertion/expected_body.json b/config_examples/assertion/expected_body.json
@@ -0,0 +1,28 @@
+[
+    "AED",
+    "ARS",
+    "AUD",
+    "BGN",
+    "BHD",
+    "BRL",
+    "CAD",
+    "CHF",
+    "CNY",
+    "DKK",
+    "DZD",
+    "EUR",
+    "FKP",
+    "INR",
+    "JEP",
+    "JPY",
+    "KES",
+    "KWD",
+    "KZT",
+    "MXN",
+    "NZD",
+    "RUB",
+    "SEK",
+    "SGD",
+    "TRY",
+    "USD"
+]
diff --git a/core/engine.go b/core/engine.go
@@ -30,6 +30,7 @@ import (
 	"go.ddosify.com/ddosify/core/proxy"
 	"go.ddosify.com/ddosify/core/report"
 	"go.ddosify.com/ddosify/core/scenario"
+	"go.ddosify.com/ddosify/core/scenario/testdata"
 	"go.ddosify.com/ddosify/core/types"
 )
 
@@ -95,6 +96,13 @@ func (e *engine) Init() (err error) {
 		return
 	}
 
+	// read test data
+	readData, err := readTestData(e.hammer.TestDataConf)
+	if err != nil {
+		return err
+	}
+	e.hammer.Scenario.Data = readData
+
 	if err = e.scenarioService.Init(e.ctx, e.hammer.Scenario, e.proxyService.GetAll(), e.hammer.Debug); err != nil {
 		return
 	}
@@ -337,3 +345,28 @@ func reverse(s interface{}) {
 		swap(i, j)
 	}
 }
+
+var readTestData = func(testDataConf map[string]types.CsvConf) (map[string]types.CsvData, error) {
+	// Read Data
+	var readData map[string]types.CsvData
+	if len(testDataConf) > 0 {
+		readData = make(map[string]types.CsvData, len(testDataConf))
+	}
+	for k, conf := range testDataConf {
+		var rows []map[string]interface{}
+		var err error
+		rows, err = testdata.ReadCsv(conf)
+		if err != nil {
+			return nil, err
+		}
+		var csvData types.CsvData
+		csvData.Rows = rows
+
+		if conf.Order == "random" {
+			csvData.Random = true
+		}
+		readData[k] = csvData
+	}
+
+	return readData, nil
+}
diff --git a/core/engine_test.go b/core/engine_test.go
@@ -1484,13 +1484,19 @@ func TestLoadRandomInfoFromData(t *testing.T) {
 		t.Errorf("TestLoadRandomInfoFromData error occurred %v", err)
 	}
 
+	originalReadTestData := readTestData
+	readTestData = func(testDataConf map[string]types.CsvConf) (map[string]types.CsvData, error) {
+		return map[string]types.CsvData{"info": csvData}, nil
+	}
+
 	err = e.Init()
 	if err != nil {
 		t.Errorf("TestLoadRandomInfoFromData error occurred %v", err)
 	}
 
 	e.Start()
 
+	readTestData = originalReadTestData
 	if !requestCalled {
 		t.Errorf("TestLoadRandomInfoFromData test server has not been called, url path injection failed")
 	}
@@ -1500,6 +1506,52 @@ func TestLoadRandomInfoFromData(t *testing.T) {
 	}
 }
 
+func TestDataCsv(t *testing.T) {
+	readConfigFile := func(path string) []byte {
+		f, _ := os.Open(path)
+
+		byteValue, _ := ioutil.ReadAll(f)
+		return byteValue
+	}
+
+	jsonReader, _ := config.NewConfigReader(readConfigFile("../config/config_testdata/config_data_csv.json"), config.ConfigTypeJson)
+
+	expectedRandom := true
+
+	h, _ := jsonReader.CreateHammer()
+
+	data, err := readTestData(h.TestDataConf)
+
+	if err != nil {
+		t.Errorf("TestDataCsv error occurred: %v", err)
+	}
+
+	csvData := data["info"]
+
+	if !reflect.DeepEqual(csvData.Random, expectedRandom) {
+		t.Errorf("TestCreateHammerDataCsv got: %t expected: %t", csvData.Random, expectedRandom)
+	}
+
+	expectedRow := map[string]interface{}{
+		"name": "Kenan",
+		"city": "Tokat",
+		"team": "Galatasaray",
+		"payload": map[string]interface{}{
+			"data": map[string]interface{}{
+				"profile": map[string]interface{}{
+					"name": "Kenan",
+				},
+			},
+		},
+		"age": 25,
+	}
+
+	if !reflect.DeepEqual(expectedRow, csvData.Rows[0]) {
+		t.Errorf("TestCreateHammerDataCsv got: %#v expected: %#v", csvData.Rows[0], expectedRow)
+	}
+
+}
+
 // The test creates a web server with Certificate auth,
 // then it spawns an Engine and verifies that the auth was successfully passsed.
 func TestTLSMutualAuth(t *testing.T) {

diff --git a/config/csv.go → core/scenario/testdata/csv.go b/config/csv.go → core/scenario/testdata/csv.go
@@ -1,4 +1,4 @@
-package config
+package testdata
 
 import (
 	"encoding/csv"
@@ -9,16 +9,18 @@ import (
 	"net/url"
 	"os"
 	"strconv"
+
+	"go.ddosify.com/ddosify/core/types"
 )
 
-func validateConf(conf CsvConf) error {
+func validateConf(conf types.CsvConf) error {
 	if !(conf.Order == "random" || conf.Order == "sequential") {
 		return fmt.Errorf("unsupported order %s, should be random|sequential", conf.Order)
 	}
 	return nil
 }
 
-func readCsv(conf CsvConf) ([]map[string]interface{}, error) {
+func ReadCsv(conf types.CsvConf) ([]map[string]interface{}, error) {
 	err := validateConf(conf)
 	if err != nil {
 		return nil, err
-Original file line number
+Diff line change
@@ -0,0 +1,28 @@
+    [
+        "AED",
+        "ARS",
+        "AUD",
+        "BGN",
+        "BHD",
+        "BRL",
+        "CAD",
+        "CHF",
+        "CNY",
+        "DKK",
+        "DZD",
+        "EUR",
+        "FKP",
+        "INR",
+        "JEP",
+        "JPY",
+        "KES",
+        "KWD",
+        "KZT",
+        "MXN",
+        "NZD",
+        "RUB",
+        "SEK",
+        "SGD",
+        "TRY",
+        "USD"
+    ]