Merge pull request #8 from kube-tarian/test-cases

Test case addtions

Makefile

@@ -1,50 +1,52 @@
 CLIENT_APP_NAME := container-bridge-client
 AGENT_APP_NAME := container-bridge-agent
-BUILD := 0.0.1
+BUILD := 0.1.1
 
-TOOLS_DIR		:= .tools/
-GOLANGCI_LINT	:= ${TOOLS_DIR}github.com/golangci/golangci-lint/cmd/[email protected]
-OAPI_CODEGEN	:= ${TOOLS_DIR}github.com/deepmap/oapi-codegen/cmd/oapi-codegen@latest
-GOIMPORTS		:= ${TOOLS_DIR}mvdan.cc/gofumpt/[email protected]
+OPEN_API_CODEGEN := github.com/deepmap/oapi-codegen/cmd/oapi-codegen@latest
 
-${OAPI_CODEGEN} ${GOLANGCI_LINT} ${GOIMPORTS}:
+${OPEN_API_CODEGEN}:
 	$(eval TOOL=$(@:%=%))
 	@echo Installing ${TOOL}...
-	go install $(TOOL:${TOOLS_DIR}%=%)
-	@mkdir -p $(dir ${TOOL})
-	@cp ${GOBIN}/$(firstword $(subst @, ,$(notdir ${TOOL}))) ${TOOL}
+	go install $(@:%=%)
 
-tools: ${OAPI_CODEGEN} ${GOLANGCI_LINT} ${GOIMPORTS}
+tools: ${OPEN_API_CODEGEN}
 
-OAPI_DIR = ./api
+OPEN_API_DIR = ./api
 
 oapi-gen: tools oapi-gen-agent oapi-gen-client
 
 oapi-gen-agent:
 	$(eval APP_NAME=agent)
 	@echo Generating server for ${APP_NAME}
-	@mkdir -p ${APP_NAME}/${OAPI_DIR}
-	${OAPI_CODEGEN} -config ./${APP_NAME}/cfg.yaml ./${APP_NAME}/openapi.yaml
+	@mkdir -p ${APP_NAME}/${OPEN_API_DIR}
+	${GOBIN}/oapi-codegen -config ./${APP_NAME}/cfg.yaml ./${APP_NAME}/openapi.yaml
 
 oapi-gen-client:
 	$(eval APP_NAME=client)
 	@echo Generating server for ${APP_NAME}
-	@mkdir -p ${APP_NAME}/${OAPI_DIR}
-	${OAPI_CODEGEN} -config ./${APP_NAME}/cfg.yaml ./${APP_NAME}/openapi.yaml
+	@mkdir -p ${APP_NAME}/${OPEN_API_DIR}
+	${GOBIN}/oapi-codegen -config ./${APP_NAME}/cfg.yaml ./${APP_NAME}/openapi.yaml
 
-start-docker-compose:
+start-docker-compose-test:
 	docker compose up -d --no-recreate
+	go test -timeout 120s -run ^Test* github.com/kube-tarian/container-bridge/integration_tests -v
 
-stop-docker-compose:
+stop-docker-compose-test:
 	docker compose down -v
 
 build:
-	go mod vendor
-	go build -o build/client client/main.go
-	go build -o build/agent agent/main.go
+	go mod download
+	CGO_ENABLED=0 go build -o build/client client/main.go
+	CGO_ENABLED=0 go build -o build/agent agent/main.go
 
 clean:
 	rm -rf build
 docker-build:
-	docker build -f Dockerfile.client -t ${CLIENT_APP_NAME}:${BUILD} .
-	docker build -f Dockerfile.agent -t ${AGENT_APP_NAME}:${BUILD} .
+	docker build -f dockerfiles/client/Dockerfile -t ${CLIENT_APP_NAME}:${BUILD} .
+	docker build -f dockerfiles/agent/Dockerfile -t ${AGENT_APP_NAME}:${BUILD} .
+
+start-manual-test:
+	docker compose -f ./docker-compose_manual_test.yaml up -d --no-recreate
+
+stop-manual-test:
+	docker compose -f ./docker-compose_manual_test.yaml down -v

Notes.md

@@ -1,16 +1,26 @@
 # Container brdige
 
-## Start docker compose
+## Chart and docker versioning
+
+The following files to be updated for chart and docker tag versionsing:
+- Makefile -> BUILD parameter
+- charts/agent/Chart.yaml
+- charts/client/Chart.yaml
+- .github/workflows/agent-docker-image.yaml
+- .github/workflows/client-docker-image.yaml
+
+Currently docker images are tagged latest always. Once stabilized versioning can be added to workflow with same version as chart by modifying above 3 files.
+
+## Start docker compose manual test
 
 ```
-export GITLAB_HOME=./gitlab-repo-server
-docker compose up -d
+docker compose -f ./docker-compose_manual_test.yaml up -d
 ```
 
 ## Stop docker compose
 
 ```
-docker compose down -v
+docker compose -f ./docker-compose_manual_test.yaml down -v
 ```
 
 ## Docker registry configuration
@@ -93,4 +103,67 @@ So have to tag with localhost:5001 prefix path
 
 - Add data source for clickhouse
 
-- 
+# local environment with cpu&mem
+
+## Docker
+
+Reference: https://docs.docker.com/config/containers/resource_constraints/
+
+```
+docker run -it --cpus="0.5" --memory=256m container-bridge-agent:0.1.1
+docker run -it --cpus="0.5" --memory=256m container-bridge-client:0.1.1
+```
+
+## Docker compose
+
+Reference: https://docs.docker.com/compose/compose-file/compose-file-v3/#resources
+
+```
+For example:
+version: "3.9"
+services:
+  agent:
+    image: container-bridge-agent:0.1.1
+    deploy:
+      resources:
+        limits:
+          cpus: '0.50'
+          memory: 256M
+        reservations:
+          cpus: '0.25'
+          memory: 256M
+```
+
+# Example docker event payload
+
+```
+{
+   "events": [
+      {
+         "id": "d539f3c5-5734-47f0-a2a8-d27de5f9edb1",
+         "timestamp": "2022-09-21T19:00:49.658010356Z",
+         "action": "push",
+         "target": {
+            "mediaType": "application/octet-stream",
+            "size": 29136663,           
+            "digest": "sha256:12f42424f10d587d02674c8a0dab1c08d3fd81ab6bac5b7f5e3799215c6c52e6",
+            "length": 29136663,         
+            "repository": "ubuntu",     
+            "url": "http://localhost:5001/v2/ubuntu/blobs/sha256:12f42424f10d587d02674c8a0dab1c08d3fd81ab6bac5b7f5e3799215c6c52e6"
+         },
+         "request": {
+            "id": "3784878a-5075-4ea5-a51b-5288ebc54c9c",
+            "addr": "172.22.0.1:36222",
+            "host": "localhost:5001",
+            "method": "PUT", 
+            "useragent": "containers/5.16.0 (github.com/containers/image)"
+         },
+         "actor": {},
+         "source": {
+            "addr": "5b458294602c:5000",
+            "instanceID": "da4bdd5f-3f85-45c6-a37d-bc97a82fbd39"
+         }
+      }
+   ]
+}
+```

agent/openapi.yaml

@@ -28,7 +28,7 @@ paths:
         '200':
           description: OK
 
-  /localregistry/event/docker:
+  /event/docker:
     post:
       tags:
         - public

agent/pkg/handler/docker_event_api_handler.go

@@ -6,7 +6,7 @@ import (
 	"net/http"
 )
 
-func (ah *APIHandler) PostLocalregistryEventDocker(w http.ResponseWriter, r *http.Request) {
+func (ah *APIHandler) PostEventDocker(w http.ResponseWriter, r *http.Request) {
 	event, err := io.ReadAll(r.Body)
 	if err != nil {
 		log.Printf("Event body read failed: %v", err)

charts/agent/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.1.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

charts/client/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.1.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

client/pkg/application/application.go

@@ -20,7 +20,7 @@ type Application struct {
 	apiServer  *handler.APIHandler
 	httpServer *http.Server
 	conn       *clients.NATSContext
-	dbClient   *clickhouse.DBClient
+	dbClient   clickhouse.DBInterface
 }
 
 func New() *Application {

client/pkg/clickhouse/db_client.go

@@ -16,7 +16,13 @@ type DBClient struct {
 	conf *config.Config
 }
 
-func NewDBClient(conf *config.Config) (*DBClient, error) {
+type DBInterface interface {
+	InsertEvent(event string)
+	FetchEvents() []map[string]interface{}
+	Close()
+}
+
+func NewDBClient(conf *config.Config) (DBInterface, error) {
 	log.Println("Create DB if not exists")
 	conn, err := clickhouse.Open(&clickhouse.Options{
 		Addr: []string{conf.DBAddress},
@@ -86,17 +92,41 @@ func NewDBClient(conf *config.Config) (*DBClient, error) {
 		return nil, err
 	}
 
-	return &DBClient{conn: conn}, nil
+	return &DBClient{conn: conn, conf: conf}, nil
 }
 
-func (c *DBClient) InsertEvent(metrics string) {
-	log.Printf("Inserting event: %v", metrics)
-	insertStmt := fmt.Sprintf("INSERT INTO container_bridge FORMAT JSONAsObject %v", metrics)
+func (c *DBClient) InsertEvent(event string) {
+	log.Printf("Inserting event: %v", event)
+	insertStmt := fmt.Sprintf("INSERT INTO container_bridge FORMAT JSONAsObject %v", event)
 	if err := c.conn.Exec(context.Background(), insertStmt); err != nil {
 		log.Printf("Insert failed, %v", err)
 	}
 }
 
+func (c *DBClient) FetchEvents() []map[string]interface{} {
+	log.Printf("Fetching events")
+	events := []map[string]interface{}{}
+	insertStmt := "select event from container_bridge;"
+	rows, err := c.conn.Query(context.Background(), insertStmt)
+	if err != nil {
+		log.Printf("Insert failed, %v", err)
+	}
+	for rows.Next() {
+		event := map[string]interface{}{}
+		if err := rows.Scan(&event); err != nil {
+			log.Printf("Rows scan failed: %v", err)
+			return events
+		}
+		fmt.Printf("row: event=%v\n", event)
+		events = append(events, event)
+	}
+	rows.Close()
+	if rows.Err() != nil {
+		log.Printf("Fetching rows failed: %v", rows.Err())
+	}
+	return events
+}
+
 func (c *DBClient) Close() {
 	_ = c.conn.Close()
 }

client/pkg/clients/nats_client.go

@@ -23,10 +23,10 @@ type NATSContext struct {
 	conf     *config.Config
 	conn     *nats.Conn
 	stream   nats.JetStreamContext
-	dbClient *clickhouse.DBClient
+	dbClient clickhouse.DBInterface
 }
 
-func NewNATSContext(conf *config.Config, dbClient *clickhouse.DBClient) (*NATSContext, error) {
+func NewNATSContext(conf *config.Config, dbClient clickhouse.DBInterface) (*NATSContext, error) {
 	log.Println("Waiting before connecting to NATS at:", conf.NatsAddress)
 	time.Sleep(1 * time.Second)
 
@@ -72,7 +72,7 @@ func (n *NATSContext) Close() {
 	n.dbClient.Close()
 }
 
-func (n *NATSContext) Subscribe(subject string, consumer string, conn *clickhouse.DBClient) {
+func (n *NATSContext) Subscribe(subject string, consumer string, conn clickhouse.DBInterface) {
 	n.stream.Subscribe(subject, func(msg *nats.Msg) {
 		type events struct {
 			Events []json.RawMessage `json:"events"`
@@ -81,6 +81,7 @@ func (n *NATSContext) Subscribe(subject string, consumer string, conn *clickhous
 		eventDocker := &events{}
 		err := json.Unmarshal(msg.Data, &eventDocker)
 		if err == nil {
+			log.Println(eventDocker)
 			msg.Ack()
 			repoName := msg.Header.Get("REPO_NAME")
 			type newEvent struct {