Merge branch 'improvement/ZENKO-3661-setup-dependency-mocks-for-repli…

…cation-tests' into q/2.1

eve/main.yml

@@ -39,6 +39,29 @@ models:
       E2E_IMAGE_TAG: '%(prop:commit_short_revision)s'
       VAULT_TEST_IMAGE_NAME: '%(prop:vault_test_image_name)s'
       VAULT_TEST_IMAGE_TAG: '%(prop:vault_test_image_tag)s'
+  - env: &secrets-env
+      AWS_BACKEND_SOURCE_LOCATION: awsbackend
+      AWS_BACKEND_DESTINATION_LOCATION: awsbackendmismatch
+      GCP_BACKEND_DESTINATION_LOCATION: gcpbackendmismatch
+      AZURE_BACKEND_DESTINATION_LOCATION: azurebackendmismatch
+      LOCATION_QUOTA_BACKEND: quotabackend
+      AWS_BUCKET_NAME: ci-zenko-aws-target-bucket
+      AWS_CRR_BUCKET_NAME: ci-zenko-aws-crr-target-bucket
+      AZURE_CRR_BUCKET_NAME: ci-zenko-azure-crr-target-bucket
+      GCP_CRR_BUCKET_NAME: ci-zenko-gcp-crr-target-bucket
+      GCP_CRR_MPU_BUCKET_NAME: ci-zenko-gcp-crr-mpu-bucket
+      GCP_ACCESS_KEY: '%(secret:aws_gcp_backend_access_key)s'
+      GCP_SECRET_KEY: '%(secret:aws_gcp_backend_secret_key)s'
+      GCP_BACKEND_SERVICE_KEY: '%(secret:gcp_backend_service_key)s'
+      GCP_BACKEND_SERVICE_EMAIL: '%(secret:gcp_backend_service_email)s'
+  - env: &mock-env
+      AZURE_ACCOUNT_NAME: devstoreaccount1
+      AZURE_BACKEND_ENDPOINT: http://azure-mock/devstoreaccount1
+      AZURE_SECRET_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+      AWS_ENDPOINT: http://aws-mock.zenko.local
+      AWS_ACCESS_KEY: accessKey1
+      AWS_SECRET_KEY: verySecretKey1
+      VERIFY_CERTIFICATES: "false"
   - Git: &git_pull
       name: git pull
       repourl: "%(prop:git_reference)s"
@@ -221,6 +244,15 @@ stages:
     - Git: *git_pull
     - ShellCommand: *wait_docker_daemon
     - ShellCommand: *private_registry_login
+    - ShellCommand:
+        name: Generate end2end config yaml
+        env:
+          <<: [*kind-env, *e2e-env, *http-env, *mock-env, *secrets-env]
+        workdir: build/tests/zenko_tests
+        command: |
+          envsubst < 'e2e-config.yaml.template' > 'e2e-config.yaml'
+          echo 'Generated e2e-config.yaml file'
+        haltOnFailure: true
     - ShellCommand: &build-e2e-image
         name: Build end2end test image
         env: *e2e-env
@@ -313,6 +345,13 @@ stages:
         command: bash scripts/keycloak-helper.sh add-user default
         workdir: build/eve/workers/end2end
         haltOnFailure: true
+    - ShellCommand: &install_mocks
+        name: Start external service mocks
+        env:
+          <<: [*kind-env, *e2e-env, *http-env]
+        command: bash scripts/install-mocks.sh "default"
+        workdir: build/eve/workers/end2end
+        haltOnFailure: true
     - ShellCommand: &configure_end2end_tests
         name: End-to-end configuration
         env:
@@ -334,6 +373,13 @@ stages:
         command: bash scripts/run-e2e-test.sh "end2end" ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG} "smoke" "default"
         workdir: build/eve/workers/end2end
         haltOnFailure: true
+    - ShellCommand: &run_backbeat_end2end_tests
+        name: Run backbeat end to end tests
+        env:
+          <<: [*kind-env, *e2e-env, *http-env, *mock-env, *secrets-env]
+        command: bash scripts/run-e2e-test.sh "end2end" ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG} "backbeat" "default"
+        workdir: build/eve/workers/end2end
+        haltOnFailure: true
     - ShellCommand: &archive_artifacts
         name: archive artifact logs and data
         command: >-
@@ -384,6 +430,10 @@ stages:
         <<: *setup_keycloak_user
         env:
           <<: [*kind-env, *https-env]
+    - ShellCommand:
+        <<: *install_mocks
+        env:
+          <<: [*kind-env, *e2e-env, *http-env]
     - ShellCommand:
         <<: *configure_end2end_tests
         env:

eve/workers/end2end/scripts/install-mocks.sh

@@ -0,0 +1,24 @@
+#! /bin/sh
+
+set -exu
+
+NAMESPACE=${1:-default}
+
+kubectl create \
+    -f ../mocks/azure-mock.yaml \
+    -f ../mocks/aws-mock.yaml \
+    --namespace ${NAMESPACE} && \
+kubectl create \
+    configmap aws-mock \
+    --from-file=../mocks/aws/mock-metadata.tar.gz \
+    --namespace ${NAMESPACE}
+
+kubectl wait \
+    --for=condition=Ready \
+    --timeout=10m \
+    pod -l component=mock \
+    --namespace ${NAMESPACE}
+
+kubectl get \
+    pod -l component=mock \
+    --namespace ${NAMESPACE}

eve/workers/end2end/scripts/patch-coredns.sh

@@ -11,6 +11,12 @@ corefile="
         lameduck 5s
     }
     ready
+    rewrite name exact ci-zenko-aws-crr-target-bucket.aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact ci-zenko-aws-target-bucket.aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact aws-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact azure-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact blob.azure-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact devstoreaccount1.blob.azure-mock.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name exact keycloak.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name exact ui.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name exact management.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local

eve/workers/end2end/scripts/run-e2e-test.sh

@@ -50,6 +50,23 @@ run_e2e_test() {
         --env=CYPRESS_KEYCLOAK_CLIENT_ID=${OIDC_CLIENT_ID} \
         --env=CYPRESS_KEYCLOAK_REALM=${OIDC_REALM} \
         --env=UI_ENDPOINT=${UI_ENDPOINT} \
+        --env=AWS_BACKEND_SOURCE_LOCATION=awsbackend \
+        --env=AWS_BACKEND_DESTINATION_LOCATION=awsbackendmismatch \
+        --env=GCP_BACKEND_DESTINATION_LOCATION=gcpbackendmismatch \
+        --env=AZURE_BACKEND_DESTINATION_LOCATION=azurebackendmismatch \
+        --env=LOCATION_QUOTA_BACKEND=quotabackend \
+        --env=AWS_BUCKET_NAME=ci-zenko-aws-target-bucket \
+        --env=AWS_CRR_BUCKET_NAME=ci-zenko-aws-crr-target-bucket \
+        --env=AZURE_CRR_BUCKET_NAME=ci-zenko-azure-crr-target-bucket \
+        --env=GCP_CRR_BUCKET_NAME=ci-zenko-gcp-crr-target-bucket \
+        --env=GCP_CRR_MPU_BUCKET_NAME=ci-zenko-gcp-crr-mpu-bucket \
+        --env=AZURE_ACCOUNT_NAME=devstoreaccount1 \
+        --env=AZURE_BACKEND_ENDPOINT=http://azure-mock/devstoreaccount1 \
+        --env=AZURE_SECRET_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== \
+        --env=AWS_ENDPOINT=http://aws-mock.zenko.local \
+        --env=AWS_ACCESS_KEY=accessKey1 \
+        --env=AWS_SECRET_KEY=verySecretKey1 \
+        --env=VERIFY_CERTIFICATES=false \
         --command -- sh -c "${2}"
 }
 
@@ -60,4 +77,6 @@ elif [ "$STAGE" = "debug" ]; then
    run_e2e_test '-ti' 'bash'
 elif [ "$STAGE" = "smoke" ]; then
    run_e2e_test '' 'cd node_tests && npm run test_smoke'
+elif [ "$STAGE" = "backbeat" ]; then
+   run_e2e_test '' 'cd node_tests && npm run test_aws_crr'
 fi

eve/workers/mocks/aws-mock.yaml

@@ -11,12 +11,36 @@ spec:
       port: 80
       targetPort: http
 ---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: aws-mock
+spec:
+  rules:
+  - host: 'aws-mock.zenko.local'
+    http:
+      paths:
+      - backend:
+          serviceName: aws-mock
+          servicePort: http
+        path: /
+        pathType: Prefix
+  - host: '*.aws-mock.zenko.local'
+    http:
+      paths:
+      - backend:
+          serviceName: aws-mock
+          servicePort: http
+        path: /
+        pathType: Prefix
+---
 apiVersion: v1
 kind: Pod
 metadata:
   name: aws-mock-pod
   labels:
     name: aws-mock
+    component: mock
 spec:
   initContainers:
   - image: zenko/cloudserver:latest
@@ -39,7 +63,7 @@ spec:
     - name: REMOTE_MANAGEMENT_DISABLE
       value: "1"
     - name: ENDPOINT
-      value: "aws-mock"
+      value: "aws-mock.zenko.local"
     - name: S3BACKEND
       value: file
     ports:
@@ -60,4 +84,4 @@ spec:
     emptyDir: {}
   - name: configmap
     configMap:
-      name: aws-mock  
+      name: aws-mock

eve/workers/mocks/azure-mock.yaml

@@ -17,6 +17,7 @@ metadata:
   name: azure-mock-pod
   labels:
     name: azure-mock
+    component: mock
 spec:
   hostname: devstoreaccount1
   subdomain: azure-mock
@@ -33,3 +34,42 @@ spec:
       requests:
         cpu: 1
         memory: 2Gi
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: azure-mock
+spec:
+  rules:
+  - host: '*.azure-mock.zenko.local'
+    http:
+      paths:
+      - backend:
+          serviceName: azure-mock
+          servicePort: http
+        path: /
+        pathType: Prefix
+  - host: 'azure-mock.zenko.local'
+    http:
+      paths:
+      - backend:
+          serviceName: azure-mock
+          servicePort: http
+        path: /
+        pathType: Prefix
+  - host: '*.blob.azure-mock.zenko.local'
+    http:
+      paths:
+      - backend:
+          serviceName: azure-mock
+          servicePort: http
+        path: /
+        pathType: Prefix
+  - host: 'blob.azure-mock.zenko.local'
+    http:
+      paths:
+      - backend:
+          serviceName: azure-mock
+          servicePort: http
+        path: /
+        pathType: Prefix

tests/zenko_tests/configuration.py

@@ -124,7 +124,8 @@ def main():
                                     namespace=NAMESPACE)
 
         for endpoint in e2e_config["endpoints"]:
-            endpoints.create_endpoint(client, UUID, endpoint)
+            endpoints.create_endpoint(client, UUID, endpoint["hostname"],
+                                      endpoint["locationName"])
 
         for location in e2e_config["locations"]:
             locations.create_location(client, UUID, location)

tests/zenko_tests/e2e-config.yaml.template

@@ -0,0 +1,25 @@
+accounts:
+  - "zenko"
+endpoints: []
+locations:
+  - name: "${AWS_BACKEND_DESTINATION_LOCATION}"
+    locationType: "location-aws-s3-v1"
+    legacyAwsBehavior: yes
+    details:
+      bucketName: "${AWS_CRR_BUCKET_NAME}"
+      endpoint: "${AWS_ENDPOINT}"
+      accessKey: "${AWS_ACCESS_KEY}"
+      secretKey: "${AWS_SECRET_KEY}"
+      bucketMatch: no
+  - name: "${AWS_BACKEND_SOURCE_LOCATION}"
+    locationType: "location-aws-s3-v1"
+    details:
+      bucketName: "${AWS_BUCKET_NAME}"
+      endpoint: "${AWS_ENDPOINT}"
+      accessKey: "${AWS_ACCESS_KEY}"
+      secretKey: "${AWS_SECRET_KEY}"
+      bucketMatch: yes
+workflows:
+  replication: []
+  lifecycle: []
+  ingestion: []

tests/zenko_tests/e2e_config/endpoints.py

@@ -4,5 +4,27 @@
 
 _log = logging.getLogger("end2end configuration")
 
-def create_endpoint(client, UUID, endpoint):
-    pass
+def create_endpoint(client, uuid, host, location):
+    """
+    Creates an endpoint for a location
+    :param client: swagger client
+    :param uuid: zenko instance uuid
+    :param host: hostname
+    :param location: location name
+    """
+    try:
+        Endpoint_V1 = client.get_model('endpoint-v1')
+        ep = Endpoint_V1(hostname=host, locationName=location)
+
+        res = (
+            client.ui_facing
+            .createConfigurationOverlayEndpoint(endpoint=ep, uuid=uuid)
+            .response()
+            .result
+        )
+
+        _log.info(res)
+        _log.info("endpoint created")
+    except Exception as e:
+        raise Exception(
+            "Failed to create endpoint for location '%s': %s" % (location, e))

tests/zenko_tests/e2e_config/locations.py

@@ -4,5 +4,29 @@
 
 _log = logging.getLogger("end2end configuration")
 
-def create_locations(client, UUID, location):
-    pass
+def create_location(client, uuid, location):
+    """
+    Creates a location
+    :param client: swagger client
+    :param uuid: zenko instance uuid
+    :param location: location details
+    """
+    try:
+        Location_V1 = client.get_model('location-v1')
+        if "bootstrapList" not in location["details"]:
+            location["details"]["bootstrapList"] = []
+        loc = Location_V1(name=location["name"],
+                            locationType=location["locationType"],
+                            details=location["details"])
+
+        res = (
+            client.ui_facing
+            .createConfigurationOverlayLocation(location=loc, uuid=uuid)
+            .response()
+            .result
+        )
+
+        _log.info("location created")
+    except Exception as e:
+        raise Exception(
+            "Failed to create location '%s': %s" % (location["name"], e))

tests/zenko_tests/e2e_config/schema.py

@@ -12,7 +12,7 @@
         items:
             type: object
             properties:
-                host:
+                hostname:
                     type: string
                 locationName:
                     type: string
@@ -25,6 +25,17 @@
                     type: string
                 locationType:
                     type: string
+                details:
+                    type: object
+                    properties:
+                        bucketName:
+                            type: string
+                        endpoint:
+                            type: string
+                        accessKey:
+                            type: string
+                        secretKey:
+                            type: string
     workflows:
         type: object
         properties:
@@ -41,4 +52,3 @@
                 items:
                     type: object
 """
-