Test on All Platforms #51
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test on All Platforms | |
| env: | |
| BRANCH_NAME_RAW: ${{ github.head_ref || github.ref_name }} | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '10 5 * * *' # 6:10 AM CET / 7:10 AM CEST | |
| - cron: '50 11 * * *' # 12:50 PM CET / 1:50 PM CEST | |
| - cron: '30 18 * * *' # 7:30 PM CET / 8:30 PM CEST | |
| jobs: | |
| test: | |
| timeout-minutes: 60 | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| - name: Get installed Playwright version | |
| id: playwright-version | |
| run: echo "::set-output name=version::$(yarn why --json @playwright/test | grep -h 'workspace:.' | jq --raw-output '.children[].locator' | sed -e 's/@playwright\/test@.*://')" | |
| - name: Cache node modules | |
| id: cache-npm | |
| uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-node-modules | |
| with: | |
| path: | | |
| ~/.npm | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-${{ env.cache-name }}- | |
| ${{ runner.os }}-build- | |
| ${{ runner.os }}- | |
| - if: ${{ steps.cache-npm.outputs.cache-hit != 'true' }} | |
| name: List the state of node modules | |
| continue-on-error: true | |
| run: | | |
| cd packages/tests-e2e | |
| npm list | |
| - name: Install Dependencies | |
| run: | | |
| cd packages/tests-e2e | |
| npm install | |
| sudo apt-get update | |
| sudo apt-get install -y libxml2-utils | |
| - uses: actions/cache@v3 | |
| id: playwright-cache | |
| with: | |
| path: '~/.cache/ms-playwright' | |
| key: '${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }}' | |
| restore-keys: | | |
| ${{ runner.os }}-playwright- | |
| - name: Install Playwright's dependencies | |
| if: steps.playwright-cache.outputs.cache-hit != 'true' | |
| run: | | |
| cd packages/tests-e2e | |
| npx playwright install --with-deps | |
| - name: Create a log stream on AWS CloudWatch | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| run: | | |
| TIMESTAMP=$(date +%s000) | |
| echo "LOG_STREAM_NAME=javascript-$TIMESTAMP" >> $GITHUB_ENV | |
| aws logs create-log-stream --log-group-name "test-results-board" --log-stream-name "javascript-$TIMESTAMP" | |
| - name: Run Complete tests for react | |
| run: | | |
| cd packages/tests-e2e | |
| BRANCH_NAME=$(echo $BRANCH_NAME_RAW | tr '/_' '-') | |
| export PLAYWRIGHT_TEST_URL="https://$BRANCH_NAME.react.playground.corbado.io" | |
| set +e | |
| npx playwright test --config=playwright.config.complete.ts | |
| EXIT_CODE=$? | |
| if [ $EXIT_CODE -ne 0 ]; then | |
| echo "COMPLETE_REACT_FAILED=true" >> $GITHUB_ENV | |
| fi | |
| env: | |
| PLAYWRIGHT_NUM_CORES: 4 | |
| PLAYWRIGHT_JWT_TOKEN: ${{ secrets.PLAYWRIGHT_JWT_TOKEN }} | |
| PLAYWRIGHT_GOOGLE_EMAIL: ${{ secrets.PLAYWRIGHT_GOOGLE_EMAIL }} | |
| PLAYWRIGHT_GOOGLE_PASSWORD: ${{ secrets.PLAYWRIGHT_GOOGLE_PASSWORD }} | |
| PLAYWRIGHT_GOOGLE_TOTP_SECRET: ${{ secrets.PLAYWRIGHT_GOOGLE_TOTP_SECRET }} | |
| GITHUB_RUN_ID: ${{ github.run_id }} | |
| SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.SLACK_BOT_USER_OAUTH_TOKEN }} | |
| GITHUB_BRANCH_NAME: ${{ env.BRANCH_NAME_RAW }} | |
| continue-on-error: true | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: playwright-report-complete-react | |
| path: packages/tests-e2e/playwright-report/ | |
| retention-days: 30 | |
| - name: Send logs to AWS CloudWatch | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| run: | | |
| cd packages/tests-e2e | |
| TESTS=$(xmllint --xpath 'string(/testsuites/@tests)' test-results/results.xml) | |
| FAILURES=$(xmllint --xpath 'string(/testsuites/@failures)' test-results/results.xml) | |
| SKIPPED=$(xmllint --xpath 'string(/testsuites/@skipped)' test-results/results.xml) | |
| ERRORS=$(xmllint --xpath 'string(/testsuites/@errors)' test-results/results.xml) | |
| TIME=$(xmllint --xpath 'string(/testsuites/@time)' test-results/results.xml) | |
| PASSED=$((TESTS - FAILURES - ERRORS - SKIPPED)) | |
| FAILED=$((FAILURES + ERRORS)) | |
| TIMESTAMP=$(date +%s000) | |
| LOG_EVENT_JSON="[{\"timestamp\":$TIMESTAMP,\"message\":\"{\\\"application\\\":\\\"complete\\\",\\\"platform\\\":\\\"react\\\",\\\"run_type\\\":\\\"nightly\\\",\\\"execution_time\\\":$TIME,\\\"passed\\\":$PASSED,\\\"failed\\\":$FAILED,\\\"link\\\":\\\"https://github.com/corbado/javascript/actions/runs/${GITHUB_RUN_ID}\\\"}\"}]" | |
| aws logs put-log-events --log-group-name "test-results-board" --log-stream-name "$LOG_STREAM_NAME" --log-events "$LOG_EVENT_JSON" | |
| - name: Run Complete tests for web-js | |
| run: | | |
| cd packages/tests-e2e | |
| BRANCH_NAME=$(echo $BRANCH_NAME_RAW | tr '/_' '-') | |
| export PLAYWRIGHT_TEST_URL="https://$BRANCH_NAME.web-js.playground.corbado.io" | |
| set +e | |
| npx playwright test --config=playwright.config.complete.ts | |
| EXIT_CODE=$? | |
| if [ $EXIT_CODE -ne 0 ]; then | |
| echo "COMPLETE_WEBJS_FAILED=true" >> $GITHUB_ENV | |
| fi | |
| env: | |
| PLAYWRIGHT_NUM_CORES: 4 | |
| PLAYWRIGHT_JWT_TOKEN: ${{ secrets.PLAYWRIGHT_JWT_TOKEN }} | |
| PLAYWRIGHT_GOOGLE_EMAIL: ${{ secrets.PLAYWRIGHT_GOOGLE_EMAIL }} | |
| PLAYWRIGHT_GOOGLE_PASSWORD: ${{ secrets.PLAYWRIGHT_GOOGLE_PASSWORD }} | |
| PLAYWRIGHT_GOOGLE_TOTP_SECRET: ${{ secrets.PLAYWRIGHT_GOOGLE_TOTP_SECRET }} | |
| GITHUB_RUN_ID: ${{ github.run_id }} | |
| SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.SLACK_BOT_USER_OAUTH_TOKEN }} | |
| GITHUB_BRANCH_NAME: ${{ env.BRANCH_NAME_RAW }} | |
| continue-on-error: true | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: playwright-report-complete-web-js | |
| path: packages/tests-e2e/playwright-report/ | |
| retention-days: 30 | |
| - name: Send logs to AWS CloudWatch | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| run: | | |
| cd packages/tests-e2e | |
| TESTS=$(xmllint --xpath 'string(/testsuites/@tests)' test-results/results.xml) | |
| FAILURES=$(xmllint --xpath 'string(/testsuites/@failures)' test-results/results.xml) | |
| SKIPPED=$(xmllint --xpath 'string(/testsuites/@skipped)' test-results/results.xml) | |
| ERRORS=$(xmllint --xpath 'string(/testsuites/@errors)' test-results/results.xml) | |
| TIME=$(xmllint --xpath 'string(/testsuites/@time)' test-results/results.xml) | |
| PASSED=$((TESTS - FAILURES - ERRORS - SKIPPED)) | |
| FAILED=$((FAILURES + ERRORS)) | |
| TIMESTAMP=$(date +%s000) | |
| LOG_EVENT_JSON="[{\"timestamp\":$TIMESTAMP,\"message\":\"{\\\"application\\\":\\\"complete\\\",\\\"platform\\\":\\\"web-js\\\",\\\"run_type\\\":\\\"nightly\\\",\\\"execution_time\\\":$TIME,\\\"passed\\\":$PASSED,\\\"failed\\\":$FAILED,\\\"link\\\":\\\"https://github.com/corbado/javascript/actions/runs/${GITHUB_RUN_ID}\\\"}\"}]" | |
| aws logs put-log-events --log-group-name "test-results-board" --log-stream-name "$LOG_STREAM_NAME" --log-events "$LOG_EVENT_JSON" | |
| - name: Run Complete tests for web-js-script | |
| run: | | |
| cd packages/tests-e2e | |
| BRANCH_NAME=$(echo $BRANCH_NAME_RAW | tr '/_' '-') | |
| export PLAYWRIGHT_TEST_URL="https://$BRANCH_NAME.web-js-script.playground.corbado.io" | |
| set +e | |
| npx playwright test --config=playwright.config.complete.ts | |
| EXIT_CODE=$? | |
| if [ $EXIT_CODE -ne 0 ]; then | |
| echo "COMPLETE_WEBJSSCRIPT_FAILED=true" >> $GITHUB_ENV | |
| fi | |
| env: | |
| PLAYWRIGHT_NUM_CORES: 4 | |
| PLAYWRIGHT_JWT_TOKEN: ${{ secrets.PLAYWRIGHT_JWT_TOKEN }} | |
| PLAYWRIGHT_GOOGLE_EMAIL: ${{ secrets.PLAYWRIGHT_GOOGLE_EMAIL }} | |
| PLAYWRIGHT_GOOGLE_PASSWORD: ${{ secrets.PLAYWRIGHT_GOOGLE_PASSWORD }} | |
| PLAYWRIGHT_GOOGLE_TOTP_SECRET: ${{ secrets.PLAYWRIGHT_GOOGLE_TOTP_SECRET }} | |
| GITHUB_RUN_ID: ${{ github.run_id }} | |
| SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.SLACK_BOT_USER_OAUTH_TOKEN }} | |
| GITHUB_BRANCH_NAME: ${{ env.BRANCH_NAME_RAW }} | |
| continue-on-error: true | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: playwright-report-complete-web-js-script | |
| path: packages/tests-e2e/playwright-report/ | |
| retention-days: 30 | |
| - name: Send logs to AWS CloudWatch | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| run: | | |
| cd packages/tests-e2e | |
| TESTS=$(xmllint --xpath 'string(/testsuites/@tests)' test-results/results.xml) | |
| FAILURES=$(xmllint --xpath 'string(/testsuites/@failures)' test-results/results.xml) | |
| SKIPPED=$(xmllint --xpath 'string(/testsuites/@skipped)' test-results/results.xml) | |
| ERRORS=$(xmllint --xpath 'string(/testsuites/@errors)' test-results/results.xml) | |
| TIME=$(xmllint --xpath 'string(/testsuites/@time)' test-results/results.xml) | |
| PASSED=$((TESTS - FAILURES - ERRORS - SKIPPED)) | |
| FAILED=$((FAILURES + ERRORS)) | |
| TIMESTAMP=$(date +%s000) | |
| LOG_EVENT_JSON="[{\"timestamp\":$TIMESTAMP,\"message\":\"{\\\"application\\\":\\\"complete\\\",\\\"platform\\\":\\\"web-js-script\\\",\\\"run_type\\\":\\\"nightly\\\",\\\"execution_time\\\":$TIME,\\\"passed\\\":$PASSED,\\\"failed\\\":$FAILED,\\\"link\\\":\\\"https://github.com/corbado/javascript/actions/runs/${GITHUB_RUN_ID}\\\"}\"}]" | |
| aws logs put-log-events --log-group-name "test-results-board" --log-stream-name "$LOG_STREAM_NAME" --log-events "$LOG_EVENT_JSON" | |
| - name: Run Connect tests for react | |
| run: | | |
| cd packages/tests-e2e | |
| BRANCH_NAME=$(echo $BRANCH_NAME_RAW | tr '/_' '-') | |
| export PLAYWRIGHT_TEST_URL="https://$BRANCH_NAME.connect-next.playground.corbado.io" | |
| set +e | |
| npx playwright test --config=playwright.config.connect.ts | |
| EXIT_CODE=$? | |
| if [ $EXIT_CODE -ne 0 ]; then | |
| echo "CONNECT_REACT_FAILED=true" >> $GITHUB_ENV | |
| fi | |
| env: | |
| PLAYWRIGHT_NUM_CORES: 4 | |
| BACKEND_API_BASIC_AUTH: ${{ secrets.BACKEND_API_BASIC_AUTH }} | |
| PLAYWRIGHT_CONNECT_PROJECT_ID: ${{ secrets.PLAYWRIGHT_CONNECT_PROJECT_ID }} | |
| PLAYWRIGHT_NGROK_AUTH_TOKEN: ${{ secrets.PLAYWRIGHT_NGROK_AUTH_TOKEN }} | |
| GITHUB_RUN_ID: ${{ github.run_id }} | |
| SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.SLACK_BOT_USER_OAUTH_TOKEN }} | |
| GITHUB_BRANCH_NAME: ${{ env.BRANCH_NAME_RAW }} | |
| continue-on-error: true | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: playwright-report-connect-react | |
| path: packages/tests-e2e/playwright-report/ | |
| retention-days: 30 | |
| - name: Send logs to AWS CloudWatch | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| run: | | |
| cd packages/tests-e2e | |
| TESTS=$(xmllint --xpath 'string(/testsuites/@tests)' test-results/results.xml) | |
| FAILURES=$(xmllint --xpath 'string(/testsuites/@failures)' test-results/results.xml) | |
| SKIPPED=$(xmllint --xpath 'string(/testsuites/@skipped)' test-results/results.xml) | |
| ERRORS=$(xmllint --xpath 'string(/testsuites/@errors)' test-results/results.xml) | |
| TIME=$(xmllint --xpath 'string(/testsuites/@time)' test-results/results.xml) | |
| PASSED=$((TESTS - FAILURES - ERRORS - SKIPPED)) | |
| FAILED=$((FAILURES + ERRORS)) | |
| TIMESTAMP=$(date +%s000) | |
| LOG_EVENT_JSON="[{\"timestamp\":$TIMESTAMP,\"message\":\"{\\\"application\\\":\\\"connect\\\",\\\"platform\\\":\\\"react\\\",\\\"run_type\\\":\\\"nightly\\\",\\\"execution_time\\\":$TIME,\\\"passed\\\":$PASSED,\\\"failed\\\":$FAILED,\\\"link\\\":\\\"https://github.com/corbado/javascript/actions/runs/${GITHUB_RUN_ID}\\\"}\"}]" | |
| aws logs put-log-events --log-group-name "test-results-board" --log-stream-name "$LOG_STREAM_NAME" --log-events "$LOG_EVENT_JSON" | |
| - name: Aggregate results | |
| run: | | |
| FAILED_STEPS="" | |
| if [ "${COMPLETE_REACT_FAILED}" = "true" ]; then | |
| FAILED_STEPS="${FAILED_STEPS} Complete-React" | |
| fi | |
| if [ "${COMPLETE_WEBJS_FAILED}" = "true" ]; then | |
| FAILED_STEPS="${FAILED_STEPS} Complete-WebJs" | |
| fi | |
| if [ "${COMPLETE_WEBJSSCRIPT_FAILED}" = "true" ]; then | |
| FAILED_STEPS="${FAILED_STEPS} Complete-WebJsScript" | |
| fi | |
| if [ "${CONNECT_REACT_FAILED}" = "true" ]; then | |
| FAILED_STEPS="${FAILED_STEPS} Connect-React" | |
| fi | |
| if [ -n "$FAILED_STEPS" ]; then | |
| echo "The following test steps have failed: $FAILED_STEPS" | |
| exit 1 | |
| else | |
| echo "All tests passed." | |
| fi |