diff --git a/.github/scripts/copy-s3.sh b/.github/scripts/copy-s3.sh index 280b330b2..7b61a9baa 100644 --- a/.github/scripts/copy-s3.sh +++ b/.github/scripts/copy-s3.sh @@ -11,7 +11,7 @@ export AWS_REQUEST_CHECKSUM_CALCULATION=when_required export AWS_RESPONSE_CHECKSUM_VALIDATION=when_required # Upload the file to S3 -aws s3 cp "$source_path" "$target_path" --endpoint-url="$endpoint_url" --debug +aws s3 cp "$source_path" "$target_path" --endpoint-url="$endpoint_url" # Check if the upload was successful if [ $? -eq 0 ]; then diff --git a/.github/scripts/delete-s3.sh b/.github/scripts/delete-s3.sh new file mode 100644 index 000000000..e189e6850 --- /dev/null +++ b/.github/scripts/delete-s3.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -uxo pipefail + +# Variables +file_path=$1 +endpoint_url=$2 + +export AWS_REQUEST_CHECKSUM_CALCULATION=when_required +export AWS_RESPONSE_CHECKSUM_VALIDATION=when_required + +# Delete the file from S3 +aws s3 rm "$file_path" --endpoint-url="$endpoint_url" + +# Check if the delete was successful +if [ $? -eq 0 ]; then + echo "File $file_path deleted" +else + echo "Failed to delete file $file_path" + exit 1 +fi diff --git a/.github/workflows/check-duckdb-schema.yml b/.github/workflows/check-duckdb-schema.yml index 35bb4d895..504a49e2e 100644 --- a/.github/workflows/check-duckdb-schema.yml +++ b/.github/workflows/check-duckdb-schema.yml @@ -46,17 +46,40 @@ jobs: credentials_json: '${{ secrets.GOOGLE_BQ_ADMIN_CREDENTIALS_JSON }}' create_credentials_file: true - # - name: Download file from Cloudflare R2 - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_S3_API_ACCESS_KEY }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_S3_API_SECRET_KEY }} - # AWS_DEFAULT_REGION: "auto" - # run: bash .github/scripts/copy-s3.sh "$R2_S3_PATH" "$SQLMESH_DUCKDB_LOCAL_PATH" "${{ secrets.CLOUDFLARE_R2_S3_API_ENDPOINT }}" + - name: Download file from Cloudflare R2 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_S3_API_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_S3_API_SECRET_KEY }} + AWS_DEFAULT_REGION: "auto" + run: bash .github/scripts/copy-s3.sh "$R2_S3_PATH" "$SQLMESH_DUCKDB_LOCAL_PATH" "${{ secrets.CLOUDFLARE_R2_S3_API_ENDPOINT }}" + + - name: Save copy of the database + run: cp $SQLMESH_DUCKDB_LOCAL_PATH $SQLMESH_DUCKDB_LOCAL_PATH.bak - name: Initialize local data run: poetry run oso metrics local initialize --max-results-per-query 50 --max-days 3 - + + - name: Compare database files + id: file-diff + run: | + # Compare two files or directories using the Unix `diff` command + if diff -q $SQLMESH_DUCKDB_LOCAL_PATH $SQLMESH_DUCKDB_LOCAL_PATH.bak > /dev/null; then + echo "::set-output name=files_different::false" + else + echo "::set-output name=files_different::true" + fi + + # Because of the checksum issue, we first delete the old file before uploading the new one + - name: Delete old Cloudflare R2 file + if: steps.file-diff.outputs.files_different == 'true' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_S3_API_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_S3_API_SECRET_KEY }} + AWS_DEFAULT_REGION: "auto" + run: bash .github/scripts/delete-s3.sh "$R2_S3_PATH" "${{ secrets.CLOUDFLARE_R2_S3_API_ENDPOINT }}" + - name: Upload file to Cloudflare R2 + if: steps.file-diff.outputs.files_different == 'true' env: AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_S3_API_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_S3_API_SECRET_KEY }}