diff --git a/.github/docker-images/debian-stretch-arm32v5/Dockerfile b/.github/docker-images/debian-stretch-arm32v5/Dockerfile deleted file mode 100644 index 2bf44327d..000000000 --- a/.github/docker-images/debian-stretch-arm32v5/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM arm32v5/debian:stretch-slim - -RUN apt-get update -y \ - && apt-get dist-upgrade -y \ - && apt-get install -y \ - sudo \ - ca-certificates \ - build-essential \ - curl \ - cmake \ - git \ - python3-dev \ - python3-pip \ - unzip \ - tar \ - && cmake --version \ - && ctest --version - -############################################################################### -# Python/AWS CLI -############################################################################### -WORKDIR /tmp - -RUN python3 -m pip install setuptools \ - && python3 -m pip install awscli \ - && aws --version - -############################################################################### -# Install entrypoint -############################################################################### -ADD entrypoint.sh /usr/local/bin/builder -RUN chmod a+x /usr/local/bin/builder -ENTRYPOINT ["/usr/local/bin/builder"] diff --git a/.github/docker-images/debian-stretch-arm32v7/Dockerfile b/.github/docker-images/debian-stretch-arm32v7/Dockerfile deleted file mode 100644 index 7c7044eaa..000000000 --- a/.github/docker-images/debian-stretch-arm32v7/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM arm32v7/debian:stretch-slim - -RUN apt-get update -y \ - && apt-get dist-upgrade -y \ - && apt-get install -y \ - sudo \ - ca-certificates \ - build-essential \ - curl \ - cmake \ - git \ - python3-dev \ - python3-pip \ - unzip \ - tar \ - && cmake --version \ - && ctest --version - -############################################################################### -# Python/AWS CLI -############################################################################### -WORKDIR /tmp - -RUN python3 -m pip install setuptools \ - && python3 -m pip install awscli \ - && aws --version - -############################################################################### -# Install entrypoint -############################################################################### -ADD entrypoint.sh /usr/local/bin/builder -RUN chmod a+x /usr/local/bin/builder -ENTRYPOINT ["/usr/local/bin/builder"] diff --git a/.github/docker-images/manylinux2014-aarch64/Dockerfile b/.github/docker-images/manylinux2014-aarch64/Dockerfile index 16f63ff12..c685b585d 100644 --- a/.github/docker-images/manylinux2014-aarch64/Dockerfile +++ b/.github/docker-images/manylinux2014-aarch64/Dockerfile @@ -22,7 +22,7 @@ RUN /opt/python/cp39-cp39/bin/python -m pip install --upgrade setuptools virtual ############################################################################### # nodejs/npm ############################################################################### -RUN curl -sL https://rpm.nodesource.com/setup_12.x | sudo bash - +RUN curl -sL https://rpm.nodesource.com/setup_14.x | sudo bash - RUN sudo yum -y install nodejs && node --version ############################################################################### diff --git a/.github/docker-images/swift-5-al2-x64/Dockerfile b/.github/docker-images/swift-5-al2-x64/Dockerfile index 358703228..26cae9dde 100644 --- a/.github/docker-images/swift-5-al2-x64/Dockerfile +++ b/.github/docker-images/swift-5-al2-x64/Dockerfile @@ -1,5 +1,5 @@ # https://hub.docker.com/_/swift -FROM swift:5.5.3-amazonlinux2 +FROM swift:5.7.3-amazonlinux2 ############################################################################### # Install prereqs diff --git a/.github/docker-images/swift-5-centos-x64/Dockerfile b/.github/docker-images/swift-5-centos-x64/Dockerfile index f5df6b64f..34063a068 100644 --- a/.github/docker-images/swift-5-centos-x64/Dockerfile +++ b/.github/docker-images/swift-5-centos-x64/Dockerfile @@ -1,5 +1,5 @@ # https://hub.docker.com/_/swift -FROM swift:5.5.3-centos7 +FROM swift:5.7.3-centos7 ############################################################################### # Install prereqs diff --git a/.github/docker-images/swift-5-ubuntu-x64/Dockerfile b/.github/docker-images/swift-5-ubuntu-x64/Dockerfile index 8b03f5b71..45b2317af 100644 --- a/.github/docker-images/swift-5-ubuntu-x64/Dockerfile +++ b/.github/docker-images/swift-5-ubuntu-x64/Dockerfile @@ -1,5 +1,5 @@ # https://hub.docker.com/_/swift -FROM swift:5.5.3-focal +FROM swift:5.7.3-focal ############################################################################### # Install prereqs diff --git a/.github/docker-images/ubuntu-22-x64/Dockerfile b/.github/docker-images/ubuntu-22-x64/Dockerfile new file mode 100644 index 000000000..5924fc114 --- /dev/null +++ b/.github/docker-images/ubuntu-22-x64/Dockerfile @@ -0,0 +1,47 @@ +FROM ubuntu:22.04 + +ENV DEBIAN_FRONTEND=noninteractive + +############################################################################### +# Install prereqs +############################################################################### +RUN apt-get update -qq \ + && apt-get -y install \ + git \ + curl \ + sudo \ + unzip \ + python3-dev \ + python3-pip \ + build-essential \ + # For PPAs + software-properties-common \ + apt-transport-https \ + ca-certificates \ + && apt-get clean + +############################################################################### +# Python/AWS CLI +############################################################################### +WORKDIR /tmp + +RUN python3 -m pip install setuptools \ + && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o awscliv2.zip \ + && unzip awscliv2.zip \ + && sudo aws/install \ + && aws --version + +############################################################################### +# Install pre-built CMake +############################################################################### +RUN curl -sSL https://d19elf31gohf1l.cloudfront.net/_binaries/cmake/cmake-3.13-manylinux1-x64.tar.gz -o cmake.tar.gz \ + && tar xvzf cmake.tar.gz -C /usr/local \ + && cmake --version \ + && rm -f /tmp/cmake.tar.gz + +############################################################################### +# Install entrypoint +############################################################################### +ADD entrypoint.sh /usr/local/bin/builder +RUN chmod a+x /usr/local/bin/builder +ENTRYPOINT ["/usr/local/bin/builder"] diff --git a/.github/workflows/create-channel.yml b/.github/workflows/create-channel.yml index 9fa1bdd7b..8f325ce57 100644 --- a/.github/workflows/create-channel.yml +++ b/.github/workflows/create-channel.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Checkout Source - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get release tag uses: ./.github/actions/release-tag @@ -71,8 +71,7 @@ jobs: - ubuntu-18-x64 - ubuntu-20-x64 - ubuntu-20-aarch64 - - debian-stretch-arm32v5 - - debian-stretch-arm32v7 + - ubuntu-22-x64 - node-10-linux-x64 - swift-5-al2-x64 - swift-5-centos-x64 @@ -80,7 +79,6 @@ jobs: - rhel8-x64 - opensuse-leap - fedora-34-x64 - - raspbian-bullseye - alpine-3.16-x64 - alpine-3.16-x86 - alpine-3.16-arm64 @@ -90,7 +88,7 @@ jobs: steps: - name: Checkout Sources - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get release tag uses: ./.github/actions/release-tag @@ -126,3 +124,35 @@ jobs: run: | export IMAGE_TAG=${{ steps.tag.outputs.release_tag }} docker push ${{ secrets.AWS_ECR_REPO }}/aws-crt-${{ matrix.variant }}:$IMAGE_TAG + + # The job will directly pull the image from the latest, and push to the new tag. + # This job is only for a temporary fix. The cheated image would not get updated. + passthrough-images: + name: ${{ matrix.variant }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + variant: + - raspbian-bullseye + + steps: + - name: Checkout Sources + uses: actions/checkout@v4 + + - name: Get release tag + uses: ./.github/actions/release-tag + id: tag + + - name: Login to docker repo + run: aws ecr get-login-password --region us-east-1 | docker login ${{ secrets.AWS_ECR_REPO }} -u AWS --password-stdin + + - name: Pull latest image and push + run: | + if ! aws --region us-east-1 ecr describe-repositories --repository-names aws-crt-${{ matrix.variant }} > /dev/null 2>&1; then \ + exit 1 + fi + export IMAGE_TAG=${{ steps.tag.outputs.release_tag }} + docker pull ${{ secrets.AWS_ECR_REPO }}/aws-crt-${{ matrix.variant }}:latest + docker tag ${{ secrets.AWS_ECR_REPO }}/aws-crt-${{ matrix.variant }}:latest ${{ secrets.AWS_ECR_REPO }}/aws-crt-${{ matrix.variant }}:$IMAGE_TAG + docker push ${{ secrets.AWS_ECR_REPO }}/aws-crt-${{ matrix.variant }}:$IMAGE_TAG diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 736e519a9..601c3d7b2 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -20,7 +20,7 @@ jobs: steps: - name: Checkout Source - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get release tag uses: ./.github/actions/release-tag @@ -65,8 +65,6 @@ jobs: - ubuntu-18-x64 - ubuntu-20-x64 - ubuntu-20-aarch64 - - debian-stretch-arm32v5 - - debian-stretch-arm32v7 - node-10-linux-x64 - swift-5-al2-x64 - swift-5-centos-x64 @@ -74,7 +72,6 @@ jobs: - rhel8-x64 - opensuse-leap - fedora-34-x64 - - raspbian-bullseye - alpine-3.16-x64 - alpine-3.16-x86 - alpine-3.16-arm64 @@ -84,7 +81,7 @@ jobs: steps: - name: Checkout Sources - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get release tag uses: ./.github/actions/release-tag @@ -132,13 +129,45 @@ jobs: | aws s3 cp - s3://${{env.AWS_S3_BUCKET}}/releases/$IMAGE_TAG/aws-crt-${{ matrix.variant }}.tar.gz aws s3 cp s3://${{env.AWS_S3_BUCKET}}/releases/$IMAGE_TAG/aws-crt-${{ matrix.variant }}.tar.gz s3://${{env.AWS_S3_BUCKET}}/LATEST/aws-crt-${{ matrix.variant }}.tar.gz + # The job will directly pull the image from the latest, and push to the new tag. + # This job is only for a temporary fix. The image would not get updated. + passthrough-images: + name: ${{ matrix.variant }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + variant: + - raspbian-bullseye + + steps: + - name: Checkout Sources + uses: actions/checkout@v4 + + - name: Get release tag + uses: ./.github/actions/release-tag + id: tag + + - name: Login to docker repo + run: aws ecr get-login-password --region us-east-1 | docker login ${{ secrets.AWS_ECR_REPO }} -u AWS --password-stdin + + - name: Pull latest image and push + run: | + if ! aws --region us-east-1 ecr describe-repositories --repository-names aws-crt-${{ matrix.variant }} > /dev/null 2>&1; then \ + exit 1 + fi + export IMAGE_TAG=${{ steps.tag.outputs.release_tag }} + docker pull ${{ secrets.AWS_ECR_REPO }}/aws-crt-${{ matrix.variant }}:latest + docker tag ${{ secrets.AWS_ECR_REPO }}/aws-crt-${{ matrix.variant }}:latest ${{ secrets.AWS_ECR_REPO }}/aws-crt-${{ matrix.variant }}:$IMAGE_TAG + docker push ${{ secrets.AWS_ECR_REPO }}/aws-crt-${{ matrix.variant }}:$IMAGE_TAG + upload-ci-script: name: Upload container ci script runs-on: ubuntu-latest steps: - name: Checkout Source - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get release tag uses: ./.github/actions/release-tag diff --git a/.github/workflows/handle-stale-discussions.yml b/.github/workflows/handle-stale-discussions.yml new file mode 100644 index 000000000..5b16398cf --- /dev/null +++ b/.github/workflows/handle-stale-discussions.yml @@ -0,0 +1,19 @@ +name: HandleStaleDiscussions +on: + schedule: + - cron: '0 */4 * * *' + discussion_comment: + types: [created] + +jobs: + handle-stale-discussions: + name: Handle stale discussions + runs-on: ubuntu-latest + permissions: + discussions: write + steps: + - name: Stale discussions action + uses: aws-github-ops/handle-stale-discussions@v1 + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5926159da..77d1f0e26 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Checkout Source - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build and Test run: | diff --git a/.github/workflows/linux-container-ci.sh b/.github/workflows/linux-container-ci.sh index 0cb5b1ee8..03fd2612a 100755 --- a/.github/workflows/linux-container-ci.sh +++ b/.github/workflows/linux-container-ci.sh @@ -11,4 +11,4 @@ shift aws ecr get-login-password | docker login 123124136734.dkr.ecr.us-east-1.amazonaws.com -u AWS --password-stdin export DOCKER_IMAGE=123124136734.dkr.ecr.us-east-1.amazonaws.com/${IMAGE_NAME}:${BUILDER_VERSION} -docker run --env GITHUB_REF --env GITHUB_HEAD_REF --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --env AWS_DEFAULT_REGION --env CXXFLAGS --env AWS_CRT_ARCH $DOCKER_IMAGE --version=${BUILDER_VERSION} $@ +docker run --env GITHUB_REF --env GITHUB_HEAD_REF --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --env AWS_DEFAULT_REGION --env CXXFLAGS --env AWS_CRT_ARCH --env CTEST_PARALLEL_LEVEL $DOCKER_IMAGE --version=${BUILDER_VERSION} $@ diff --git a/.github/workflows/sanity-test.yml b/.github/workflows/sanity-test.yml index 92cd84070..ae905c121 100644 --- a/.github/workflows/sanity-test.yml +++ b/.github/workflows/sanity-test.yml @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Source - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Run Tests run: python3 -m unittest discover tests --verbose @@ -42,7 +42,7 @@ jobs: steps: - name: Checkout Source - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get release tag uses: ./.github/actions/release-tag @@ -78,12 +78,12 @@ jobs: strategy: fail-fast: false matrix: - host: [ubuntu-20.04, macos-11, macos-12, windows-2022] + host: [ubuntu-22.04, macos-11, macos-12, windows-2022] needs: package runs-on: ${{ matrix.host }} steps: - name: Checkout Source - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install builder uses: actions/download-artifact@v3 @@ -145,7 +145,7 @@ jobs: steps: - name: Checkout Source - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install builder uses: actions/download-artifact@v3 @@ -183,7 +183,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Checkout Source - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install builder uses: actions/download-artifact@v3 @@ -205,7 +205,7 @@ jobs: std: [c++11, c++14, c++17, c++2a] steps: - name: Checkout Source - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install builder uses: actions/download-artifact@v3 @@ -255,7 +255,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Checkout Source - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install builder uses: actions/download-artifact@v3 @@ -284,7 +284,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Source - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get release tag uses: ./.github/actions/release-tag @@ -317,7 +317,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Source - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get release tag uses: ./.github/actions/release-tag diff --git a/builder/actions/setup_cross_ci_crt_environment.py b/builder/actions/setup_cross_ci_crt_environment.py index db3efaaa8..f0dc953fd 100644 --- a/builder/actions/setup_cross_ci_crt_environment.py +++ b/builder/actions/setup_cross_ci_crt_environment.py @@ -205,6 +205,8 @@ def _common_setup(self, env): "ci/mqtt5/us/authorizer/signed/tokenkeyname") self._setenv_secret(env, "AWS_TEST_MQTT5_IOT_CORE_SIGNING_AUTHORIZER_TOKEN_SIGNATURE", "ci/mqtt5/us/authorizer/signed/signature") + self._setenv_secret(env, "AWS_TEST_MQTT5_IOT_CORE_SIGNING_AUTHORIZER_TOKEN_SIGNATURE_UNENCODED", + "ci/mqtt5/us/authorizer/signed/signature/unencoded") # JAVA KEYSTORE (Java uses PKCS#8 keys internally, which currently only Linux supports ATM) if (self.is_linux == True): @@ -291,6 +293,8 @@ def _common_setup(self, env): "ci/mqtt5/us/authorizer/signed/tokenkeyname") self._setenv_secret(env, "AWS_TEST_MQTT311_IOT_CORE_SIGNING_AUTHORIZER_TOKEN_SIGNATURE", "ci/mqtt5/us/authorizer/signed/signature") + self._setenv_secret(env, "AWS_TEST_MQTT311_IOT_CORE_SIGNING_AUTHORIZER_TOKEN_SIGNATURE_UNENCODED", + "ci/mqtt5/us/authorizer/signed/signature/unencoded") # JAVA KEYSTORE (Java uses PKCS#8 keys internally, which currently only Linux supports ATM) if (self.is_linux == True): diff --git a/builder/actions/setup_cross_ci_helpers.py b/builder/actions/setup_cross_ci_helpers.py index 1419f92ee..968426028 100644 --- a/builder/actions/setup_cross_ci_helpers.py +++ b/builder/actions/setup_cross_ci_helpers.py @@ -215,7 +215,7 @@ def _get_token_slots(env): def _get_softhsm2_version(env): output = _exec_softhsm2_util(env, '--version').output - match = re.match('([0-9+])\.([0-9]+).([0-9]+)', output) + match = re.match(r'([0-9+])\.([0-9]+).([0-9]+)', output) return (int(match.group(1)), int(match.group(2)), int(match.group(3))) ################################################################################ diff --git a/builder/core/toolchain.py b/builder/core/toolchain.py index 97ab4020a..c33fc9815 100644 --- a/builder/core/toolchain.py +++ b/builder/core/toolchain.py @@ -17,19 +17,19 @@ def _compiler_version(cc): for text in lines: # Apple clang - m = re.match('Apple (LLVM|clang) version (\d+)', text) + m = re.match(r'Apple (LLVM|clang) version (\d+)', text) if m: return 'appleclang', m.group(2) # LLVM clang - m = re.match('.*(LLVM|clang) version (\d+)', text) + m = re.match(r'.*(LLVM|clang) version (\d+)', text) if m: return 'clang', m.group(2) # GCC 4.x - m = re.match('gcc .+ (4\.\d+)', text) + m = re.match(r'gcc .+ (4\.\d+)', text) if m: return 'gcc', m.group(1) # GCC 5+ - m = re.match('gcc .+ (\d+)\.', text) + m = re.match(r'gcc .+ (\d+)\.', text) if m: return 'gcc', m.group(1) return None, None diff --git a/builder/imports/llvm.py b/builder/imports/llvm.py index ccb932fb7..7a2372fc6 100644 --- a/builder/imports/llvm.py +++ b/builder/imports/llvm.py @@ -130,7 +130,7 @@ def install(self, env): sudo = ['sudo'] if sudo else [] # Strip minor version info - version = env.toolchain.compiler_version.replace('\..+', '') + version = env.toolchain.compiler_version.replace(r'\..+', '') script = tempfile.NamedTemporaryFile(delete=False) script_path = script.name diff --git a/builder/imports/nodejs.py b/builder/imports/nodejs.py index 0d395af35..36ff88266 100644 --- a/builder/imports/nodejs.py +++ b/builder/imports/nodejs.py @@ -14,7 +14,7 @@ import re -NVM = """\ +NVM = r"""\ #!/usr/bin/env bash export NVM_DIR="$HOME/.nvm" [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" @@ -126,7 +126,7 @@ def install_node_via_unofficial_build(self, env): # Normaliz version format, please note 12.16.3 is the last version has x86 support def normalize_version(v): append_times = 0 - while re.match('^([0-9]+\.){2}[0-9]+$', v) == None: + while re.match(r'^([0-9]+\.){2}[0-9]+$', v) == None: # Only try append sub version twice if append_times < 2: v += ".0"