From cc1e9f736b06054c145ee066944a0405d0d37518 Mon Sep 17 00:00:00 2001 From: "Sean T. Allen" Date: Sat, 9 Dec 2023 23:23:19 +0000 Subject: [PATCH] Fix truncation of good first issues when posting to Zulip The Zulip API has a max message size and our "good first issues" posting hits it resulting in message truncation and a loss of content. This commit adds a Python script to batch up "good first issues" content into 1 or more messages so that it doesn't get truncated. Also includes new image to run in as we need additional dependencies that the "post sync" image doesn't need. And this adds a PR check to link the python script and make sure it is all kosher. Fixes #18 --- .ci-dockerfiles/good-first-issues/Dockerfile | 16 ++++++++ .../good-first-issues/build-and-push.bash | 33 +++++++++++++++ .../breakage-against-linux-pony-latest.yml | 36 ++++++++++++++++- .github/workflows/post-good-first-issues.yml | 25 +++--------- .github/workflows/pr.yml | 19 +++++++++ post-good-first-issues.py | 40 +++++++++++++++++++ 6 files changed, 149 insertions(+), 20 deletions(-) create mode 100644 .ci-dockerfiles/good-first-issues/Dockerfile create mode 100644 .ci-dockerfiles/good-first-issues/build-and-push.bash create mode 100644 post-good-first-issues.py diff --git a/.ci-dockerfiles/good-first-issues/Dockerfile b/.ci-dockerfiles/good-first-issues/Dockerfile new file mode 100644 index 0000000..dee59c7 --- /dev/null +++ b/.ci-dockerfiles/good-first-issues/Dockerfile @@ -0,0 +1,16 @@ +ARG FROM_TAG=release +FROM ghcr.io/ponylang/shared-docker-ci-x86-64-unknown-linux-builder-with-libressl-3.7.3:${FROM_TAG} + +RUN apk add --update --no-cache \ + pcre2-dev \ + python3 \ + python3-dev \ + py3-pip + +RUN pip3 install --upgrade pip \ + pylint \ + zulip + +COPY post-good-first-issues.py /post-good-first-issues.py + +RUN chmod a+x /post-good-first-issues.py diff --git a/.ci-dockerfiles/good-first-issues/build-and-push.bash b/.ci-dockerfiles/good-first-issues/build-and-push.bash new file mode 100644 index 0000000..c29aca6 --- /dev/null +++ b/.ci-dockerfiles/good-first-issues/build-and-push.bash @@ -0,0 +1,33 @@ +#!/bin/bash + +set -o errexit +set -o nounset + +# +# *** You should already be logged in to GitHub Container Registrṡ when you run +# this *** +# +# *** +# This must be run from the root of the repository like: +# bash .ci-dockerfiles/good-first-issues/build-and-push.bash +# *** +# + +DOCKERFILE_DIR="$(dirname "$0")" +NAME="ghcr.io/ponylang/pony-sync-helper-ci-good-first-issues" + +# built from x86-64-unknown-linux-builder release tag +FROM_TAG=release +TAG_AS=release +docker build --pull --build-arg FROM_TAG="${FROM_TAG}" \ + -t "${NAME}:${TAG_AS}" \ + -f "${DOCKERFILE_DIR}/Dockerfile" . +docker push "${NAME}:${TAG_AS}" + +# built from x86-64-unknown-linux-builder latest tag +FROM_TAG=latest +TAG_AS=latest +docker build --pull --build-arg FROM_TAG="${FROM_TAG}" \ + -t "${NAME}:${TAG_AS}" \ + -f "${DOCKERFILE_DIR}/Dockerfile" . +docker push "${NAME}:${TAG_AS}" diff --git a/.github/workflows/breakage-against-linux-pony-latest.yml b/.github/workflows/breakage-against-linux-pony-latest.yml index babc5ea..24daf0c 100644 --- a/.github/workflows/breakage-against-linux-pony-latest.yml +++ b/.github/workflows/breakage-against-linux-pony-latest.yml @@ -31,6 +31,32 @@ jobs: topic: ${{ github.repository }} scheduled job failure content: ${{ github.server_url}}/${{ github.repository }}/actions/runs/${{ github.run_id }} failed. + rebuild-good-first-issues: + name: Update builder + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Login to GitHub Container Registry + # v2.2.0 + uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push + run: bash .ci-dockerfiles/good-first-issues/build-and-push.bash + - name: Send alert on failure + if: ${{ failure() }} + uses: zulip/github-actions-zulip/send-message@b62d5a0e48a4d984ea4fce5dd65ba691963d4db4 + with: + api-key: ${{ secrets.ZULIP_SCHEDULED_JOB_FAILURE_API_KEY }} + email: ${{ secrets.ZULIP_SCHEDULED_JOB_FAILURE_EMAIL }} + organization-url: 'https://ponylang.zulipchat.com/' + to: notifications + type: stream + topic: ${{ github.repository }} scheduled job failure + content: ${{ github.server_url}}/${{ github.repository }}/actions/runs/${{ github.run_id }} failed. + vs-ponyc-main-linux: name: Verify main against ponyc main on Linux needs: @@ -62,12 +88,20 @@ jobs: name: Prune untagged images runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + image: + - pony-sync-helper-ci-builder + - pony-sync-helper-ci-good-first-issues + steps: - name: Prune # v4.1.1 uses: actions/delete-package-versions@0d39a63126868f5eefaa47169615edd3c0f61e20 with: - package-name: 'pony-sync-helper-ci-builder' + package-name: ${{ matrix.image }} package-type: 'container' min-versions-to-keep: 1 delete-only-untagged-versions: 'true' diff --git a/.github/workflows/post-good-first-issues.yml b/.github/workflows/post-good-first-issues.yml index 963da6b..b9281e0 100644 --- a/.github/workflows/post-good-first-issues.yml +++ b/.github/workflows/post-good-first-issues.yml @@ -10,31 +10,18 @@ jobs: name: Post good first issues to Zulip runs-on: ubuntu-latest container: - image: ghcr.io/ponylang/pony-sync-helper-ci-builder:release + image: ghcr.io/ponylang/pony-sync-helper-ci-good-first-issues:release steps: - uses: actions/checkout@v3 - - name: Today's date - run: echo "DATE=$(date +%Y-%m-%d)" >> "$GITHUB_ENV" - name: Build sync-helper run: | corral fetch corral run -- ponyc -Dopenssl_0.9.0 - - name: Run sync-helper + - name: Get good first issues and post to Zulip run: | - { - echo 'ISSUES<> "$GITHUB_ENV" + ./pony-sync-helper --label "good first issue" --org ponylang --github_token "$PONY_SYNC_HELPER_GITHUB_TOKEN" | /post-good-first-issues.py env: PONY_SYNC_HELPER_GITHUB_TOKEN: ${{ secrets.PONYLANG_MAIN_API_TOKEN }} - - name: Post Issues - uses: zulip/github-actions-zulip/send-message@b62d5a0e48a4d984ea4fce5dd65ba691963d4db4 - with: - api-key: ${{ secrets.ZULIP_SYNC_EVENT_API_KEY }} - email: ${{ secrets.ZULIP_SYNC_EVENT_EMAIL }} - organization-url: 'https://ponylang.zulipchat.com/' - to: contribute to Pony - type: stream - topic: Good first issues as of ${{env.DATE}} - content: ${{ env.ISSUES }} + ZULIP_API_KEY: ${{ secrets.ZULIP_SYNC_EVENT_API_KEY }} + ZULIP_EMAIL: ${{ secrets.ZULIP_SYNC_EVENT_EMAIL }} + ZULIP_SITE: 'https://ponylang.zulipchat.com/' diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index acd220d..9d97115 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -42,3 +42,22 @@ jobs: - name: Build docker image working-directory: ./.ci-dockerfiles/builder run: docker build . --file Dockerfile + + lint-post-good-first-issues: + name: Verify that post-good-first-issues.py passes linting checks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Build docker image + run: docker build --pull --build-arg FROM_TAG="latest" -t "ghcr.io/ponylang/pony-sync-helper-ci-good-first-issues:latest" -f .ci-dockerfiles/good-first-issues/Dockerfile . + - name: Lint post-good-first-issues.py + run: docker run --entrypoint pylint --rm "ghcr.io/ponylang/pony-sync-helper-ci-good-first-issues:latest" /post-good-first-issues.py + + verify-good-first-issues-image-builds: + name: Verify the good-first-issues image builds + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Build docker image + working-directory: ./.ci-dockerfiles/good-first-issues + run: docker build . --file Dockerfile diff --git a/post-good-first-issues.py b/post-good-first-issues.py new file mode 100644 index 0000000..d841514 --- /dev/null +++ b/post-good-first-issues.py @@ -0,0 +1,40 @@ +#!/usr/bin/python3 +# pylint: disable=C0103 +# pylint: disable=C0114 + +from datetime import datetime +import sys +import zulip + +messages = [] +current_message = "" +for line in sys.stdin.readlines(): + if len(current_message) + len(line) <= 7000: + current_message += line + "\n" + else: + # We have a decent chunk of cotent to post at this point. + # We are now looking for the next repository to start a new message. + # We don't want to break during a repository as it messes up the + # markdown formatting of the message in Zulip. + if line.startswith("**ponylang/"): + messages.append(current_message) + current_message = line + "\n" + else: + current_message += line + "\n" + +# add the final message in to messages +messages.append(current_message) + +# set up topic with today's date +msg_date = datetime.today().strftime('%Y-%m-%d') +msg_topic = "Good first issues as of " + msg_date + +zulip_client = zulip.Client() +for message in messages: + request = { + "type": "stream", + "to": "contribute to Pony", + "topic": msg_topic, + "content": message + } + result = zulip_client.send_message(request)