Fix truncation of good first issues when posting to Zulip

The Zulip API has a max message size and our "good first issues" posting hits it resulting in message truncation and a loss of content. This commit adds a Python script to batch up "good first issues" content into 1 or more messages so that it doesn't get truncated. Also includes new image to run in as we need additional dependencies that the "post sync" image doesn't need. And this adds a PR check to link the python script and make sure it is all kosher. Fixes #18
ponylang · Dec 10, 2023 · 5ba1acf · 5ba1acf
1 parent e7dd8f2
commit 5ba1acf
Show file tree

Hide file tree

Showing 6 changed files with 153 additions and 14 deletions.
diff --git a/.ci-dockerfiles/good-first-issues/Dockerfile b/.ci-dockerfiles/good-first-issues/Dockerfile
@@ -0,0 +1,16 @@
+ARG FROM_TAG=release
+FROM ghcr.io/ponylang/shared-docker-ci-x86-64-unknown-linux-builder-with-libressl-3.7.3:${FROM_TAG}
+
+RUN apk add --update --no-cache \
+  pcre2-dev \
+  python3 \
+  python3-dev \
+  py3-pip
+
+RUN pip3 install --upgrade pip \
+  pylint \
+  zulip
+
+COPY post-good-first-issues.py /post-good-first-issues.py
+
+RUN chmod a+x /post-good-first-issues.py
diff --git a/.ci-dockerfiles/good-first-issues/build-and-push.bash b/.ci-dockerfiles/good-first-issues/build-and-push.bash
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+
+#
+# *** You should already be logged in to GitHub Container Registrṡ when you run
+#     this ***
+#
+# ***
+# This must be run from the root of the repository like:
+#     bash .ci-dockerfiles/good-first-issues/build-and-push.bash
+# ***
+#
+
+DOCKERFILE_DIR="$(dirname "$0")"
+NAME="ghcr.io/ponylang/pony-sync-helper-ci-good-first-issues"
+
+# built from x86-64-unknown-linux-builder release tag
+FROM_TAG=release
+TAG_AS=release
+docker build --pull --build-arg FROM_TAG="${FROM_TAG}" \
+  -t "${NAME}:${TAG_AS}" \
+  -f "${DOCKERFILE_DIR}/Dockerfile" .
+docker push "${NAME}:${TAG_AS}"
+
+# built from x86-64-unknown-linux-builder latest tag
+FROM_TAG=latest
+TAG_AS=latest
+docker build --pull --build-arg FROM_TAG="${FROM_TAG}" \
+  -t "${NAME}:${TAG_AS}" \
+  -f "${DOCKERFILE_DIR}/Dockerfile" .
+docker push "${NAME}:${TAG_AS}"
diff --git a/.github/workflows/breakage-against-linux-pony-latest.yml b/.github/workflows/breakage-against-linux-pony-latest.yml
@@ -31,6 +31,32 @@ jobs:
           topic: ${{ github.repository }} scheduled job failure
           content: ${{ github.server_url}}/${{ github.repository }}/actions/runs/${{ github.run_id }} failed.
 
+  rebuild-good-first-issues:
+    name: Update builder
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Login to GitHub Container Registry
+        # v2.2.0
+        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push
+        run: bash .ci-dockerfiles/good-first-issues/build-and-push.bash
+      - name: Send alert on failure
+        if: ${{ failure() }}
+        uses: zulip/github-actions-zulip/send-message@b62d5a0e48a4d984ea4fce5dd65ba691963d4db4
+        with:
+          api-key: ${{ secrets.ZULIP_SCHEDULED_JOB_FAILURE_API_KEY }}
+          email: ${{ secrets.ZULIP_SCHEDULED_JOB_FAILURE_EMAIL }}
+          organization-url: 'https://ponylang.zulipchat.com/'
+          to: notifications
+          type: stream
+          topic: ${{ github.repository }} scheduled job failure
+          content: ${{ github.server_url}}/${{ github.repository }}/actions/runs/${{ github.run_id }} failed.
+
   vs-ponyc-main-linux:
     name: Verify main against ponyc main on Linux
     needs:
@@ -62,12 +88,20 @@ jobs:
 
     name: Prune untagged images
     runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          - pony-sync-helper-ci-builder
+          - pony-sync-helper-ci-good-first-issues
+
     steps:
       - name: Prune
         # v4.1.1
         uses: actions/delete-package-versions@0d39a63126868f5eefaa47169615edd3c0f61e20
         with:
-          package-name: 'pony-sync-helper-ci-builder'
+          package-name: ${{ matrix.image }}
           package-type: 'container'
           min-versions-to-keep: 1
           delete-only-untagged-versions: 'true'

diff --git a/.github/workflows/post-good-first-issues.yml b/.github/workflows/post-good-first-issues.yml
@@ -10,11 +10,9 @@ jobs:
     name: Post good first issues to Zulip
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/ponylang/pony-sync-helper-ci-builder:release
+      image: ghcr.io/ponylang/pony-sync-helper-ci-good-first-issues:release
     steps:
       - uses: actions/checkout@v3
-      - name: Today's date
-        run: echo "DATE=$(date +%Y-%m-%d)" >> "$GITHUB_ENV"
       - name: Build sync-helper
         run: |
           corral fetch
@@ -28,13 +26,9 @@ jobs:
           } >> "$GITHUB_ENV"
         env:
           PONY_SYNC_HELPER_GITHUB_TOKEN: ${{ secrets.PONYLANG_MAIN_API_TOKEN }}
-      - name: Post Issues
-        uses: zulip/github-actions-zulip/send-message@b62d5a0e48a4d984ea4fce5dd65ba691963d4db4
-        with:
-          api-key: ${{ secrets.ZULIP_SYNC_EVENT_API_KEY }}
-          email: ${{ secrets.ZULIP_SYNC_EVENT_EMAIL }}
-          organization-url: 'https://ponylang.zulipchat.com/'
-          to: contribute to Pony
-          type: stream
-          topic: Good first issues as of ${{env.DATE}}
-          content: ${{ env.ISSUES }}
+      - name: Post to Zulip
+        run: /post-good-first-issues.py
+        env:
+          ZULIP_API_KEY: ${{ secrets.ZULIP_SYNC_EVENT_API_KEY }}
+          ZULIP_EMAIL: ${{ secrets.ZULIP_SYNC_EVENT_EMAIL }}
+          ZULIP_SITE: 'https://ponylang.zulipchat.com/'
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -33,3 +33,22 @@ jobs:
         run: |
           corral fetch
           corral run -- ponyc -Dopenssl_0.9.0
+
+  lint-post-good-first-issues:
+    name: Verify that post-good-first-issues.py passes linting checks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build docker image
+        run: docker build --pull --build-arg FROM_TAG="latest" -t "ghcr.io/ponylang/pony-sync-helper-ci-good-first-issues:latest" -f .ci-dockerfiles/good-first-issues/Dockerfile .
+      - name: Lint post-good-first-issues.py
+        run: docker run --entrypoint pylint --rm "ghcr.io/ponylang/pony-sync-helper-ci-good-first-issues:latest" /post-good-first-issues.py
+
+  verify-good-first-issues-image-builds:
+    name: Verify the good-first-issues image builds
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build docker image
+        run: docker build --pull --build-arg FROM_TAG="latest" -t "ghcr.io/ponylang/pony-sync-helper-ci-good-first-issues:latest" -f .ci-dockerfiles/good-first-issues/Dockerfile .
+
diff --git a/post-good-first-issues.py b/post-good-first-issues.py
@@ -0,0 +1,43 @@
+#!/usr/bin/python3
+# pylint: disable=C0103
+# pylint: disable=C0114
+
+from datetime import datetime
+import os
+import zulip
+
+issues_str = os.environ['ISSUES']
+issues_lines = issues_str.splitlines()
+
+messages = []
+current_message = ""
+for line in issues_lines:
+    if len(current_message) + len(line) <= 7000:
+        current_message += line + "\n"
+    else:
+        # We have a decent chunk of potent to post at this point.
+        # We are now looking for the next repository to start a new message.
+        # We don't want to break during a repository as it messes up the
+        # markdown formatting of the message in Zulip.
+        if line.startswith("**ponylang/"):
+          messages.append(current_message)
+          current_message = line + "\n"
+        else:
+          current_message += line + "\n"
+
+# add the final message in to messages
+messages.append(current_message)
+
+# set up topic with today's date
+msg_date = datetime.today().strftime('%Y-%m-%d')
+msg_topic = "Good first issues as of " + msg_date
+
+zulip_client = zulip.Client()
+for message in messages:
+    request = {
+        "type": "stream",
+        "to": "contribute to Pony",
+        "topic": msg_topic,
+        "content": message
+    }
+    result = zulip_client.send_message(request)