Add repo publishing (#4)

* WIP: Experimental GH Pages publishing

* fix: Use a PAT to deploy cross-repo, sign commits

* fix: Don't attempt to toggle signing for local repo, which doesn't exist

* feat: Switch to aptly action v2, prep for release o11n

* fix: Try to get README.md to render as index.html

* fix: Better conditionals

* fix: Correct by broken-ass logic

* fix: Work around GitHub being an ass

* chore: Test backporting logic

* refactor: Switch testing to base from a repo snapshot instead of debs

* feat: Flip to go-live values. Will fail to run until upstream PR is in.

.github/workflows/create-apt-repo.yml

@@ -16,13 +16,11 @@ jobs:
       fail-fast: false
       matrix:
         category:
-          # TODO: replace refs with real ones once PR is approved
           - name: stable
-            # ref: v1.4.0
-            ref: stable_deb_testing
+            ref: 1.3.0  # Previous stable branch + backport
+            # ref: v1.4.0  # Future stables should probably use release tags
           - name: nightly
-            # ref: master
-            ref: the_debbening
+            ref: master
         os:
           - ubuntu-22.04  # Upstream LTS support ends 2027.
           - ubuntu-24.04  # Upstream LTS support ends 2029.
@@ -40,8 +38,7 @@ jobs:
       - name: Checkout Kanidm
         uses: actions/checkout@v4
         with:
-          # TODO: replace repository with upstream once PR is approved
-          repository: "jinnatar/kanidm"
+          repository: "kanidm/kanidm"
           ref: "${{ matrix.category.ref }}"
           submodules: false
       # Overlay the latest packaging tools instead of using the submodule reference which is intended for human use.
@@ -115,21 +112,58 @@ jobs:
         run: |
           find $(pwd) -name '*.deb'
       - name: Create Aptly repo
-        uses: jinnatar/actions-aptly-repo@v1.1.0
+        uses: jinnatar/actions-aptly-repo@v2.0.0
         with:
-          name: kanidm
+          name: kanidm_ppa
+          repo_url: https://kanidm.github.io/kanidm_ppa
           artifact_name: kanidm_ppa_snapshot
+          # TODO: Flip stable repo defs false -> true, _after_ we've done the first publish.
+          # This enables the "import & extend" model.
           repos: |
-              noble,stable,\"amd64,arm64\",debs/stable-ubuntu-24.04-*-unknown-linux-gnu/*.deb
-              jammy,stable,\"amd64,arm64\",debs/stable-ubuntu-22.04-*-unknown-linux-gnu/*.deb
-              noble,nightly,\"amd64,arm64\",debs/nightly-ubuntu-24.04-*-unknown-linux-gnu/*.deb
-              bookworm,stable,\"amd64,arm64\",debs/stable-debian-12-*-unknown-linux-gnu/*.deb
-              bookworm,nightly,\"amd64,arm64\",debs/nightly-debian-12-*-unknown-linux-gnu/*.deb
+              noble,stable,\"amd64,arm64\",false,debs/stable-ubuntu-24.04-*-unknown-linux-gnu/*.deb
+              jammy,stable,\"amd64,arm64\",false,debs/stable-ubuntu-22.04-*-unknown-linux-gnu/*.deb
+              noble,nightly,\"amd64,arm64\",false,debs/nightly-ubuntu-24.04-*-unknown-linux-gnu/*.deb
+              bookworm,stable,\"amd64,arm64\",false,debs/stable-debian-12-*-unknown-linux-gnu/*.deb
+              bookworm,nightly,\"amd64,arm64\",false,debs/nightly-debian-12-*-unknown-linux-gnu/*.deb
           # When GPG secrets are not available (say a PR), the repo WILL NOT be signed.
           # Provide your own key material in a fork to test with signed repo snapshots.
           gpg_private_key: "${{ secrets.GPG_PRIVATE_KEY }}"
           gpg_passphrase: "${{ secrets.PASSPHRASE }}"
 
-  # Step 4. Publish the created repo if this meets the requirements for publishing
-  # i.e. need to ignore pull-requests and perhaps even merges that don't have a specific tag.
-  # Point being to avoid pushing out churn all the way to downstream users unless we actually want them to update.
+  # Step 4. Publish the created repo if and only if it's a push to main.
+  publish:
+    name: Deploy to GitHub Pages
+    needs: create-repo
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Download repo snapshot
+        uses: actions/download-artifact@v4
+        with:
+          name: kanidm_ppa_snapshot
+          path: snapshot
+          merge-multiple: true  # Flatten artifact name out
+      - name: Add README.md  # Pull in book chapter on PPA usage
+        shell: bash
+        run: |
+          curl https://raw.githubusercontent.com/kanidm/kanidm/refs/heads/master/book/src/packaging/ppa_packages.md > snapshot/README.md
+      - name: Import GPG key  # So we can sign the repository commit
+        uses: crazy-max/ghaction-import-gpg@v6
+        env:
+          # GitHub is a real ass about checking whether secrets are available or not.
+          private_key_check: ${{ secrets.GPG_PRIVATE_KEY }}
+        if: env.private_key_check != ''  # Not present for PRs on purpose.
+        with:
+          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
+          passphrase: ${{ secrets.PASSPHRASE }}
+          git_commit_gpgsign: true
+      - name: Publish to PPA
+        uses: crazy-max/ghaction-github-pages@v4
+        if: github.ref == 'refs/heads/main'
+        with:
+          repo: kanidm/kanidm_ppa
+          target_branch: main
+          keep_history: false
+          build_dir: snapshot
+          allow_empty_commit: false
+        env:
+          GH_PAT: ${{ secrets.DEPLOY_PAT }}

testing/.gitignore

@@ -5,4 +5,6 @@
 *.pid
 *.yaml
 *.socket
+*.zip
 ssh_ed25519*
+snapshot/

testing/README.md

@@ -6,29 +6,28 @@ What is here was dangerous and repulsive to us. This message is a warning about
 Testing other architectures is even more Fun than packaging for them. The scripts here make it plausible, if not exactly great.
 
 1. `cd` to the root of the `testing/` dir.
-1. Modify `unixd.toml` & `kanidm.toml` to match your live deployed Kanidm environment that has a user with posix & ssh keys set up.
-1. Download and unpack GHA deb artifacts into `debs/{stable,nightly}`. The default sequence only tests stable, so may not want to bother with nightly.
-   - A correct looking path would be: `debs/stable/stable-debian-12-aarch64-unknown-linux-gnu/kanidm-unixd_1.3.3-202410071359+ae1df93_arm64.deb`
-1. Run `scripts/run-all.sh`, you may want to modify the port allocations or target sequence in it first.
-1. QEMU VMs cross-architecture are slow, very very slow.
+1. Download a GHA repo snapshot artifact zip and place it in the current directory as `kanidm_ppa_snapshot.zip`
+1. Run `IDM_URI=https://idm.example.com scripts/run-all.sh`, you may want to override other bits of env, see the top of the script.
+   - At first your snapshot is unpacked and a mirror is launched with the contents listening on localhost.
    - You can view what's going on in the console with `nc localhost 4321`
    - You can poke at the qemu console itself with `sudo socat -,echo=0,icanon=0 unix-connect:qemu-monitor.socket`
-1. Once the VM is up and reachable, integration starts. Cross-arch this is also very slow. Once it's following the sshd log you're ready to test.
+1. Once the VM is up and reachable, integration starts. Once it's following the sshd log you're ready to test. If anything goes wrong, execution will pause instead with a warning.
 1. Testing time.
    - A good basic test is to run in another terminal:
    ```shell
-   ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null localhost -p 2222 "uname -a && cat /etc/os-release"
+   ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null localhost -p 2222 "uname -a && cat /etc/os-release && kanidm login -D anonymous && kanidm self whoami"
    ```
    - Or if that doesn't work, troubleshoot via the cloud-init injected root key:
    ```shell
    sudo -E ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ssh_ed25519 root@localhost -p 2222
    ```
 1. Once  happy with the permutation, hit `^C` in the original terminal to terminate the permutation. Hit `Enter` to continue to the next one.
 1. Iterate until your willpower has crumbled or you reach the end of the target list.
+1. You may need to kill the mirror http server manually if you `^C` out early, or let it stay running for the next attempt.
 
 ### Known issues
-- aarch64 is super slow. If you're a QEMU wizard, try optimizing `scripts/launch-one.sh`
+- aarch64 is super slow cross-arch, so we disable cross-arch by default. Instead, run the same testing natively on an aarch64 platform and it'll work ok.
 - We throw 4 cores for the cpu so that mounting the rootfs is fast enough to not time out on ubuntu. Yes, that's a crazy problem to have.
-- Newer versions of systemd on the target image are susceptible to odd crashes. Anything tried to make them better instead made everything else so much worse.
+- If you insist in running aarch64 cross-arch, beware that systemd will throw weird hissyfits. The arguments try to work around it, but it's not foolproof.
 - We expand every disk image a bit because some default image sizes only leave ~200MiB of free space which isn't enough for our deb sizes to go through dpkg copy logic. This makes GPT unhappy, but that seems to be ok.
-- Too many things require sudo. You could probably somehow run qemu without it but meh.
+- Too many things require sudo. You could instead add your user to the `kvm` group and not sudo as much.

testing/kanidm_ppa.list

@@ -0,0 +1 @@
+deb [signed-by=/etc/apt/trusted.gpg.d/kanidm_ppa.asc] http://10.0.2.2:%MIRROR_PORT% %VERSION_CODENAME% stable

testing/scripts/launch-one.sh

@@ -51,8 +51,9 @@ case "$arch" in
 		;;
 esac
 
-SSH_PORT="${SSH_PORT:-222}"
+SSH_PORT="${SSH_PORT:-2222}"
 TELNET_PORT="${TELNET_PORT:-4321}"
+MIRROR_PORT="${MIRROR_PORT:-31625}"
 
 
 >&2 echo "Booting $arch $MACHINE with $EFI from $img"
@@ -81,9 +82,9 @@ while true; do
 done
 
 >&2 echo "Up! Transferring assets."
-scp $SSH_OPTS -P "$SSH_PORT" -r $debglob test_payload.sh unixd.toml kanidm.toml root@localhost:
+scp $SSH_OPTS -P "$SSH_PORT" test_payload.sh kanidm_ppa.list snapshot/kanidm_ppa.asc root@localhost:
 >&2 echo "Launching test payload."
-ssh $SSH_OPTS -p "$SSH_PORT" root@localhost ./test_payload.sh
+ssh $SSH_OPTS -p "$SSH_PORT" root@localhost "./test_payload.sh $IDM_URI $IDM_GROUP $MIRROR_PORT"
 
 >&2 echo "Done, killing qemu"
 kill $(cat qemu.pid)

testing/scripts/run-all.sh

@@ -2,7 +2,12 @@
 scripts/install-deps.sh
 scripts/get-images.sh
 
-export SSH_PORT=2222
+# Configs specific to your environment
+export SSH_PORT="${SSH_PORT:-2222}" # Any free port will do
+export IDM_URI="${IDM_URI?}" # No reasonable default!
+export IDM_GROUP="${IDM_GROUP:-posix_login}"
+export TELNET_PORT="${TELNET_PORT:-4321}"
+export MIRROR_PORT="${MIRROR_PORT:-31625}"
 
 function prompt(){
   read -p "Happy? ^C to stop full run, enter to continue to next target."
@@ -17,6 +22,13 @@ function run(){
 	sleep 2s  # Wait for qemu to release ports
 }
 
+### Launch the repo snapshot in the background
+# Assumes you've downloaded kanidm_ppa_snapshot.zip from a signed fork branch.
+>&2 echo "Launching mirror snapshot"
+scripts/run-mirror.sh kanidm_ppa_snapshot.zip &
+mirror_pid="$!"
+sleep 2s  # A bit of time for the unzip before we try to use the mirror
+
 ### Sequencing of permutations. The defaults only test current stable on current native arch
 # You could just enable aarch64 manually below, but better off running on a pi5 natively!
 
@@ -30,3 +42,5 @@ run debian-12 debs/stable/stable-debian-12-${target}-unknown-linux-gnu/kanidm*
 run jammy debs/stable/stable-ubuntu-22.04-${target}-unknown-linux-gnu/kanidm*
 run noble debs/stable/stable-ubuntu-24.04-${target}-unknown-linux-gnu/kanidm*
 
+>&2 echo "Killing mirror snapshot"
+kill "$mirror_pid"

testing/scripts/run-mirror.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -eu
+
+MIRROR_PORT="${MIRROR_PORT:-31625}"
+
+zip=$(readlink -f ${1?})
+
+mkdir -p snapshot && cd snapshot
+unzip -o "$zip"
+
+python3 -m http.server -b 127.0.0.1 "$MIRROR_PORT" -d .

testing/test_payload.sh

@@ -1,35 +1,35 @@
 #!/bin/bash
-ls *.deb
+
+IDM_URI="${1?}"
+IDM_GROUP="${2?}"
+MIRROR_PORT="${3?}"
+
+set -eu
+
+function debug(){
+	>&2 echo "Something went wrong, pausing for debug, to connect:"
+	>&2 echo "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@localhost -p 2222 -i ssh_ed25519"
+	sleep infinity
+}
 
 # Make apt shut up about various things to see relevant output better
 export DEBIAN_FRONTEND=noninteractive
 export LC_CTYPE=en_US.UTF-8
 export LC_ALL=en_US.UTF-8
 
-# Speed up aarch64 images that have snapd because it keeps churning back and forth with apt calls
->&2 echo "Disabling snapd, if it's there"
-systemctl disable --now snapd.socket
-systemctl disable --now snapd.seeded.service
-systemctl disable --now snapd.service
+source /etc/os-release
+sed "s/%MIRROR_PORT%/${MIRROR_PORT}/;s/%VERSION_CODENAME%/${VERSION_CODENAME}/" kanidm_ppa.list > /etc/apt/sources.list.d/kanidm_ppa.list
+mv kanidm_ppa.asc /etc/apt/trusted.gpg.d/
 
-apt update
-apt install -y zsh  # So if the test user has zsh, it'll work here
-
-# The alphabetical order just happens to be the right order, but the libs need to be intalled first.
-for pkg in *.deb; do
-	apt install -y "./${pkg}"
-done
+apt update || debug
+apt install -y zsh kanidm-unixd kanidm || debug
 
 >&2 echo "Configuring kanidm-unixd"
+sed "s_#uri.*_uri = \"${IDM_URI}\"_" -i /etc/kanidm/config
+sed "s@#pam_allowed_login_groups.*@pam_allowed_login_groups = \[\"${IDM_GROUP}\"\]@" -i /etc/kanidm/unixd
 
-mkdir /etc/kanidm
-mv unixd.toml /etc/kanidm/unixd
-mv kanidm.toml /etc/kanidm/config
-
->&2 echo "Starting unixd"
-
-systemctl start kanidm-unixd.service
-systemctl start kanidm-unixd-tasks.service
+>&2 echo "Restarting unixd"
+systemctl restart kanidm-unixd.service || debug
 
 >&2 echo "Configuring NSS"
 sed -E 's/(passwd|group): (.*)/\1: \2 kanidm/' -i /etc/nsswitch.conf
@@ -43,7 +43,7 @@ AuthorizedKeysCommand /usr/sbin/kanidm_ssh_authorizedkeys %u
 AuthorizedKeysCommandUser nobody
 LogLevel DEBUG1
 EOT
-systemctl restart ssh.service
+systemctl restart ssh.service || debug
 
 >&2 echo "Go test ssh login! Do a ^C here when you're done"
 >&2 echo "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null localhost -p 2222"