diff --git a/.github/workflows/test_deployment.yml b/.github/workflows/test_deployment.yml
new file mode 100644
index 0000000000..b452b49bd9
--- /dev/null
+++ b/.github/workflows/test_deployment.yml
@@ -0,0 +1,150 @@
+---
+name: Test collection roles
+on:
+  - push
+  - pull_request
+env:
+  test_path: ".galaxy-build"
+
+jobs:
+  check_roles:
+    name: Verify modified roles
+    runs-on: ubuntu-latest
+    outputs:
+      ipaserver: ${{ steps.mod_files.outputs.ipaserver }}
+      ipareplica: ${{ steps.mod_files.outputs.ipareplica }}
+      ipaclient: ${{ steps.mod_files.outputs.ipaclient }}
+    steps:
+      - uses: actions/checkout@v4.1.1
+        with:
+          fetch-depth: 1
+
+      - name: Check modified roles
+        id: mod_files
+        shell: bash
+        run: |
+          set -eu
+          git remote add upstream https://github.com/freeipa/ansible-freeipa
+          git fetch --depth 1 upstream master
+          git diff -M --name-only upstream/master \
+            | grep "^roles" \
+            | cut -d/ -f2 \
+            | sort | uniq \
+            | sed "s/\(.*\)/Testing: \1/"
+          git diff -M --name-only upstream/master \
+            | grep "^roles" \
+            | cut -d/ -f2 \
+            | sort | uniq \
+            | sed "s/\(.*\)/\1=true/" >> $GITHUB_OUTPUT
+
+  test_ipaserver:
+    name: Test ipaserver role
+    needs: [check_roles]
+    if: ${{ needs.check_roles.outputs.ipaserver }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        distro:
+          - fedora-latest
+          - c10s
+        cluster_config:
+          - ipaserver_test_base.yml
+          # - ipaserver_ca_dns_nokra_noad.yml
+          # - ipaserver_ipaclient_ca_only.yml
+    steps:
+      - uses: actions/checkout@v4.1.1
+        with:
+          fetch-depth: 1
+
+      - name: Build and install collection
+        run: |
+          utils/build-galaxy-release.sh -k \
+                -o "$(curl -L https://api.github.com/repos/freeipa/ansible-freeipa/releases 2>/dev/null \
+                      | jq '.[0].tag_name' | tr -d '"v')-dev$(date +"%Y%m%d%H%M")"
+
+      - name: Test ipaserver
+        uses: rjeffman/FreeIPA-Cluster-Test@v1.1.0
+        with:
+          cluster_configuration: tests/deployment/${{ matrix.cluster_config }}
+          # ansible_vars: test/deployment/ipaserver-vars.yml
+          distro: ${{ matrix.distro }}
+          test_playbooks: >-
+            ${{ env.test_path }}/tests/deployment/test_kinit.yml
+            ${{ env.test_path }}/tests/user/test_user.yml
+            ${{ env.test_path }}/tests/group/test_group.yml
+            ${{ env.test_path }}/tests/host/test_host.yml
+            ${{ env.test_path }}/tests/hbacrule/test_hbacrule.yml
+
+  test_ipareplica:
+    name: Test ipareplica role
+    needs: [check_roles]
+    if: ${{ needs.check_roles.outputs.ipareplica }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        distro:
+          - fedora-latest
+          - c10s
+        # The ipaserver capabilities are "ca-dns-kra"
+        cluster_config:
+          - ipareplica_ca.yml
+          # - ipareplica_none.yml
+    steps:
+      - uses: actions/checkout@v4.1.1
+        with:
+          fetch-depth: 1
+
+      - name: Build and install collection
+        run: |
+          utils/build-galaxy-release.sh -k \
+                -o "$(curl -L https://api.github.com/repos/freeipa/ansible-freeipa/releases 2>/dev/null \
+                      | jq '.[0].tag_name' | tr -d '"v')-dev$(date +"%Y%m%d%H%M")"
+
+      - name: Test ipareplica
+        uses: rjeffman/FreeIPA-Cluster-Test@v1.1.0
+        with:
+          cluster_configuration: tests/deployment/${{ matrix.cluster_config }}
+          ansible_vars: test/deployment/ipareplica-vars.yml
+          distro: ${{ matrix.distro }}
+          test_playbooks: >-
+            ${{ env.test_path }}/tests/deployment/test_kinit.yml
+            ${{ env.test_path }}/tests/user/test_user.yml
+            ${{ env.test_path }}/tests/group/test_group.yml
+            ${{ env.test_path }}/tests/host/test_host.yml
+            ${{ env.test_path }}/tests/hbacrule/test_hbacrule.yml
+
+  test_ipaclientt:
+    name: Test ipaclient role
+    needs: [check_roles]
+    if: ${{ needs.check_roles.outputs.ipaclient }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        distro:
+          - fedora-latest
+        # The ipaserver capabilities are "ca-dns-kra"
+        cluster_config:
+          - ipaserver_1_client.yml
+    steps:
+      - uses: actions/checkout@v4.1.1
+        with:
+          fetch-depth: 1
+
+      - name: Build and install collection
+        run: |
+          utils/build-galaxy-release.sh -k \
+                -o "$(curl -L https://api.github.com/repos/freeipa/ansible-freeipa/releases 2>/dev/null \
+                      | jq '.[0].tag_name' | tr -d '"v')-dev$(date +"%Y%m%d%H%M")"
+
+      - name: Test ipaclient
+        uses: rjeffman/FreeIPA-Cluster-Test@v1.1.0
+        with:
+          cluster_configuration: tests/deployment/${{ matrix.cluster_config }}
+          # ansible_vars: test/deployment/ipaclient-vars.yml
+          distro: ${{ matrix.distro }}
+          test_playbooks: >-
+            ${{ env.test_path }}/tests/deployment/test_kinit.yml
+            ${{ env.test_path }}/tests/user/test_user_client_context.yml
+            ${{ env.test_path }}/tests/group/test_group_client_context.yml
+            ${{ env.test_path }}/tests/host/test_host_client_context.yml
+            ${{ env.test_path }}/tests/hbacrule/test_hbacrule_client_context.yml
diff --git a/roles/ipaclient/README.md b/roles/ipaclient/README.md
index 39a315e6a0..0871a787fc 100644
--- a/roles/ipaclient/README.md
+++ b/roles/ipaclient/README.md
@@ -247,3 +247,5 @@ Authors
 Florence Blanc-Renaud
 
 Thomas Woerner
+
+# build
diff --git a/roles/ipareplica/README.md b/roles/ipareplica/README.md
index 5d1ece56fe..95754fa21d 100644
--- a/roles/ipareplica/README.md
+++ b/roles/ipareplica/README.md
@@ -317,3 +317,5 @@ Authors
 =======
 
 Thomas Woerner
+
+# build
diff --git a/roles/ipaserver/README.md b/roles/ipaserver/README.md
index 5776985616..d23baa1979 100644
--- a/roles/ipaserver/README.md
+++ b/roles/ipaserver/README.md
@@ -382,3 +382,5 @@ Authors
 =======
 
 Thomas Woerner
+
+# blah
diff --git a/tests/deployment/ipareplica-vars.yml b/tests/deployment/ipareplica-vars.yml
new file mode 100644
index 0000000000..adc1bcfe97
--- /dev/null
+++ b/tests/deployment/ipareplica-vars.yml
@@ -0,0 +1,2 @@
+---
+ipa_test_host: ipareplicas
diff --git a/tests/deployment/ipareplica_ca.yml b/tests/deployment/ipareplica_ca.yml
new file mode 100644
index 0000000000..d9537cb1f8
--- /dev/null
+++ b/tests/deployment/ipareplica_ca.yml
@@ -0,0 +1,11 @@
+ipa_deployments:
+  - name: ipareplica_ca
+    domain: ipa.test
+    admin_password: SomeADMINpassword
+    dm_password: SomeDMpassword
+    cluster:
+      servers:
+        - name: server
+          capabilities: ["CA", "DNS", "KRA"]
+        - name: replica
+          capabilities: ["CA"]
diff --git a/tests/deployment/ipaserver_1_client.yml b/tests/deployment/ipaserver_1_client.yml
new file mode 100644
index 0000000000..09dee3f557
--- /dev/null
+++ b/tests/deployment/ipaserver_1_client.yml
@@ -0,0 +1,11 @@
+ipa_deployments:
+  - name: ipaserver_1_client
+    domain: ipa.test
+    admin_password: SomeADMINpassword
+    dm_password: SomeDMpassword
+    cluster:
+      servers:
+        - name: server
+          capabilities: ["CA", "DNS", "KRA"]
+      clients:
+        - name: cli-01
diff --git a/tests/deployment/ipaserver_test_base.yml b/tests/deployment/ipaserver_test_base.yml
new file mode 100644
index 0000000000..a7dfa569de
--- /dev/null
+++ b/tests/deployment/ipaserver_test_base.yml
@@ -0,0 +1,9 @@
+ipa_deployments:
+  - name: ipaserver_test_base
+    domain: ipa.test
+    admin_password: SomeADMINpassword
+    dm_password: SomeDMpassword
+    cluster:
+      servers:
+        - name: server
+          capabilities: ["CA", "DNS", "KRA"]
diff --git a/tests/deployment/test_kinit.yml b/tests/deployment/test_kinit.yml
new file mode 100644
index 0000000000..6126f54777
--- /dev/null
+++ b/tests/deployment/test_kinit.yml
@@ -0,0 +1,17 @@
+---
+- name: Test if admin user can get TGT on the host
+  hosts: all
+  become: false
+  gather_facts: false
+
+  tasks:
+  - name: Run kinit on host
+    ansible.builtin.shell: "kinit admin <<< {{ ipaadmin_password }}"
+    register: kinit
+    changed_when: not kinit.failed
+
+  - name: Cleanup TGT ticket
+    ansible.builtin.shell: "kdestroy -A"
+    when: not kinit.failed
+    register: cleanup
+    changed_when: not cleanup.failed
diff --git a/tests/env_freeipa_facts.yml b/tests/env_freeipa_facts.yml
index 280e8efa43..a2547db2a3 100644
--- a/tests/env_freeipa_facts.yml
+++ b/tests/env_freeipa_facts.yml
@@ -42,3 +42,8 @@
     ansible.builtin.set_fact:
       ipaserver_domain: "ipa.test"
     when: "'fqdn' not in ansible_facts"
+
+- name: Set ipaserver_realm based on ipaserver_domain, if not set
+  ansible.builtin.set_fact:
+    ipaserver_realm: "{{ ipaserver_domain | upper }}"
+  when: "ipaserver_realm is not defined"
diff --git a/tests/group/test_group.yml b/tests/group/test_group.yml
index 8cb76946d2..b3bcce7336 100644
--- a/tests/group/test_group.yml
+++ b/tests/group/test_group.yml
@@ -1,7 +1,7 @@
 ---
 - name: Test group
   hosts: "{{ ipa_test_host | default('ipaserver') }}"
-  become: true
+  become: false
   gather_facts: true
   module_defaults:
     ipauser:
diff --git a/tests/hbacrule/test_hbacrule.yml b/tests/hbacrule/test_hbacrule.yml
index ac12b8cf99..e08defab1c 100644
--- a/tests/hbacrule/test_hbacrule.yml
+++ b/tests/hbacrule/test_hbacrule.yml
@@ -1,7 +1,8 @@
 ---
 - name: Test hbacrule
   hosts: "{{ ipa_test_host | default('ipaserver') }}"
-  become: true
+  become: false
+  gather_facts: true
 
   tasks:
   - name: Get Domain from server name
diff --git a/tests/host/test_host.yml b/tests/host/test_host.yml
index 12a421a400..e5c283147f 100644
--- a/tests/host/test_host.yml
+++ b/tests/host/test_host.yml
@@ -1,7 +1,8 @@
 ---
 - name: Test host
   hosts: "{{ ipa_test_host | default('ipaserver') }}"
-  become: true
+  become: false
+  gather_facts: true
 
   tasks:
   - name: Get Domain from server name
diff --git a/tests/user/test_user.yml b/tests/user/test_user.yml
index 48e930dc6f..c6609cf5f9 100644
--- a/tests/user/test_user.yml
+++ b/tests/user/test_user.yml
@@ -1,7 +1,7 @@
 ---
 - name: Test user
   hosts: "{{ ipa_test_host | default('ipaserver') }}"
-  become: true
+  become: false
   gather_facts: false
 
   tasks: