Merge branch 'master' into scale-out_addhosts

.codespell.ignore.files

@@ -0,0 +1 @@
+venv,.git

.codespell.ignore.words

@@ -0,0 +1 @@
+

.github/workflows/formulaCI.yml

@@ -1,60 +1,186 @@
+---
 name: Formula CI
 # this workflow will
 # - run formula validation step - ci/validate-formula.sh
 # - deliver the package content to the configured repository
 # - submit the new package content to the upstream repository
-on: [push, pull_request]
+on: [push, pull_request]  # yamllint disable-line rule:truthy
 env:
-  PACKAGE_NAME: saphanabootstrap-formula 
+  PACKAGE_NAME: saphanabootstrap-formula
 jobs:
+  tab:
+    name: 'tabspace checking'
+    runs-on: ubuntu-20.04
+
+    # Use the Bash shell regardless whether the GitHub Actions runner is ubuntu-latest, macos-latest, or windows-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      # Checkout the repository to the GitHub Actions runner
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: tab
+        run: make test-tab
+  codespell:
+    name: 'spell checking'
+    runs-on: ubuntu-20.04
+
+    # Use the Bash shell regardless whether the GitHub Actions runner is ubuntu-latest, macos-latest, or windows-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      # Checkout the repository to the GitHub Actions runner
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install linting tools
+        run: |
+          sudo apt-get install -y git python3 python3-pip
+          python3 -m pip install codespell
+
+      - name: codespell
+        run: make test-codespell
+
+  shellcheck:
+    name: 'script syntax check'
+    runs-on: ubuntu-20.04
+
+    # Use the Bash shell regardless whether the GitHub Actions runner is ubuntu-latest, macos-latest, or windows-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      # Checkout the repository to the GitHub Actions runner
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install linting tools
+        run: |
+          sudo apt-get install -y git python3 python3-pip shellcheck
+
+      - name: shellcheck
+        run: make test-shellcheck
+
+  yamllint:
+    name: 'yaml linting'
+    runs-on: ubuntu-20.04
+
+    # Use the Bash shell regardless whether the GitHub Actions runner is ubuntu-latest, macos-latest, or windows-latest
+    defaults:
+      run:
+        shell: bash
 
-  validation:
-    runs-on: ubuntu-18.04
     steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-    - name: Add salt repository
-      uses: myci-actions/add-deb-repo@4
-      with:
-        repo: deb http://repo.saltstack.com/apt/ubuntu/18.04/amd64/3000/ bionic main
-        key-server: http://repo.saltstack.com/apt/ubuntu/18.04/amd64/3000/SALTSTACK-GPG-KEY.pub
-    - name: install salt
-      run: sudo apt-get install salt-common
-    - name: validation script
-      run: ci/validate-formula.sh
+      # Checkout the repository to the GitHub Actions runner
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install linting tools
+        run: |
+          sudo apt-get install -y git python3 python3-pip
+          python3 -m pip install codespell
+
+      - name: yamllint
+        run: make test-yamllint
+
+  jsonlint:
+    name: 'json linting'
+    runs-on: ubuntu-20.04
+
+    # Use the Bash shell regardless whether the GitHub Actions runner is ubuntu-latest, macos-latest, or windows-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      # Checkout the repository to the GitHub Actions runner
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install linting tools
+        run: |
+          sudo apt-get install -y git python3 python3-pip
+          python3 -m pip install jsonlint
+
+      - name: jsonlint
+        run: make test-jsonlint
+
+  mlc:
+    name: 'markup link checker'
+    runs-on: ubuntu-20.04
+
+    # Use the Bash shell regardless whether the GitHub Actions runner is ubuntu-latest, macos-latest, or windows-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      # Checkout the repository to the GitHub Actions runner
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install linting tools
+        run: |
+          mkdir -p bin
+          curl -L https://github.com/becheran/mlc/releases/download/v0.14.3/mlc-x86_64-linux -o bin/mlc
+          chmod +x bin/mlc
+          echo "$PWD/bin" >> $GITHUB_PATH
+
+      - name: mlc
+        run: make test-mlc
+
+  salt:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: install salt
+        run: |
+          sudo curl -fsSL -o /usr/share/keyrings/salt-archive-keyring.gpg https://repo.saltproject.io/py3/ubuntu/20.04/amd64/latest/salt-archive-keyring.gpg
+          echo "deb [signed-by=/usr/share/keyrings/salt-archive-keyring.gpg arch=amd64] https://repo.saltproject.io/py3/ubuntu/20.04/amd64/latest focal main" | sudo tee /etc/apt/sources.list.d/salt.list
+          sudo apt-get update
+          sudo apt-get install salt-common
+      - name: salt
+        run: make test-salt
 
   delivery:
-    needs: validation
-    runs-on: ubuntu-18.04
-    if: ${{ github.event_name != 'pull_request' }}  
-    container: 
+    needs: [tab, codespell, shellcheck, yamllint, jsonlint, mlc, salt]
+    runs-on: ubuntu-20.04
+    if: ${{ github.event_name != 'pull_request' }}
+    container:
       image: shap/continuous_deliver
       env:
         OBS_USER: ${{ secrets.OBS_USER }}
         OBS_PASS: ${{ secrets.OBS_PASS }}
         OBS_PROJECT: ${{ secrets.OBS_PROJECT }}
     steps:
-    - uses: actions/checkout@v2 
-      with:
-        fetch-depth: 0
-    - name: configure OSC  
-    # OSC credentials must be configured beforehand as the HOME variables cannot be changed from /github/home
-    # that is used to run osc commands 
-      run: | 
-        /scripts/init_osc_creds.sh
-        mkdir -p $HOME/.config/osc
-        cp /root/.config/osc/oscrc $HOME/.config/osc
-    - name: deliver package
-      run: |
-        sed -i 's~%%VERSION%%~${{ github.sha }}~' _service && \
-        sed -i 's~%%REPOSITORY%%~${{ github.repository }}~' _service && \
-        /scripts/upload.sh 
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: configure OSC
+        # OSC credentials must be configured beforehand as the HOME variables cannot be changed from /github/home
+        # that is used to run osc commands
+        run: |
+          /scripts/init_osc_creds.sh
+          mkdir -p $HOME/.config/osc
+          cp /root/.config/osc/oscrc $HOME/.config/osc
+      - name: deliver package
+        run: |
+          sed -i 's~%%VERSION%%~${{ github.sha }}~' _service && \
+          sed -i 's~%%REPOSITORY%%~${{ github.repository }}~' _service && \
+          /scripts/upload.sh
 
   submit:
-    needs: [validation, delivery]
-    runs-on: ubuntu-18.04
-    if: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/master' }}    
+    needs: [tab, codespell, shellcheck, yamllint, jsonlint, mlc, salt, delivery]
+    runs-on: ubuntu-20.04
+    if: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/master' }}
     container:
       image: shap/continuous_deliver
       env:
@@ -63,16 +189,16 @@ jobs:
         OBS_PROJECT: ${{ secrets.OBS_PROJECT}}
         TARGET_PROJECT: ${{ secrets.TARGET_PROJECT}}
     steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-    - name: configure OSC
-      run: | 
-        /scripts/init_osc_creds.sh
-        mkdir -p $HOME/.config/osc
-        cp /root/.config/osc/oscrc $HOME/.config/osc
-    - name: submit package
-      run: |
-       sed -i 's~%%VERSION%%~${{ github.sha }}~' _service && \
-       sed -i 's~%%REPOSITORY%%~${{ github.repository }}~' _service && \
-       /scripts/submit.sh
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: configure OSC
+        run: |
+          /scripts/init_osc_creds.sh
+          mkdir -p $HOME/.config/osc
+          cp /root/.config/osc/oscrc $HOME/.config/osc
+      - name: submit package
+        run: |
+         sed -i 's~%%VERSION%%~${{ github.sha }}~' _service && \
+         sed -i 's~%%REPOSITORY%%~${{ github.repository }}~' _service && \
+         /scripts/submit.sh

.gitignore

@@ -3,3 +3,8 @@
 /minion
 /top.sls
 /var/
+venv
+.envrc
+.direnv
+shell.nix
+.ropeproject

.yamllint.yaml

@@ -0,0 +1,11 @@
+---
+extends: default
+
+ignore: |
+  venv
+
+rules:
+  # 80 chars should be enough, but don't fail if a line is longer
+  line-length:
+    max: 220
+    level: warning

Makefile

@@ -0,0 +1,98 @@
+# kudos:
+#   - https://medium.com/@exustash/three-good-practices-for-better-ci-cd-makefiles-5b93452e4cc3
+#   - https://le-gall.bzh/post/makefile-based-ci-chain-for-go/
+#   - https://makefiletutorial.com/
+#   - https://www.cl.cam.ac.uk/teaching/0910/UnixTools/make.pdf
+#
+#SHELL := /usr/bin/bash # set default shell
+#.SHELLFLAGS = -c # Run commands in a -c flag 
+
+.NOTPARALLEL: ;          # wait for this target to finish
+.EXPORT_ALL_VARIABLES: ; # send all vars to shell
+
+.PHONY: all # All targets are accessible for user
+.DEFAULT: help # Running Make will run the help target
+
+BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
+ifeq ($(BRANCH), HEAD)
+	BRANCH := ${CI_BUILD_REF_NAME}
+endif
+
+# help: @ List available tasks of the project
+help:
+	@grep -E '[a-zA-Z\.\-]+:.*?@ .*$$' $(MAKEFILE_LIST)| tr -d '#'  | awk 'BEGIN {FS = ":.*?@ "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+## test section
+# All tests are called on "." if possible.
+# If this is not possible a special loop is used
+# to sum up all error codes.
+
+# test: @ Run all defined tests
+test: test-tab test-codespell test-shellcheck test-yamllint test-jsonlint test-salt
+	@echo "All tests Done!"
+
+# test-tab: @ Run linting to find files containing tabspaces
+test-tab:
+	@for file in $(shell find . -regextype egrep -regex '.*\.(sls|yml|yaml)' ! -path "**/venv/*"); do\
+		grep -q -P '\t' $${file} ;\
+		if [ "$$?" -eq 0 ]; then\
+			err_add=1 ;\
+			echo "Tab found in $${file}" ;\
+			grep -H -n -P '\t' $${file} ;\
+		else \
+			err_add=0 ;\
+		fi;\
+		err=$$((err_add + err)) ;\
+	done; exit $$err
+
+# test-codespell: @ Run spell check
+test-codespell:
+	codespell -H -f -s -I .codespell.ignore.words -S $(shell cat .codespell.ignore.files) -C 4 -q 6
+
+# test-shellcheck: @ Run linting on all shell scripts
+test-shellcheck:
+	for file in $(shell find . -name '*.sh' ! -path "**/venv/*"); do\
+		echo $${file} ;\
+		shellcheck -s bash -x $${file};\
+		err=$$(($$? + err)) ;\
+	done; exit $$err
+
+# test-yamllint: @ Run linting on all yaml files
+test-yamllint:
+	# yamllint -c .yamllint.yaml -s .
+	yamllint -c .yamllint.yaml .
+
+# test-jsonlint: @ Run linting on all json files
+test-jsonlint:
+	for file in $(shell find . -name '*.json' ! -path "**/venv/*"); do\
+		echo $${file} ;\
+		jq << $${file} >/dev/null;\
+		err=$$(($$? + err)) ;\
+	done; exit $$err
+
+# test-mlc: @ Run markup link checker
+test-mlc:
+	mkdir -p aws/.terraform # make sure ingore-path exists
+	mlc --throttle 1000
+
+# test-salt: @ Run Salt Unit Tests
+test-salt:
+	cp pillar.example example/pillar/hana.sls
+	cp example/salt/top.sls .
+	echo "==========================================="
+	echo " Using primary host                      "
+	echo "==========================================="
+	cp ci/grains_hana01 grains
+	cp ci/minion minion
+	salt-call state.show_highstate --local --file-root=./ --config-dir=. --pillar-root=example/pillar --retcode-passthrough -l debug
+	echo
+	echo "==========================================="
+	echo " Using secondary host                      "
+	echo "==========================================="
+	cp ci/grains_hana02 grains
+	cp ci/minion minion
+	salt-call state.show_highstate --local --file-root=./ --config-dir=. --pillar-root=example/pillar --retcode-passthrough -l debug
+
+
+# all: @ Runs everything
+all: test

README.md

@@ -98,7 +98,7 @@ The `pillar.example` can be found either as a link to the file in the master bra
 
 Pillars are expected to contain private data such as user passwords required for the automated installation or other operations. Therefore, such pillar data need to be stored in an encrypted state, which can be decrypted during pillar compilation.
 
-SaltStack GPG renderer provides a secure encryption/decryption of pillar data. The configuration of GPG keys and procedure for pillar encryption are desribed in the Saltstack documentation guide:
+SaltStack GPG renderer provides a secure encryption/decryption of pillar data. The configuration of GPG keys and procedure for pillar encryption are described in the Saltstack documentation guide:
 
 - [SaltStack pillar encryption](https://docs.saltstack.com/en/latest/topics/pillar/#pillar-encryption)
 

ci/grains_hana01

@@ -0,0 +1 @@
+host: hana01

ci/grains_hana02

@@ -0,0 +1,2 @@
+host: hana02
+hana_inst_folder: myfold

ci/minion

@@ -0,0 +1,2 @@
+root_dir: ./
+id: ci