Merge branch 'master' into feature-umap

.ci/check-shaders-ignore.txt

@@ -0,0 +1,33 @@
+plugins/mesh_gl/shaders/mesh_gl/dfr_gltf_textured_example.frag.glsl
+plugins/moldyn_gl/shaders/moldyn_gl/grim_renderer/deferred.frag.glsl
+plugins/moldyn_gl/shaders/moldyn_gl/grim_renderer/deferred_sphere.vert.glsl
+plugins/moldyn_gl/shaders/moldyn_gl/grim_renderer/depth_mip.frag.glsl
+plugins/moldyn_gl/shaders/moldyn_gl/grim_renderer/init_depth_map.frag.glsl
+plugins/moldyn_gl/shaders/moldyn_gl/grim_renderer/sphere.vert.glsl
+plugins/moldyn_gl/shaders/moldyn_gl/grim_renderer/vert_cnt_2.vert.glsl
+plugins/moldyn_gl/shaders/moldyn_gl/mdao_volume_generator/mdao_volume.geom.glsl
+plugins/moldyn_gl/shaders/moldyn_gl/sphere_renderer/sphere_bufferarray.frag.glsl
+plugins/moldyn_gl/shaders/moldyn_gl/sphere_renderer/sphere_geometry.frag.glsl
+plugins/moldyn_gl/shaders/moldyn_gl/sphere_renderer/sphere_mdao_deferred.frag.glsl
+plugins/moldyn_gl/shaders/moldyn_gl/sphere_renderer/sphere_mdao.frag.glsl
+plugins/moldyn_gl/shaders/moldyn_gl/sphere_renderer/sphere_mdao_geometry.frag.glsl
+plugins/moldyn_gl/shaders/moldyn_gl/sphere_renderer/sphere_outline.frag.glsl
+plugins/moldyn_gl/shaders/moldyn_gl/sphere_renderer/sphere_simple.frag.glsl
+plugins/moldyn_gl/shaders/moldyn_gl/sphere_renderer/sphere_splat.frag.glsl
+plugins/moldyn_gl/shaders/moldyn_gl/sphere_renderer/sphere_splat.vert.glsl
+plugins/moldyn_gl/shaders/moldyn_gl/sphere_renderer/sphere_ssbo.frag.glsl
+plugins/moldyn_gl/shaders/moldyn_gl/sphere_renderer/sphere_ssbo.vert.glsl
+plugins/molecularmaps/shaders/molecularmaps/geolinesShader.geom.glsl
+plugins/molecularmaps/shaders/molecularmaps/mapShader.frag.glsl
+plugins/molecularmaps/shaders/molecularmaps/mapShader.geom.glsl
+plugins/probe_gl/shaders/glyphs/scalar_distribution_probe_glyph.frag.glsl
+plugins/probe_gl/shaders/glyphs/scalar_probe_glyph.frag.glsl
+plugins/probe_gl/shaders/glyphs/scalar_probe_glyph_v2.frag.glsl
+plugins/probe_gl/shaders/glyphs/textured_probe_glyph.frag.glsl
+plugins/probe_gl/shaders/glyphs/vector_probe_glyph.frag.glsl
+plugins/probe_gl/shaders/hull/dfr_shell_elements.vert.glsl
+plugins/probe_gl/shaders/probes/dfr_probeDetailView.frag.glsl
+plugins/protein_cuda/shaders/*
+plugins/protein_gl/shaders/*
+plugins/pwdemos_gl/shaders/*
+plugins/test_gl/shaders/test_gl/srtest/*

.ci/check-shaders-problem-matchers.json

@@ -0,0 +1,16 @@
+{
+  "problemMatcher": [
+    {
+      "owner": "glslang-check",
+      "pattern": [
+        {
+          "regexp": "^([A-Z]+):\\s(.+):([0-9]+):\\s(.*)$",
+          "file": 2,
+          "line": 3,
+          "severity": 1,
+          "message": 4
+        }
+      ]
+    }
+  ]
+}

.ci/check_format.sh

@@ -2,25 +2,53 @@
 set -e
 set -o pipefail
 
+# Command line parameter
+_fix=false
+_uncommitted=false
+_branch=false
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -f|--fix) _fix=true ;;
+    -u|--uncommitted) _uncommitted=true ;;
+    -b|--branch) _branch=true ;;
+    *) echo "Unknown parameter: $1"; exit 1 ;;
+  esac
+  shift
+done
+
 EXIT_CODE=0
 
-# Find all files, ignore .git dirs.
-file_list=$(find . -type d -name '.git' -prune -o -type f -print | sort)
+# Fast mode, only check changed files
+if [[ "$_uncommitted" == true ]]; then
+  # Git diff including staged + untracked files
+  file_list=$(git diff --name-only HEAD ; git ls-files --exclude-standard --others .)
+elif [[ "$_branch" == true ]]; then
+  # Git diff work dir to master
+  file_list=$(git diff --name-only master ; git ls-files --exclude-standard --others .)
+else
+  # Find all files, ignore .git dirs. Remove leading './' from results.
+  file_list=$(find . -type d -name '.git' -prune -o -type f -print | sort | cut -c3-)
+fi
 
 while read -r file; do
+  # Skip empty or deleted filename
+  if [[ ! -f "$file" ]]; then
+    continue
+  fi
+
   # ignore files ignored by git
   if git check-ignore -q "$file"; then
     continue
   fi
 
   # only process file if mime type is text
   mime=$(file -b --mime-type "$file")
-  if ! [[ $mime == "text/"* ]]; then
+  if [[ $mime != "text/"* ]]; then
     continue
   fi
 
   # ignore vcpkg ports, which are taken from upstream
-  if [[ $file == "./cmake/vcpkg_ports/"* ]]; then
+  if [[ $file == "cmake/vcpkg_ports/"* ]]; then
     if [[ $file == *"/implot/"* ]]; then
       continue
     fi
@@ -30,13 +58,7 @@ while read -r file; do
   if [[ $file == *"/3rd/"* ]]; then
     continue
   fi
-  if [[ $file == "./plugins/protein_gl/msms/"* ]]; then
-    continue
-  fi
-
-  # ignore externals
-  # TODO we probably want to distinguish more granular between 3rd-party and our files here
-  if [[ $file == "./externals/"* ]]; then
+  if [[ $file == "plugins/protein_gl/msms/"* ]]; then
     continue
   fi
 
@@ -50,7 +72,7 @@ while read -r file; do
 
   # ClangFormat
   if [[ "$is_cpp" == true ]]; then
-    if [[ $1 == "fix" ]]; then
+    if [[ "$_fix" == true ]]; then
       clang-format-14 -i "$file"
     else
       # Workaround "set -e" and store exit code
@@ -68,8 +90,8 @@ while read -r file; do
 
   # Check if file is UTF-8 (or ASCII)
   encoding=$(file -b --mime-encoding "$file")
-  if ! [[ $encoding == "us-ascii" || $encoding == "utf-8" ]]; then
-    if [[ $1 == "fix" ]]; then
+  if [[ $encoding != "us-ascii" && $encoding != "utf-8" ]]; then
+    if [[ "$_fix" == true ]]; then
       tmp_file=$(mktemp)
       iconv -f "$encoding" -t utf-8 -o "$tmp_file" "$file"
       mv -f "$tmp_file" "$file"
@@ -82,7 +104,7 @@ while read -r file; do
   # Check if file contains CRLF line endings
   fileinfo=$(file -k "$file")
   if [[ $fileinfo == *"CRLF"* ]]; then
-    if [[ $1 == "fix" ]]; then
+    if [[ "$_fix" == true ]]; then
       sed -i 's/\r$//' "$file"
     else
       EXIT_CODE=1
@@ -92,7 +114,7 @@ while read -r file; do
 
   # Check if file starts with BOM
   if [[ $fileinfo == *"BOM"* ]]; then
-    if [[ $1 == "fix" ]]; then
+    if [[ "$_fix" == true ]]; then
       sed -i '1s/^\xEF\xBB\xBF//' "$file"
     else
       EXIT_CODE=1
@@ -102,7 +124,7 @@ while read -r file; do
 
   # Check if file ends with newline
   if [[ -n "$(tail -c 1 "$file")" ]]; then
-    if [[ $1 == "fix" ]]; then
+    if [[ "$_fix" == true ]]; then
       sed -i -e '$a\' "$file"
     else
       EXIT_CODE=1
@@ -112,7 +134,7 @@ while read -r file; do
 
   # Check if file contains tabs
   if grep -qP "\t" "$file"; then
-    if [[ $1 == "fix" ]]; then
+    if [[ "$_fix" == true ]]; then
       tmp_file=$(mktemp)
       expand -t 4 "$file" > "$tmp_file"
       mv -f "$tmp_file" "$file"

.ci/check_plugins.sh

@@ -26,44 +26,51 @@ for pdir in *; do
     if [[ -d "$found_dir" ]]; then
       if [[ $found_dir != "$pdir/$dir" ]]; then
         EXIT_CODE=1
-        echo "The directory \"$found_dir\" must be all lower case!"
+        echo "::error::The directory \"$found_dir\" must be all lower case!"
       fi
     fi
   done
 
   # Check include dir has exactly one subdir <plugin-name>
   if [[ -d "$pdir/include" ]]; then
-    count=$(ls -1q "$pdir/include" | wc -l)
+    count=$(find "$pdir/include" -maxdepth 1 -mindepth 1 | wc -l)
     if [[ ! -d "$pdir/include/$pname" ]] || [[ $count -ne 1 ]]; then
       EXIT_CODE=1
-      echo "The directory \"$pdir/include\" must have exactly one subdir named \"$pname\"!"
+      echo "::error::The directory \"$pdir/include\" must have exactly one subdir named \"$pname\"!"
     fi
   fi
 
   # Check shaders dir has exactly one subdir <plugin-name>
   if [ -d "$pdir/shaders" ]; then
-    count=$(ls -1q "$pdir/shaders" | wc -l)
+    count=$(find "$pdir/shaders" -maxdepth 1 -mindepth 1 | wc -l)
     if [[ ! -d "$pdir/shaders/$pname" ]] || [[ $count -ne 1 ]]; then
       # TODO legacy feature, as long as btf files are present, allow bad structure
       btf_num=$(find "$pdir/shaders" -name "*.btf" | wc -l)
       if [[ $btf_num -eq 0 ]]; then
         EXIT_CODE=1
-        echo "The directory \"$pdir/shaders\" must have exactly one subdir named \"$pname\"!"
+        echo "::error::The directory \"$pdir/shaders\" must have exactly one subdir named \"$pname\"!"
       fi
     fi
+    # Check if all shaders end with .glsl (ignore /3rd/, also allow .txt and .md files) (TODO and .btf for now)
+    readarray -d '' bad_shader_files < <(find "$pdir/shaders" -type f -not -path "*/3rd/*" -not -name "*.glsl" -not -name "*.md" -not -name "*.txt" -not -name "*.btf" -print0)
+    if [[ ${#bad_shader_files[@]} -ne 0 ]]; then
+      EXIT_CODE=1
+      echo "::error::Found shader files with invalid extension in \"$pdir/shaders\"! Use .glsl, .md or .txt."
+      printf '  %s\n' "${bad_shader_files[@]}"
+    fi
   fi
 
   # Check CMake target name
-  target=$(cat "$pdir/CMakeLists.txt" | tr -d '\n' | grep -oP "megamol_plugin[[:space:]]*\([[:space:]]*\K[a-zA-Z0-9_-]+")
+  target=$(< "$pdir/CMakeLists.txt" tr -d '\n' | grep -oP "megamol_plugin[[:space:]]*\([[:space:]]*\K[a-zA-Z0-9_-]+")
   if ! [[ $target == "$pname" ]]; then
     EXIT_CODE=1
-    echo "The CMake target in \"$pdir/CMakeLists.txt\" is not named \"$pname\", found \"$target\"!"
+    echo "::error::The CMake target in \"$pdir/CMakeLists.txt\" is not named \"$pname\", found \"$target\"!"
   fi
 
   # Check main cpp file is named <plugin-name>.cpp
   if [[ ! -f "$pdir/src/$pname.cpp" ]]; then
     EXIT_CODE=1
-    echo "The main plugin cpp file is missing or named wrong, expected \"$pdir/src/$pname.cpp\"!"
+    echo "::error::The main plugin cpp file is missing or named wrong, expected \"$pdir/src/$pname.cpp\"!"
   fi
 done
 

.ci/check_shaders.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+set -e
+set -o pipefail
+
+EXIT_CODE=0
+
+SHADER_DIRS=()
+SHADER_DIRS+=('core_gl/shaders')
+
+for pdir in plugins/*; do
+  # Ignore files
+  if [[ ! -d "$pdir" ]]; then
+    continue
+  fi
+
+  # Check for shaders dir
+  if [ -d "$pdir/shaders" ]; then
+    SHADER_DIRS+=("${pdir}/shaders")
+  fi
+done
+
+# Build include path string
+INCLUDE_PATHS=()
+for sdir in "${SHADER_DIRS[@]}"; do
+  INCLUDE_PATHS+=("-I$(pwd)/${sdir}")
+done
+
+# Read ignore list
+readarray -t ignore_list < .ci/check-shaders-ignore.txt
+
+echo "::add-matcher::.ci/check-shaders-problem-matchers.json"
+
+# Iterate over all shaders
+for sdir in "${SHADER_DIRS[@]}"; do
+  echo "=================================================="
+  echo "=== $sdir"
+  echo "=================================================="
+
+  shader_files=$(find "$sdir" -type f -regex '.*\.\(vert\|tesc\|tese\|geom\|frag\|comp\|mesh\)\.glsl$' | sort)
+
+  while read -r sfile; do
+    # Skip empty or deleted filename
+    if [[ ! -f "$sfile" ]]; then
+      continue
+    fi
+
+    # Skip ignored files
+    ignore=false
+    for ignore_entry in "${ignore_list[@]}"; do
+      if [[ "$sfile" == $ignore_entry ]]; then
+        ignore=true
+        break
+      fi
+    done
+    if [[ "$ignore" == true ]]; then
+      echo "::warning::Ignore file $sfile"
+      continue
+    fi
+
+    LOCAL_INCLUDE_PATHS=("-I$(pwd)/$(dirname "$sfile")")
+
+    glslang_exit_code=0
+    output="$(glslangValidator -l "${INCLUDE_PATHS[@]}" --p "#extension GL_GOOGLE_include_directive : require" "${LOCAL_INCLUDE_PATHS[@]}" $sfile)" || glslang_exit_code=$?
+    if [[ $glslang_exit_code -ne 0 ]]; then
+      echo ""
+      echo "::error::########## glslangValidator found issues in $sfile ##########"
+      echo "$output"
+      EXIT_CODE=1
+    fi
+
+  done <<< "$shader_files"
+done
+
+echo "::remove-matcher owner=glslang-check::"
+
+exit $EXIT_CODE

.ci/docker/fedora/Dockerfile

@@ -25,8 +25,4 @@ RUN dnf -y update && dnf -y install \
   autoconf-archive \
 # OpenGL
   mesa-libGL-devel mesa-libGLU-devel \
-# vislib
-  ncurses-devel \
-# image_gl
-  libjpeg-turbo-devel \
   && dnf clean all

.ci/docker/ubuntu/Dockerfile

@@ -21,12 +21,6 @@ RUN apt-get update && apt-get install -y \
   autoconf libtool \
 # mpfr:
   autoconf-archive \
-# glslang:
-  python3 \
-# vislib:
-  libncurses-dev \
 # GL/glu.h:
   libglu1-mesa-dev \
-# plugin image_gl:
-  libjpeg-dev \
   && rm -rf /var/lib/apt/lists/*

.github/workflows/shader_check.yml

@@ -0,0 +1,23 @@
+name: Shader-Check
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  shader_check:
+    name: Shader-Check
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Download glslang
+        run: |
+          mkdir bin
+          wget "https://github.com/KhronosGroup/glslang/releases/download/main-tot/glslang-main-linux-Release.zip"
+          unzip -j "glslang-main-linux-Release.zip" "bin/glslangValidator" -d ./bin
+          rm "glslang-main-linux-Release.zip"
+          echo "./bin" >> $GITHUB_PATH
+      - name: Run shader check
+        run: .ci/check_shaders.sh

.github/workflows/style_fix.yml

@@ -40,7 +40,7 @@ jobs:
         run: |
           # Run style fix with script from master branch
           cd pr
-          ../master/.ci/check_format.sh fix
+          ../master/.ci/check_format.sh --fix
 
           # Commit changes
           if ! git diff --quiet --exit-code; then