Initial commit

.github/workflows/pr-title-validation.yml

@@ -0,0 +1,60 @@
+name: "PR title validation"
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - edited
+      - synchronize
+
+jobs:
+  main:
+    name: Validate PR title
+    runs-on: ubuntu-latest
+    steps:
+      - uses: amannn/action-semantic-pull-request@v4
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          # Configure which types are allowed.
+          # Default: https://github.com/commitizen/conventional-commit-types
+          types: |
+            fix
+            feat
+            docs
+            style
+            refactor
+            perf
+            test 
+            build
+            ci 
+            chore 
+            revert 
+          # Configure which scopes are allowed.
+          scopes: |
+            core
+            ui
+          # Configure that a scope must always be provided.
+          requireScope: false 
+          # Configure which scopes are disallowed in PR titles. For instance by setting
+          # the value below, `chore(release): ...` and `ci(e2e,release): ...` will be rejected.
+          disallowScopes: |
+            release
+          # Configure additional validation for the subject based on a regex.
+          # This example ensures the subject doesn't start with an uppercase character.
+          subjectPattern: 
+          subjectPatternError: 
+          # If the PR contains one of these labels, the validation is skipped.
+          # Multiple labels can be separated by newlines.
+          # If you want to rerun the validation when labels change, you might want
+          # to use the `labeled` and `unlabeled` event triggers in your workflow.
+          ignoreLabels: |
+            bot
+            ignore-semantic-pull-request
+          # For work-in-progress PRs you can typically use draft pull requests 
+          # from GitHub. However, private repositories on the free plan don't have 
+          # this option and therefore this action allows you to opt-in to using the 
+          # special "[WIP]" prefix to indicate this state. This will avoid the 
+          # validation of the PR title and the pull request checks remain pending.
+          # Note that a second check will be reported if this is enabled.
+          wip: true

.github/workflows/swift-address-sanitizer.yml

@@ -0,0 +1,33 @@
+name: Swift address sanitizer
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  address-sanitizer-linux:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        # , macos-12]
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Install jemalloc Linux
+      run: sudo apt-get install -y libjemalloc-dev
+
+    - name: Run address sanitizer
+      run: swift test --sanitize=address | swift demangle
+
+    - name: Clean before release build address sanitizier
+      run: swift package clean
+
+    - name: Run address sanitizer on release build
+      run: swift test --sanitize=address -c release -Xswiftc -enable-testing | swift demangle

.github/workflows/swift-api-breakage.yml

@@ -0,0 +1,29 @@
+name: Swift check API breaks
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, synchronize]
+
+jobs:
+  analyze-api-breakage:
+
+    runs-on: [swift-latest, ubuntu-latest]
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: Install jemalloc Linux
+      run: sudo apt-get install -y libjemalloc-dev
+    - name: Extract default SPM library target
+      id: spm_target
+      run: |
+        SPM_DEFAULT_TARGET=$(swift package dump-package | jq -r '.products | .[] | select(.type | has("library")) | .name' | head -1)
+        echo "::set-output name=spmlibrarytarget::$SPM_DEFAULT_TARGET"
+    - name: Build
+      if: ${{ steps.spm_target.outputs.spmlibrarytarget }}
+      run: swift build
+    - name: Analyze API breakage
+      if: ${{ steps.spm_target.outputs.spmlibrarytarget }}
+      run: swift package diagnose-api-breaking-changes origin/main --targets ${{ steps.spm_target.outputs.spmlibrarytarget }}

.github/workflows/swift-build.yml

@@ -0,0 +1,27 @@
+name: Swift build and test
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build-benchmark:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        # , macos-12] disabled until we get jemalloc on the runner
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install jemalloc Linux
+      run: sudo apt-get install -y libjemalloc-dev
+    - name: Build
+      run: swift build
+    - name: Run tests
+      run: swift test --parallel

.github/workflows/swift-codecov.yml

@@ -0,0 +1,31 @@
+name: Code coverage
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build-test-and-upload-test-coverage:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install jemalloc Linux
+      run: sudo apt-get install -y libjemalloc-dev
+    - name: Run tests
+      run: swift test --enable-code-coverage
+    - name: Set path
+      run: echo "/usr/lib/llvm-9/bin" >> $GITHUB_PATH
+    - name: Install LLVM and Clang
+      uses: KyleMayes/install-llvm-action@v1
+      with:
+        version: "13"
+    - name: Export code coverage
+      run: llvm-cov export -format="lcov" .build/debug/${{ github.event.repository.name }}PackageTests.xctest -instr-profile .build/debug/codecov/default.profdata > info.lcov
+    - name: Upload codecov
+      uses: codecov/codecov-action@v2
+      with: 
+        files: info.lcov

.github/workflows/swift-lint.yml

@@ -0,0 +1,19 @@
+name: Swift lint
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - '.github/workflows/swiftlint.yml'
+      - '.swiftlint.yml'
+      - '**/*.swift'
+
+jobs:
+  SwiftLint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: GitHub Action for SwiftLint with --strict
+        uses: norio-nomura/[email protected]
+        with:
+          args: --strict

.gitignore

@@ -0,0 +1,9 @@
+.DS_Store
+/.build
+/Packages
+/*.xcodeproj
+xcuserdata/
+DerivedData/
+.swiftpm/config/registries.json
+.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
+.netrc

.swift-version

@@ -0,0 +1 @@
+5.6

.swiftformat

@@ -0,0 +1,3 @@
+--decimalgrouping 3,4
+--disable wrapMultilineStatementBraces
+--disable trailingCommas

.swiftlint.yml

@@ -0,0 +1,81 @@
+# Documentation is under https://github.com/realm/SwiftLint
+####################################################################
+
+included:
+  - Sources
+  - Platform
+  - Benchmarks
+  - Tests
+excluded:
+  - Sources/Benchmark/BenchmarkInternals.swift
+  - Sources/Statistics/Statistics.swift
+  - Sources/BenchmarkSupport/MallocStats/MallocStats+jemalloc-support.swift
+opt_in_rules:
+  - anyobject_protocol
+  - array_init
+  - attributes
+  - closure_end_indentation
+  - closure_spacing
+  - collection_alignment
+  - contains_over_filter_count
+  - contains_over_filter_is_empty
+  - contains_over_first_not_nil
+  - contains_over_range_nil_comparison
+  - discouraged_none_name
+  - discouraged_object_literal
+  - empty_collection_literal
+  - empty_count
+  - empty_string
+  - empty_xctest_method
+  - enum_case_associated_values_count
+  - explicit_init
+  - extension_access_modifier
+  - fallthrough
+  - fatal_error_message
+  - file_name
+  - first_where
+  - flatmap_over_map_reduce
+  - identical_operands
+  - joined_default_parameter
+  - last_where
+  - legacy_multiple
+  - literal_expression_end_indentation
+  - lower_acl_than_parent
+  - modifier_order
+  - nimble_operator
+  - nslocalizedstring_key
+  - number_separator
+  - object_literal
+  - operator_usage_whitespace
+  - overridden_super_call
+  - override_in_extension
+  - pattern_matching_keywords
+  - prefer_self_in_static_references
+  - prefer_self_type_over_type_of_self
+  - private_action
+  - private_outlet
+  - prohibited_interface_builder
+  - prohibited_super_call
+  - quick_discouraged_call
+  - quick_discouraged_focused_test
+  - quick_discouraged_pending_test
+  - reduce_into
+  - redundant_nil_coalescing
+  - redundant_type_annotation
+  - single_test_class
+  - sorted_first_last
+  - sorted_imports
+  - static_operator
+  - strong_iboutlet
+  - test_case_accessibility
+  - toggle_bool
+  - unavailable_function
+  - unused_import
+  - unneeded_parentheses_in_closure_argument
+  - unowned_variable_capture
+  - untyped_error_in_catch
+  - vertical_parameter_alignment_on_call
+  - vertical_whitespace_closing_braces
+  - vertical_whitespace_opening_braces
+  - xct_specific_matcher
+  - yoda_condition

Documentation/GettingStarted.md

@@ -0,0 +1,57 @@
+# Getting Started
+
+## Prerequisites and platform support
+
+The main external dependency is that the plugin uses the [jemalloc](https://jemalloc.net) memory allocation library.
+
+It is a prerequisited install on any machine used for benchmarking, to be able to get the required malloc statistics.
+
+It's used as it has extensive debug information with an accessible API for extracting it - in addition to having good runtime performance. 
+
+The plugin depends on the [jemalloc module wrapper](https://github.com/ordo-one/package-jemalloc) for accessing it.
+
+## Installing `jemalloc`
+
+### macOS installing `jemalloc`
+```
+brew install jemalloc
+````
+
+### Ubuntu installing `jemalloc`
+```
+sudo apt-get install -y libjemalloc-dev
+```
+
+Some Linux distributions may have jemalloc already installed on the system.
+
+## Add dependencies
+Add a dependency on the plugin:
+```
+        .package(url: "https://github.com/ordo-one/package-benchmark", .upToNextMajor(from: "0.2.0")),
+```
+
+## Add exectuable targets with a `Benchmark` suffix
+
+To add targets for benchmarking, you create an executable target for each benchmark suite that should be measured.
+
+Each benchmark suite to be run *must have an executable target with the `Benchmark` suffix* that depends on `BenchmarkSupport`, e.g.
+```
+            .executableTarget(
+                name: "My-Benchmark",
+                dependencies: [
+                    .product(name: "BenchmarkSupport", package: "package-benchmark"),
+                ],
+                path: "Benchmarks/My-Benchmark"
+            ),
+```
+
+## Baselines storage
+The results from benchmark runs can be stored as benchmark baselines - they are then stored in your packages directory in a folder called `.benchmarkBaselines`.  
+
+## Dedicated GitHub runner instances
+For reproducible and good comparable results, it is *highly* recommended to set up a private GitHub runner that is
+dedicated to performance benchmark runs.
+
+## Howto
+There's a [sample project](https://github.com/ordo-one/package-benchmark-samples) showing usage of the basic API which
+can be a good starting point.

Documentation/Metrics.md

@@ -0,0 +1,37 @@
+# Metrics and thresholds
+
+A fairly wide range of metrics can be captured by the benchmarks - most metrics are avilable on both macOS and Linux, but a few are not easily obtained and will thus not yield results on that platform, even if specified. 
+
+## Metrics
+Currently supported metrics are:
+
+* `cpuUser` - CPU user space time spent for running the test
+* `cpuSystem` - CPU system time spent for running the test
+* `cpuTotal` - CPU total time spent for running the test (system + user)
+* `wallClock` - Wall clock time for running the test
+* `throughput` - The throughput in operations / second
+* `peakMemoryResident` - The resident memory usage - sampled during runtime
+* `peakMemoryVirtual` -  The virtual memory usage - sampled during runtime
+* `mallocCountSmall` - The number of small malloc calls according to jemalloc
+* `mallocCountLarge` - The number of large malloc calls according to jemalloc
+* `mallocCountTotal` - The total number of mallocs according to jemalloc
+* `allocatedResidentMemory` - The amount of allocated resident memory by the application (not including allocator metadata overhead etc) according to jemalloc
+* `memoryLeaked` -The number of small+large mallocs - small+large frees in resident memory (just a possible leak)
+* `syscalls` - The number of syscalls made during the test -- macOS only
+* `contextSwitches` - The number of context switches made during the test -- macOS only
+* `threads` - The maximum number of threads in the process under the test (not exact, sampled)
+* `threadsRunning` - The maximum number of threads actually running under the test (not exact, sampled) -- macOS only
+* `readSyscalls` - The number of I/O read syscalls performed e.g. read(2) / pread(2) -- Linux only
+* `writeSyscalls` - The number of I/O write syscalls performed e.g. write(2) / pwrite(2) -- Linux only
+* `readBytesLogical` - The number of bytes read from storage (but may be satisfied by pagecache!) -- Linux only
+* `writeBytesLogical` - The number bytes written to storage (but may be cached) -- Linux only
+* `readBytesPhysical` - The number of bytes physically read from a block device (i.e. disk) -- Linux only
+* `writeBytesPhysical` - The number of bytes physicall written to a block device (i.e. disk) -- Linux only
+
+Additionally, _custom metrics_ are supported `custom(_ name: String, polarity: Polarity = .prefersSmaller)` as outlined in the writing benchmarks documentation.
+
+## Thresholds
+
+For comparison (`swift package benchmark compare`) operations, there's a set of default thresholds that are used which are fairly strict. It is also possible to define both absolute and relative thresholds, _per metric_, that will be used for such comparisons (or that a given metric should be skipped completely).
+
+See the "writing benchmarks" documentation or look at the sample code to see how custom thresholds can be set up.