[Experiment] Store snapshots in git lfs (#643)

This a living experiment to determine how much LFS bandwidth we would
use if we stored our snapshot tests in Git LFS.

The intention is once this is merged, we will then add some additional
snapshot tests, which use this new folder, but which won't fail (at
least not on CI?) if LFS downloading failed. That way, we have a
graceful degradation if we do run out of bandwidth, but improving
resilience if LFS is working.

.gitattributes

@@ -0,0 +1,3 @@
+# LFS settings
+# If changing, also change in .github/workflows/ci.yml
+vello_tests/snapshots/*.png filter=lfs diff=lfs merge=lfs -text

.github/workflows/ci.yml

@@ -129,8 +129,31 @@ jobs:
       - name: cargo clippy (auxiliary)
         run: cargo hack clippy --workspace ${{ env.NO_WASM_PKGS }} --locked --target wasm32-unknown-unknown --each-feature --optional-deps --tests --benches --examples -- -D warnings
 
+  prime-lfs-cache:
+    name: Prime LFS Cache
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Cache git lfs
+      id: lfs-cache
+      uses: actions/cache@v4
+      with:
+        path: .git/lfs
+        # The files targeted with git lfs
+        key: vello-lfs-${{ hashFiles('vello_tests/snapshots/*.png') }}
+        restore-keys: vello-lfs-
+        enableCrossOsArchive: true
+
+    - name: Fetch lfs data
+      if: ${{ steps.lfs-cache.outputs.cache-hit != 'true' }}
+      run: git lfs fetch
+
   test-stable:
     name: cargo test
+    needs: prime-lfs-cache
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -146,10 +169,24 @@ jobs:
             gpu: 'no'
     steps:
       - uses: actions/checkout@v4
+        # We intentionally do not use lfs: true here
 
       - name: restore cache
         uses: Swatinem/rust-cache@v2
 
+      - name: Restore lfs cache
+        id: lfs-cache
+        uses: actions/cache@v4
+        with:
+          path: .git/lfs
+          # The files targeted with git lfs
+          key: vello-lfs-${{ hashFiles('vello_tests/snapshots/*.png') }}
+          enableCrossOsArchive: true
+
+      - name: Checkout LFS files
+        run: git lfs checkout
+        continue-on-error: true
+
       - name: install stable toolchain
         uses: dtolnay/rust-toolchain@master
         with:
@@ -179,6 +216,10 @@ jobs:
         run: cargo nextest run --workspace --locked --all-features
         env:
           VELLO_CI_GPU_SUPPORT: ${{ matrix.gpu }}
+          # We are experimenting with git lfs, and we don't expect to run out of bandwidth.
+          # However, if we do, the tests are designed to be robust against that, if this environment variable is set.
+          # If we do run out of bandwidth, uncomment the following line and inform @DJMcNab.
+          # VELLO_SKIP_LFS_SNAPSHOTS: all
 
   test-stable-wasm:
     name: cargo test (wasm32)

examples/scenes/src/test_scenes.rs

@@ -511,6 +511,7 @@ mod impls {
 
     pub(super) fn fill_types(scene: &mut Scene, params: &mut SceneParams) {
         use PathEl::*;
+        params.resolution = Some((1400., 700.).into());
         let rect = Rect::from_origin_size(Point::new(0., 0.), (500., 500.));
         let star = [
             MoveTo((250., 0.).into()),

vello_tests/README.md

@@ -0,0 +1,32 @@
+# Vello Tests
+
+This folder contains the infrastructure used for testing Vello.
+The kinds of test currently used are:
+
+- Property tests
+    - These tests are run on both the GPU and CPU.
+    - These create scenes with
+- Snapshot tests
+    - These tests use the GPU shaders as a source of truth, but the CPU shaders are also ran for these tests.
+    - These have a non-exact comparison metric, because of small differences between rendering on different platforms.
+      This includes differences from "fast math" on Apple platforms.
+- Comparison tests
+    - These tests compare the results from running a scene through the CPU and GPU pathways.
+    - This ensures that the GPU renderer matches the reference CPU renderer.
+    - We hope to largely phase these out in favour of additional snapshot tests.
+
+## LFS
+
+We have two groups of snapshot tests.
+The first of these groups are the smoke snapshot tests.
+This is a small set of tests for which the reference files are included within this repository.
+These reference files can be found in `smoke_snapshots`.
+These are always required to pass.
+
+We use git Large File Storage for the rest of the snapshot tests.
+This is an experiment to determine how suitable git LFS is for our needs.
+These tests will detect whether the LFS files failed to download properly, and will pass on CI in that case.
+LFS downloads could fail if the Linebender organisation has run out of LFS bandwidth or storage.
+If this occurs, we will re-evaluate our LFS based snapshot testing solution.
+
+To run these tests locally, install [git lfs](https://git-lfs.com/), then run `git lfs pull`.

vello_tests/smoke_snapshots/.gitignore

@@ -0,0 +1,3 @@
+*.oversized.png
+*.new.png
+!.gitignore

vello_tests/snapshots/.gitignore

@@ -1,2 +1,3 @@
+*.oversized.png
 *.new.png
 !.gitignore

vello_tests/src/compare.rs

@@ -73,8 +73,8 @@ impl GpuCpuComparison {
     }
 
     fn handle_failure(&mut self, message: String) -> Result<()> {
-        write_png_to_file(&self.params, &self.cpu_path, &self.cpu_rendered)?;
-        write_png_to_file(&self.params, &self.gpu_path, &self.gpu_rendered)?;
+        write_png_to_file(&self.params, &self.cpu_path, &self.cpu_rendered, None)?;
+        write_png_to_file(&self.params, &self.gpu_path, &self.gpu_rendered, None)?;
         eprintln!(
             "Wrote CPU result from test {} to {:?}\n\
             Wrote GPU result to {:?}\n",

vello_tests/src/lib.rs

@@ -9,6 +9,8 @@ use std::path::Path;
 use std::sync::Arc;
 
 use anyhow::{anyhow, bail, Result};
+use scenes::{ExampleScene, ImageCache, SceneParams, SimpleText};
+use vello::kurbo::{Affine, Vec2};
 use vello::peniko::{Blob, Color, Format, Image};
 use vello::wgpu::{
     self, BufferDescriptor, BufferUsages, CommandEncoderDescriptor, Extent3d, ImageCopyBuffer,
@@ -20,7 +22,9 @@ mod compare;
 mod snapshot;
 
 pub use compare::{compare_gpu_cpu, compare_gpu_cpu_sync, GpuCpuComparison};
-pub use snapshot::{snapshot_test, snapshot_test_sync, Snapshot};
+pub use snapshot::{
+    smoke_snapshot_test_sync, snapshot_test, snapshot_test_sync, Snapshot, SnapshotDirectory,
+};
 
 pub struct TestParams {
     pub width: u32,
@@ -55,7 +59,7 @@ pub async fn render_then_debug(scene: &Scene, params: &TestParams) -> Result<Ima
         .join(name)
         .with_extension("png");
     if env_var_relates_to("VELLO_DEBUG_TEST", &params.name, params.use_cpu) {
-        write_png_to_file(params, &out_path, &image)?;
+        write_png_to_file(params, &out_path, &image, None)?;
         let (width, height) = (image.width, image.height);
         println!("Wrote debug result ({width}x{height}) to {out_path:?}");
     } else {
@@ -161,6 +165,7 @@ pub fn write_png_to_file(
     params: &TestParams,
     out_path: &std::path::Path,
     image: &Image,
+    max_size_in_bytes: Option<u64>,
 ) -> Result<(), anyhow::Error> {
     let width = params.width;
     let height = params.height;
@@ -171,6 +176,20 @@ pub fn write_png_to_file(
     let mut writer = encoder.write_header()?;
     writer.write_image_data(image.data.data())?;
     writer.finish()?;
+    let size = file.metadata().unwrap().len();
+    drop(file);
+    let oversized_path = out_path.with_extension("oversized.png");
+    if max_size_in_bytes.is_some_and(|max_size_in_bytes| size > max_size_in_bytes) {
+        std::fs::rename(out_path, &oversized_path)?;
+        bail!(
+            "File was oversized, expected {} bytes, got {size} bytes. New file written to {to}",
+            max_size_in_bytes.unwrap(),
+            to = oversized_path.display()
+        );
+    } else {
+        // Intentionally do not handle errors here
+        drop(std::fs::remove_file(oversized_path));
+    }
     Ok(())
 }
 
@@ -201,3 +220,34 @@ fn env_var_relates_to(env_var: &'static str, name: &str, use_cpu: bool) -> bool
     }
     false
 }
+
+pub fn encode_test_scene(mut test_scene: ExampleScene, test_params: &mut TestParams) -> Scene {
+    let mut inner_scene = Scene::new();
+    let mut image_cache = ImageCache::new();
+    let mut text = SimpleText::new();
+    let mut scene_params = SceneParams {
+        base_color: None,
+        complexity: 100,
+        time: 0.,
+        images: &mut image_cache,
+        interactive: false,
+        resolution: None,
+        text: &mut text,
+    };
+    test_scene
+        .function
+        .render(&mut inner_scene, &mut scene_params);
+    if test_params.base_colour.is_none() {
+        test_params.base_colour = scene_params.base_color;
+    }
+    if let Some(resolution) = scene_params.resolution {
+        // Automatically scale the rendering to fill as much of the window as possible
+        let factor = Vec2::new(test_params.width as f64, test_params.height as f64);
+        let scale_factor = (factor.x / resolution.x).min(factor.y / resolution.y);
+        let mut outer_scene = Scene::new();
+        outer_scene.append(&inner_scene, Some(Affine::scale(scale_factor)));
+        outer_scene
+    } else {
+        inner_scene
+    }
+}