Improvements to DynamicPPLBenchmarks

benchmarks/Project.toml

@@ -3,11 +3,17 @@ uuid = "d94a1522-c11e-44a7-981a-42bf5dc1a001"
 version = "0.1.0"
 
 [deps]
+AbstractPPL = "7a57a42e-76ec-4ea3-a279-07e840d6d9cf"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66"
 DiffUtils = "8294860b-85a6-42f8-8c35-d911f667b5f6"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
+DrWatson = "634d3b9d-ee7a-5ddf-bec9-22491ea816e1"
 DynamicPPL = "366bfd00-2699-11ea-058f-f148b4cae6d8"
+InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
 Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
+Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 Weave = "44d3d7a6-8a23-5bf8-98c5-b353f8df5ec9"

benchmarks/benchmark_body.jmd

@@ -1,14 +1,14 @@
 ```julia
-@time model_def(data)();
+@time model_def(data...)();
 ```
 
 ```julia
-m = time_model_def(model_def, data);
+m = time_model_def(model_def, data...);
 ```
 
 ```julia
 suite = make_suite(m);
-results = run(suite);
+results = run(suite; seconds=WEAVE_ARGS[:seconds]);
 ```
 
 ```julia
@@ -19,11 +19,37 @@ results["evaluation_untyped"]
 results["evaluation_typed"]
 ```
 
+```julia
+let k = "evaluation_simple_varinfo_nt"
+    haskey(results, k) && results[k]
+end
+```
+
+```julia
+let k = "evaluation_simple_varinfo_componentarray"
+    haskey(results, k) && results[k]
+end
+```
+
+```julia
+let k = "evaluation_simple_varinfo_dict"
+    haskey(results, k) && results[k]
+end
+```
+
+```julia
+let k = "evaluation_simple_varinfo_dict_from_nt"
+    haskey(results, k) && results[k]
+end
+```
+
 ```julia; echo=false; results="hidden";
-BenchmarkTools.save(joinpath("results", WEAVE_ARGS[:name], "$(nameof(m))_benchmarks.json"), results)
+BenchmarkTools.save(
+    joinpath("results", WEAVE_ARGS[:name], "$(nameof(m))_benchmarks.json"), results
+)
 ```
 
-```julia; wrap=false
+```julia; wrap=false; echo=false
 if WEAVE_ARGS[:include_typed_code]
     typed = typed_code(m)
 end
@@ -37,7 +63,7 @@ end
 ```
 
 ```julia; wrap=false; echo=false;
-if haskey(WEAVE_ARGS, :name_old)
+if WEAVE_ARGS[:include_typed_code] && haskey(WEAVE_ARGS, :name_old)
     # We want to compare the generated code to the previous version.
     import DiffUtils
     typed_old = deserialize(joinpath("results", WEAVE_ARGS[:name_old], "$(nameof(m)).jls"));

benchmarks/benchmarks.jmd

@@ -1,18 +1,25 @@
-# Benchmarks
+`j display("text/markdown", "## $(WEAVE_ARGS[:name]) ##")`
 
-## Setup
+### Setup
 
 ```julia
 using BenchmarkTools, DynamicPPL, Distributions, Serialization
 ```
 
 ```julia
-import DynamicPPLBenchmarks: time_model_def, make_suite, typed_code, weave_child
+using DynamicPPLBenchmarks
+using DynamicPPLBenchmarks: time_model_def, make_suite, typed_code, weave_child
 ```
 
-## Models
+### Environment
 
-### `demo1`
+```julia; echo=false; skip="notebook"
+DynamicPPLBenchmarks.display_environment()
+```
+
+### Models
+
+#### `demo1`
 
 ```julia
 @model function demo1(x)
@@ -23,17 +30,17 @@ import DynamicPPLBenchmarks: time_model_def, make_suite, typed_code, weave_child
 end
 
 model_def = demo1;
-data = 1.0;
+data = (1.0,);
 ```
 
 ```julia; results="markup"; echo=false
-weave_child(WEAVE_ARGS[:benchmarkbody], mod = @__MODULE__, args = WEAVE_ARGS)
+weave_child(WEAVE_ARGS[:benchmarkbody]; mod=@__MODULE__, args=WEAVE_ARGS)
 ```
 
-### `demo2`
+#### `demo2`
 
 ```julia
-@model function demo2(y) 
+@model function demo2(y)
     # Our prior belief about the probability of heads in a coin.
     p ~ Beta(1, 1)
 
@@ -43,17 +50,19 @@ weave_child(WEAVE_ARGS[:benchmarkbody], mod = @__MODULE__, args = WEAVE_ARGS)
         # Heads or tails of a coin are drawn from a Bernoulli distribution.
         y[n] ~ Bernoulli(p)
     end
+
+    return (; p)
 end
 
 model_def = demo2;
-data = rand(0:1, 10);
+data = (rand(0:1, 10),);
 ```
 
 ```julia; results="markup"; echo=false
-weave_child(WEAVE_ARGS[:benchmarkbody], mod = @__MODULE__, args = WEAVE_ARGS)
+weave_child(WEAVE_ARGS[:benchmarkbody]; mod=@__MODULE__, args=WEAVE_ARGS)
 ```
 
-### `demo3`
+#### `demo3`
 
 ```julia
 @model function demo3(x)
@@ -74,9 +83,10 @@ weave_child(WEAVE_ARGS[:benchmarkbody], mod = @__MODULE__, args = WEAVE_ARGS)
     k = Vector{Int}(undef, N)
     for i in 1:N
         k[i] ~ Categorical(w)
-        x[:,i] ~ MvNormal([μ[k[i]], μ[k[i]]], 1.)
+        x[:, i] ~ MvNormal([μ[k[i]], μ[k[i]]], 1.0)
     end
-    return k
+
+    return (; μ1, μ2, k)
 end
 
 model_def = demo3
@@ -88,43 +98,99 @@ N = 30
 μs = [-3.5, 0.0]
 
 # Construct the data points.
-data = mapreduce(c -> rand(MvNormal([μs[c], μs[c]], 1.), N), hcat, 1:2);
+data = (mapreduce(c -> rand(MvNormal([μs[c], μs[c]], 1.0), N), hcat, 1:2),);
 ```
 
 ```julia; echo=false
-weave_child(WEAVE_ARGS[:benchmarkbody], mod = @__MODULE__, args = WEAVE_ARGS)
+weave_child(WEAVE_ARGS[:benchmarkbody]; mod=@__MODULE__, args=WEAVE_ARGS)
 ```
 
-### `demo4`: loads of indexing
+#### `demo4`: lots of variables
 
 ```julia
-@model function demo4(n, ::Type{TV}=Vector{Float64}) where {TV}
+@model function demo4_1k(::Type{TV}=Vector{Float64}) where {TV}
     m ~ Normal()
-    x = TV(undef, n)
+    x = TV(undef, 1_000)
     for i in eachindex(x)
         x[i] ~ Normal(m, 1.0)
     end
+
+    return (; m, x)
 end
 
-model_def = demo4
-data = (100_000, );
+model_def = demo4_1k
+data = ();
 ```
 
 ```julia; echo=false
 weave_child(WEAVE_ARGS[:benchmarkbody], mod = @__MODULE__, args = WEAVE_ARGS)
 ```
 
 ```julia
-@model function demo4_dotted(n, ::Type{TV}=Vector{Float64}) where {TV}
+@model function demo4_10k(::Type{TV}=Vector{Float64}) where {TV}
+    m ~ Normal()
+    x = TV(undef, 10_000)
+    for i in eachindex(x)
+        x[i] ~ Normal(m, 1.0)
+    end
+
+    return (; m, x)
+end
+
+model_def = demo4_10k
+data = ();
+```
+
+```julia; echo=false
+weave_child(WEAVE_ARGS[:benchmarkbody]; mod=@__MODULE__, args=WEAVE_ARGS)
+```
+
+```julia
+@model function demo4_100k(::Type{TV}=Vector{Float64}) where {TV}
+    m ~ Normal()
+    x = TV(undef, 100_000)
+    for i in eachindex(x)
+        x[i] ~ Normal(m, 1.0)
+    end
+
+    return (; m, x)
+end
+
+model_def = demo4_100k
+data = ();
+```
+
+```julia; echo=false
+weave_child(WEAVE_ARGS[:benchmarkbody]; mod=@__MODULE__, args=WEAVE_ARGS)
+```
+
+#### `demo4_dotted`: `.~` for large number of variables
+
+```julia
+@model function demo4_100k_dotted(::Type{TV}=Vector{Float64}) where {TV}
     m ~ Normal()
-    x = TV(undef, n)
+    x = TV(undef, 100_000)
     x .~ Normal(m, 1.0)
+
+    return (; m, x)
 end
 
-model_def = demo4_dotted
-data = (100_000, );
+model_def = demo4_100k_dotted
+data = ();
 ```
 
 ```julia; echo=false
-weave_child(WEAVE_ARGS[:benchmarkbody], mod = @__MODULE__, args = WEAVE_ARGS)
+weave_child(WEAVE_ARGS[:benchmarkbody]; mod=@__MODULE__, args=WEAVE_ARGS)
+```
+
+```julia; echo=false
+if haskey(WEAVE_ARGS, :name_old)
+    display(MIME"text/markdown"(), "## Comparison with $(WEAVE_ARGS[:name_old]) ##")
+end
+```
+
+```julia; echo=false
+if haskey(WEAVE_ARGS, :name_old)
+    DynamicPPLBenchmarks.judgementtable(WEAVE_ARGS[:name], WEAVE_ARGS[:name_old])
+end
 ```