Merge branch 'main' into fix-wikipedia-cockroackdb-limits

cmu-db · Dec 14, 2023 · bf2852c · bf2852c
2 parents 027aff9 + 5e44cbf
commit bf2852c
Show file tree

Hide file tree

Showing 35 changed files with 349 additions and 36 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -46,6 +46,7 @@
                 "github.vscode-github-actions",
                 "github.vscode-pull-request-github",
                 "huntertran.auto-markdown-toc",
+                "redhat.fabric8-analytics",
                 "vscjava.vscode-java-pack"
             ]
         }

diff --git a/.editorconfig b/.editorconfig
@@ -19,3 +19,9 @@ end_of_line = lf
 
 [*.yml]
 indent_size = 2
+
+[*.java]
+# https://google.github.io/styleguide/javaguide.html#s4.2-block-indentation
+indent_size = 2
+# https://google.github.io/styleguide/javaguide.html#s4.4-column-limit
+max_line_length = 100
diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
@@ -0,0 +1,2 @@
+# Enforce and apply Google Java Style formatting (#416)
+56e3191bf8df94dcdf226ae814a9aa2c08b931b9
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
@@ -76,8 +76,9 @@ jobs:
         cache: 'maven'
         distribution: 'temurin'
 
-    - name: Package with Maven
-      run: mvn -B package -P ${{matrix.profile}} --file pom.xml -DskipTests -D descriptors=src/main/assembly/tgz.xml
+    - name: Package and verify with Maven
+      run: |
+        mvn -B package verify -P ${{matrix.profile}} --file pom.xml -DskipTests -D descriptors=src/main/assembly/tgz.xml
 
     - name: Upload TGZ artifact
       uses: actions/upload-artifact@v3
@@ -120,15 +121,17 @@ jobs:
         run: |
           # For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation
           # In this case, we load the tpcc data.
-          if [[ ${{matrix.benchmark}} == templated ]]; then
+          if [[ ${{matrix.benchmark}} == templated ]] || [[ ${{matrix.benchmark}} == tpcc ]]; then
             # Disable synchronous mode for sqlite tpcc data loading to save some time.
             java -jar benchbase.jar -b tpcc -c config/sqlite/sample_tpcc_nosync_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
             # Run the templated benchmark.
             java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlite/sample_${{matrix.benchmark}}_config.xml --create=false --load=false --execute=true --json-histograms results/histograms.json
           else
             java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlite/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
           fi
+
           # FIXME: Reduce the error rate so we don't need these overrides.
+          results_benchmark=${{matrix.benchmark}}
           if [ ${{matrix.benchmark}} == auctionmark ]; then
               ERRORS_THRESHOLD=0.02
           elif [ ${{matrix.benchmark}} == resourcestresser ]; then
@@ -138,6 +141,7 @@ jobs:
           elif [ ${{matrix.benchmark}} == tatp ]; then
               ERRORS_THRESHOLD=0.05
           fi
+          ./scripts/check_latest_benchmark_results.sh $results_benchmark
           ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD
 
   ## ----------------------------------------------------------------------------------
@@ -149,6 +153,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # FIXME: Add tpch back in (#333).
         benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
     services:
       mariadb: # https://hub.docker.com/_/mariadb
@@ -202,11 +207,13 @@ jobs:
           fi
 
           # FIXME: Reduce the error rate so we don't need these overrides.
+          results_benchmark=${{matrix.benchmark}}
           if [ ${{matrix.benchmark}} == auctionmark ]; then
               ERRORS_THRESHOLD=0.02
           elif [ ${{matrix.benchmark}} == tatp ]; then
               ERRORS_THRESHOLD=0.05
           fi
+          ./scripts/check_latest_benchmark_results.sh $results_benchmark
           ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD
 
   ## ----------------------------------------------------------------------------------
@@ -218,7 +225,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
+        benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
     services:
       mysql: # https://hub.docker.com/_/mysql
         image: mysql:latest
@@ -270,11 +277,13 @@ jobs:
           fi
 
           # FIXME: Reduce the error rate so we don't need these overrides.
+          results_benchmark=${{matrix.benchmark}}
           if [ ${{matrix.benchmark}} == auctionmark ]; then
               ERRORS_THRESHOLD=0.02
           elif [ ${{matrix.benchmark}} == tatp ]; then
               ERRORS_THRESHOLD=0.05
           fi
+          ./scripts/check_latest_benchmark_results.sh $results_benchmark
           ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD
 
   ## ----------------------------------------------------------------------------------
@@ -286,7 +295,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'tpcc', 'twitter', 'voter', 'wikipedia', 'ycsb', 'templated' ]
+        benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'tpcc', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb', 'templated' ]
     services:
       oracle:
         image: gvenzl/oracle-xe:21.3.0-slim-faststart
@@ -340,7 +349,9 @@ jobs:
           else
             java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/oracle/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
           fi
+
           # FIXME: Reduce the error rate so we don't need these overrides.
+          results_benchmark=${{matrix.benchmark}}
           if [ ${{matrix.benchmark}} == auctionmark ]; then
               ERRORS_THRESHOLD=0.04
           elif [ ${{matrix.benchmark}} == tatp ]; then
@@ -352,6 +363,7 @@ jobs:
           elif [ ${{matrix.benchmark}} == wikipedia ]; then
               ERRORS_THRESHOLD=0.02
           fi
+          ./scripts/check_latest_benchmark_results.sh $results_benchmark
           ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD
 
   ## ----------------------------------------------------------------------------------
@@ -413,11 +425,13 @@ jobs:
           fi
 
           # FIXME: Reduce the error rate so we don't need these overrides.
+          results_benchmark=${{matrix.benchmark}}
           if [ ${{matrix.benchmark}} == auctionmark ]; then
               ERRORS_THRESHOLD=0.02
           elif [ ${{matrix.benchmark}} == tatp ]; then
               ERRORS_THRESHOLD=0.05
           fi
+          ./scripts/check_latest_benchmark_results.sh $results_benchmark
           ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD
 
   ## ----------------------------------------------------------------------------------
@@ -429,7 +443,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'tpcc', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
+        benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
     services:
       cockroach: # https://hub.docker.com/repository/docker/timveil/cockroachdb-single-node
         image: timveil/cockroachdb-single-node:latest
@@ -460,14 +474,23 @@ jobs:
 
       - name: Run benchmark
         run: |
-          java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/cockroachdb/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
+          # For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation
+          # In this case, we load the tpcc data.
+          if [[ ${{matrix.benchmark}} == templated ]]; then
+            java -jar benchbase.jar -b tpcc -c config/cockroachdb/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
+            java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/cockroachdb/sample_${{matrix.benchmark}}_config.xml --create=false --load=false --execute=true --json-histograms results/histograms.json
+          else
+            java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/cockroachdb/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
+          fi
           # FIXME: Reduce the error rate so we don't need these overrides.
           # FIXME: Reduce the error rate so we don't need these overrides.
+          results_benchmark=${{matrix.benchmark}}
           if [ ${{matrix.benchmark}} == auctionmark ]; then
               ERRORS_THRESHOLD=0.02
           elif [ ${{matrix.benchmark}} == tatp ]; then
               ERRORS_THRESHOLD=0.05
           fi
+          ./scripts/check_latest_benchmark_results.sh $results_benchmark
           ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD
 
   ## ----------------------------------------------------------------------------------
@@ -479,8 +502,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # TODO: add more benchmarks
-        #benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'tpcc', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
+        # TODO: add auctionmark and seats benchmark
         benchmark: [ 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'sibench', 'smallbank', 'tatp', 'tpcc', 'templated', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
     services:
       sqlserver:
@@ -552,9 +574,11 @@ jobs:
           fi
 
           # FIXME: Reduce the error rate so we don't need these overrides.
+          results_benchmark=${{matrix.benchmark}}
           if [ ${{matrix.benchmark}} == tatp ]; then
               ERRORS_THRESHOLD=0.05
           fi
+          ./scripts/check_latest_benchmark_results.sh $results_benchmark
           ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD
 
   ## ----------------------------------------------------------------------------------
@@ -631,6 +655,7 @@ jobs:
               "$image" -b "$benchmark" -c /tmp/config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
             # Test that the results files were produced.
             ls results/${benchmark}_*.csv
+            ./scripts/check_latest_benchmark_results.sh ${benchmark}
             ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD
           done
       # Publish the docker image if the build/test was successful.

diff --git a/.vscode/extensions.json b/.vscode/extensions.json
@@ -3,6 +3,7 @@
         "EditorConfig.EditorConfig",
         "github.vscode-pull-request-github",
         "huntertran.auto-markdown-toc",
+        "redhat.fabric8-analytics",
         "vscjava.vscode-java-pack"
     ]
 }
diff --git a/config/cockroachdb/sample_templated_config.xml b/config/cockroachdb/sample_templated_config.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0"?>
+<parameters>
+
+    <!-- Connection details -->
+    <type>COCKROACHDB</type>
+    <driver>org.postgresql.Driver</driver>
+    <url>jdbc:postgresql://localhost:26257/benchbase?sslmode=disable&amp;ApplicationName=templated&amp;reWriteBatchedInserts=true</url>
+    <username>root</username>
+    <password></password>
+    <isolation>TRANSACTION_SERIALIZABLE</isolation>
+    <batchsize>128</batchsize>
+
+    <!-- Note: these example template reuses TPC-C queries and data as a demonstration, but others are possible.
+      To test this sample as is, you must first run benchbase with parameters `load=true` and
+      `config=sample_tpcc_config.xml` to load the TPC-C data to the target database.
+      Other templated benchmarks need to ensure that the data is loaded manually prior to running the benchmark.
+    -->
+
+    <query_templates_file>data/templated/example.xml</query_templates_file>
+
+    <!-- The workload -->
+    <terminals>1</terminals>
+    <works>
+        <work>
+            <time>10</time>
+            <rate>100</rate>
+            <weights>20,20,10,10,10,10,10<!-- FIXME: ,10 --></weights>
+        </work>
+    </works>
+
+    <!-- Select all templates that are executed. -->
+    <transactiontypes>
+        <transactiontype>
+            <name>GetOrder</name>
+        </transactiontype>
+        <transactiontype>
+            <name>GetCust</name>
+        </transactiontype>
+        <transactiontype>
+            <name>GetCustNull</name>
+        </transactiontype>
+        <transactiontype>
+            <name>GetWarehouse</name>
+        </transactiontype>
+        <transactiontype>
+            <name>GetItemByPrice</name>
+        </transactiontype>
+        <transactiontype>
+            <name>UpdateItemPrice</name>
+        </transactiontype>
+        <transactiontype>
+            <name>DeleteItem</name>
+        </transactiontype>
+        <!-- FIXME:
+        <transactiontype>
+            <name>InsertItem</name>
+        </transactiontype>
+        -->
+    </transactiontypes>
+</parameters>
diff --git a/config/mariadb/sample_templated_config.xml b/config/mariadb/sample_templated_config.xml
@@ -24,7 +24,7 @@
         <work>
             <time>10</time>
             <rate>100</rate>
-            <weights>20,20,10,10,10,10,10,10</weights>
+            <weights>20,20,10,10,10,10,10<!-- FIXME: ,10 --></weights>
         </work>
     </works>
 
@@ -51,8 +51,10 @@
         <transactiontype>
             <name>DeleteItem</name>
         </transactiontype>
+        <!-- FIXME:
         <transactiontype>
             <name>InsertItem</name>
         </transactiontype>
+        -->
     </transactiontypes>
 </parameters>
diff --git a/config/mysql/sample_templated_config.xml b/config/mysql/sample_templated_config.xml
@@ -24,7 +24,7 @@
         <work>
             <time>10</time>
             <rate>100</rate>
-            <weights>20,20,10,10,10,10,10,10</weights>
+            <weights>20,20,10,10,10,10,10<!-- FIXME: ,10 --></weights>
         </work>
     </works>
 
@@ -51,8 +51,10 @@
         <transactiontype>
             <name>DeleteItem</name>
         </transactiontype>
+        <!-- FIXME:
         <transactiontype>
             <name>InsertItem</name>
         </transactiontype>
+        -->
     </transactiontypes>
 </parameters>
diff --git a/config/oracle/sample_templated_config.xml b/config/oracle/sample_templated_config.xml
@@ -24,7 +24,7 @@
         <work>
             <time>10</time>
             <rate>100</rate>
-            <weights>30,20,10,30,10</weights>
+            <weights>20,20,10,10,10,10,10<!-- FIXME: ,10 --></weights>
         </work>
     </works>
 
@@ -45,5 +45,16 @@
         <transactiontype>
             <name>GetItemByPrice</name>
         </transactiontype>
+        <transactiontype>
+            <name>UpdateItemPrice</name>
+        </transactiontype>
+        <transactiontype>
+            <name>DeleteItem</name>
+        </transactiontype>
+        <!-- FIXME:
+        <transactiontype>
+            <name>InsertItem</name>
+        </transactiontype>
+        -->
     </transactiontypes>
 </parameters>
diff --git a/config/postgres/sample_templated_config.xml b/config/postgres/sample_templated_config.xml
@@ -24,7 +24,7 @@
         <work>
             <time>10</time>
             <rate>100</rate>
-            <weights>20,20,10,10,10,10,10,10</weights>
+            <weights>20,20,10,10,10,10,10<!-- FIXME: ,10 --></weights>
         </work>
     </works>
 
@@ -51,8 +51,10 @@
         <transactiontype>
             <name>DeleteItem</name>
         </transactiontype>
+        <!-- FIXME:
         <transactiontype>
             <name>InsertItem</name>
         </transactiontype>
+        -->
     </transactiontypes>
 </parameters>
diff --git a/config/sqlite/sample_templated_config.xml b/config/sqlite/sample_templated_config.xml
@@ -22,7 +22,7 @@
         <work>
             <time>10</time>
             <rate>100</rate>
-            <weights>20,20,10,10,10,10,10,10</weights>
+            <weights>20,20,10,10,10,10,10<!-- FIXME: ,10 --></weights>
         </work>
     </works>
 
@@ -49,8 +49,10 @@
         <transactiontype>
             <name>DeleteItem</name>
         </transactiontype>
+        <!-- FIXME:
         <transactiontype>
             <name>InsertItem</name>
         </transactiontype>
+        -->
     </transactiontypes>
 </parameters>
diff --git a/config/sqlserver/sample_hyadapt_config.xml b/config/sqlserver/sample_hyadapt_config.xml
@@ -114,6 +114,6 @@
         <transactiontype>
             <name>SumRecord10</name>
         </transactiontype>
-        
+
     </transactiontypes>
 </parameters>
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		# Enforce and apply Google Java Style formatting (#416)
		56e3191bf8df94dcdf226ae814a9aa2c08b931b9