Release 2.1.0

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "huniq"
-version = "2.0.3"
+version = "2.1.0"
 authors = ["Karolin Varner <[email protected]>"]
 edition = "2018"
 license = "BSD-3-Clause"

benchmark.sh

@@ -12,15 +12,32 @@ measure() {
   env time -f'%e %M' "$@"
 }
 
-bench_rust() {
-  { measure ./target/release/huniq "$@" >/dev/null; } 2>&1
+meas_datamash() {
+    if [[ "$@" = "" ]]; then
+      { measure datamash -s groupby 1 first 1 >/dev/null; } 2>&1
+    else
+      { measure datamash -s groupby 1 count 1 >/dev/null; } 2>&1
+    fi
 }
 
-bench_cpp() {
-  { measure "$huniq1bin" "$@" >/dev/null; } 2>&1
+meas_awk() {
+  if [[ "$@" = "" ]]; then
+    { measure awk '!visited[$0]++' >/dev/null; } 2>&1
+  else
+    {
+      measure awk '
+        {
+          visited[$0]++;
+        }
+        END {
+          for (k in visited)
+            print(k, visited[k]);
+        }' >/dev/null
+    } 2>&1
+  fi
 }
 
-bench_shell() {
+meas_shell() {
   if [[ "$@" = "" ]]; then
     { measure sort -u >/dev/null; } 2>&1
   else
@@ -38,6 +55,18 @@ bench_shell() {
   fi
 }
 
+meas_exe() {
+  { measure "$@" >/dev/null; } 2>&1
+}
+
+bench() {
+  local name="$1"; shift
+  yes | head -n "$repeats" \
+    | while read _; do cat /usr/share/dict/*; done \
+    | "$@" ${modeargs[${mode}]} \
+    | while read results; do echo "$mode $repeats $name $results"; done
+}
+
 main() {
   test -e "$huniq2bin" || {
     cargo build --release
@@ -59,13 +88,15 @@ main() {
 
   while true; do
     for mode in "uniq" "count"; do
-      for repeats in 1 2 5 10 50; do
-        for impl in rust cpp shell; do
-          yes | head -n "$repeats" \
-            | while read _; do cat /usr/share/dict/*; done \
-            | "bench_${impl}" ${modeargs[${mode}]} \
-            | while read results; do echo "$mode $repeats $impl $results"; done
-        done
+      for repeats in 1 2 5 10 50 100; do
+        bench 'huniq2-rust' meas_exe "${huniq2bin}"
+        bench 'huniq1-c++ ' meas_exe "${huniq1bin}"
+        bench 'awk        ' meas_awk
+        if which datamash 2>/dev/null >/dev/null; then
+          bench 'datamash   ' meas_datamash
+        fi
+        bench 'shell      ' meas_shell
+        echo
       done
     done
   done

benchmark_results.txt

@@ -1,86 +1,106 @@
-uniq 1 rust 0.57 29648
-uniq 1 cpp 2.05 26092
-uniq 1 shell 8.62 9932
-uniq 2 rust 1.54 29616
-uniq 2 cpp 4.47 26060
-uniq 2 shell 23.99 9932
-uniq 5 rust 4.56 29512
-uniq 5 cpp 7.45 26116
-uniq 5 shell 50.88 9996
-uniq 10 rust 11.54 29512
-uniq 10 cpp 16.33 26144
-uniq 10 shell 101.04 10156
-uniq 50 rust 34.13 29632
-uniq 50 cpp 58.62 26112
-uniq 50 shell 421.27 10884
-count 1 rust 1.31 132096
-count 1 cpp 1.65 60068
-count 1 shell 7.09 11500
-count 2 rust 1.95 132064
-count 2 cpp 2.73 60076
-count 2 shell 13.55 11792
-count 5 rust 4.16 132220
-count 5 cpp 5.80 60152
-count 5 shell 36.35 11988
-count 10 rust 8.12 132104
-count 10 cpp 11.02 60128
-count 10 shell 72.01 11984
-count 50 rust 36.15 132100
-count 50 cpp 54.13 60052
-count 50 shell 356.69 13136
-uniq 1 rust 0.61 29608
-uniq 1 cpp 2.13 26040
-uniq 1 shell 8.02 9824
-uniq 2 rust 1.24 29644
-uniq 2 cpp 3.03 26084
-uniq 2 shell 15.73 9728
-uniq 5 rust 3.16 29544
-uniq 5 cpp 5.99 26080
-uniq 5 shell 40.67 10084
-uniq 10 rust 6.07 29540
-uniq 10 cpp 11.28 26044
-uniq 10 shell 79.62 10052
-uniq 50 rust 30.84 29620
-uniq 50 cpp 54.64 26116
-uniq 50 shell 414.30 10956
-count 1 rust 1.27 132032
-count 1 cpp 1.71 60088
-count 1 shell 7.14 11616
-count 2 rust 2.05 132100
-count 2 cpp 2.76 60096
-count 2 shell 13.95 11616
-count 5 rust 4.21 132216
-count 5 cpp 6.05 60060
-count 5 shell 35.81 11824
-count 10 rust 7.84 132068
-count 10 cpp 11.41 60124
-count 10 shell 71.40 12080
-count 50 rust 38.31 132068
-count 50 cpp 55.67 60120
-count 50 shell 353.87 12828
-uniq 1 rust 0.58 29620
-uniq 1 cpp 2.49 26116
-uniq 1 shell 7.48 9732
-uniq 2 rust 1.18 29608
-uniq 2 cpp 2.89 26112
-uniq 2 shell 15.35 9784
-uniq 5 rust 2.99 29692
-uniq 5 cpp 5.98 26112
-uniq 5 shell 39.94 10084
-uniq 10 rust 6.12 29692
-uniq 10 cpp 11.16 26056
-uniq 10 shell 78.64 10252
-uniq 50 rust 32.54 29616
-uniq 50 cpp 54.30 26112
-uniq 50 shell 417.17 11172
-count 1 rust 1.82 132124
-count 1 cpp 3.23 60068
-count 1 shell 13.03 11552
-count 2 rust 4.07 132220
-count 2 cpp 4.95 60056
-count 2 shell 18.76 11628
-count 5 rust 4.18 132088
-count 5 cpp 8.89 60100
-count 5 shell 47.15 11800
-count 10 rust 9.99 132220
-count 10 cpp 16.48 60036
+uniq 1 huniq2-rust 0.26 29524
+uniq 1 huniq1-c++  1.67 26188
+uniq 1 awk         1.63 321936
+uniq 1 datamash    1.78 9644
+uniq 1 shell       7.30 9736
+
+uniq 2 huniq2-rust 0.84 29592
+uniq 2 huniq1-c++  3.28 26180
+uniq 2 awk         3.71 322012
+uniq 2 datamash    4.60 9636
+uniq 2 shell       16.68 9740
+
+uniq 5 huniq2-rust 2.02 29648
+uniq 5 huniq1-c++  6.21 26184
+uniq 5 awk         7.69 322012
+uniq 5 datamash    9.10 9992
+uniq 5 shell       44.71 10184
+
+uniq 10 huniq2-rust 3.40 29676
+uniq 10 huniq1-c++  12.84 26172
+uniq 10 awk         16.73 321940
+uniq 10 datamash    24.44 10032
+uniq 10 shell       93.75 10036
+
+uniq 50 huniq2-rust 14.68 29612
+uniq 50 huniq1-c++  55.32 26200
+uniq 50 awk         74.91 321940
+uniq 50 datamash    103.54 10936
+uniq 50 shell       453.94 10956
+
+uniq 100 huniq2-rust 43.65 29492
+uniq 100 huniq1-c++  154.99 26180
+uniq 100 awk         239.66 321956
+uniq 100 datamash    285.94 12148
+uniq 100 shell       1062.07 12208
+
+count 1 huniq2-rust 1.47 132096
+count 1 huniq1-c++  1.85 60196
+count 1 awk         2.79 362940
+count 1 datamash    2.28 9636
+count 1 shell       7.71 11716
+
+count 2 huniq2-rust 2.32 132052
+count 2 huniq1-c++  2.98 60156
+count 2 awk         4.65 363016
+count 2 datamash    5.27 9732
+count 2 shell       16.37 11680
+
+count 5 huniq2-rust 4.98 132092
+count 5 huniq1-c++  7.54 60128
+count 5 awk         9.37 363016
+count 5 datamash    11.22 9964
+count 5 shell       44.77 11948
+
+count 10 huniq2-rust 8.81 132048
+count 10 huniq1-c++  13.55 60196
+count 10 awk         16.19 363032
+count 10 datamash    25.12 9908
+count 10 shell       90.01 11976
+
+count 50 huniq2-rust 45.89 132092
+count 50 huniq1-c++  74.04 60104
+count 50 awk         85.43 362956
+count 50 datamash    141.90 10996
+count 50 shell       454.42 12876
+
+count 100 huniq2-rust 90.80 132080
+count 100 huniq1-c++  150.41 60196
+count 100 awk         163.13 363008
+count 100 datamash    322.70 12212
+count 100 shell       933.67 14100
+
+uniq 1 huniq2-rust 0.38 29672
+uniq 1 huniq1-c++  2.39 26144
+uniq 1 awk         2.23 321972
+uniq 1 datamash    1.96 9644
+uniq 1 shell       10.40 9884
+
+uniq 2 huniq2-rust 0.70 29764
+uniq 2 huniq1-c++  3.56 26176
+uniq 2 awk         3.69 321972
+uniq 2 datamash    3.50 9640
+uniq 2 shell       20.67 9936
+
+uniq 5 huniq2-rust 1.71 29640
+uniq 5 huniq1-c++  7.60 26124
+uniq 5 awk         8.61 322036
+uniq 5 datamash    13.94 10000
+uniq 5 shell       53.19 10048
+
+uniq 10 huniq2-rust 3.53 29580
+uniq 10 huniq1-c++  13.81 26120
+uniq 10 awk         16.63 322036
+uniq 10 datamash    24.32 10100
+uniq 10 shell       97.46 10052
+
+uniq 50 huniq2-rust 16.41 29676
+uniq 50 huniq1-c++  65.71 26120
+uniq 50 awk         82.96 321988
+uniq 50 datamash    138.09 11060
+uniq 50 shell       507.67 11032
+
+uniq 100 huniq2-rust 32.65 29548
+uniq 100 huniq1-c++  130.76 26140
+uniq 100 awk         167.15 321960
+uniq 100 datamash    288.22 11976