Allow to store Date in UTC (#114)

Allow to store Date in UTC

.github/workflows/tests.yml

@@ -50,6 +50,34 @@ jobs:
       env:
         CGO_ENABLED: 1
 
+    - name: Test
+      run: make test
+      env:
+        CGO_ENABLED: 1
+
+    - name: Test (with GMT+5)
+      run: |
+        go clean -testcache
+        TZ=Etc/GMT+5 make test
+      env:
+        CGO_ENABLED: 1
+
+# Some tests are broen in GMT-5
+# --- FAIL: TestProm1UnpackFast (1.34s)
+#     prometheus_test.go:74: 
+#         	Error Trace:	prometheus_test.go:74
+#         	Error:      	Not equal: 
+#         	            	expected:
+#         	Test:       	TestProm1UnpackFast
+# FAIL
+# FAIL	github.com/lomik/carbon-clickhouse/receiver	2.515s
+    # - name: Test (with GMT-5)
+    #   run: |
+    #     go clean -testcache
+    #     TZ=Etc/GMT-5 make test
+    #   env:
+    #     CGO_ENABLED: 1
+
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:

Dockerfile

@@ -4,7 +4,7 @@ WORKDIR /go/src/github.com/lomik/carbon-clickhouse
 
 COPY . .
 
-RUN apk --no-cache add make git
+RUN apk --no-cache add make git tzdata
 
 RUN make
 

README.md

@@ -74,6 +74,11 @@ Usage of carbon-clickhouse:
   -version=false: Print version
 ```
 
+Date are broken by default (not always in UTC), but this used from start of project, and can produce some bugs.  
+Change to UTC requires points/index/tags tables rebuild (Date recalc to true UTC) or queries with wide Date range.  
+Set `data.utc-date = true` for this.  
+Without UTC date is required to run carbon-clickhouse and graphite-clickhouse in one timezone.
+
 ```toml
 [common]
 # Prefix for store all internal carbon-clickhouse graphs. Supported macroses: {host}
@@ -115,6 +120,9 @@ compression = "none"
 # For "lz4" 0 means use normal LZ4, >=1 use LZ4HC with this depth (the higher - the better compression, but slower)
 compression-level = 0
 
+# Date are broken by default (not always in UTC)
+#utc-date = false
+
 [upload.graphite]
 type = "points"
 table = "graphite"

carbon/config.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/BurntSushi/toml"
+	rb "github.com/lomik/carbon-clickhouse/helper/RowBinary"
 	"github.com/lomik/carbon-clickhouse/helper/config"
 	"github.com/lomik/carbon-clickhouse/helper/tags"
 	"github.com/lomik/carbon-clickhouse/uploader"
@@ -93,6 +94,7 @@ type dataConfig struct {
 	AutoInterval *config.ChunkAutoInterval `toml:"chunk-auto-interval"`
 	CompAlgo     *config.Compression       `toml:"compression"`
 	CompLevel    int                       `toml:"compression-level"`
+	UTCDate      bool                      `toml:"utc-date"`
 }
 
 // Config ...
@@ -286,5 +288,9 @@ func ReadConfig(filename string) (*Config, error) {
 		}
 	}
 
+	if cfg.Data.UTCDate {
+		rb.SetUTCDate()
+	}
+
 	return cfg, nil
 }

helper/RowBinary/date.go

@@ -2,6 +2,18 @@ package RowBinary
 
 import "time"
 
+var TimestampToDays func(timestamp uint32) uint16
+
+// UTCTimestampToDays is always UTC, but mismatch SlowTimestampToDays and need points/index/tags table rebuild (with Date recalc)
+func SetUTCDate() {
+	TimestampToDays = UTCTimestampToDays
+}
+
+// PrecalcTimestampToDays is broken, not always UTC, like SlowTimestampToDays, but used from start of project
+func SetDefaultDate() {
+	TimestampToDays = PrecalcTimestampToDays
+}
+
 var daysTimestampStart []int64
 
 func init() {
@@ -14,21 +26,18 @@ func init() {
 		daysTimestampStart = append(daysTimestampStart, ts)
 		t = t.Add(24 * time.Hour)
 	}
+	SetDefaultDate()
 }
 
-func TimestampToDays(timestamp uint32) uint16 {
-	if int64(timestamp) < daysTimestampStart[0] {
-		return 0
-	}
-
+// PrecalcTimestampToDays is broken, not always UTC, like SlowTimestampToDays
+func PrecalcTimestampToDays(timestamp uint32) uint16 {
 	i := int(timestamp / 86400)
 	ts := int64(timestamp)
 
 	if i < 10 || i > len(daysTimestampStart)-10 {
 		// fallback to slow method
 		return SlowTimestampToDays(timestamp)
 	}
-
 FindLoop:
 	for {
 		if ts < daysTimestampStart[i] {
@@ -43,7 +52,27 @@ FindLoop:
 	}
 }
 
+// SlowTimestampToDays is broken, not always UTC
 func SlowTimestampToDays(timestamp uint32) uint16 {
 	t := time.Unix(int64(timestamp), 0)
 	return uint16(time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC).Unix() / 86400)
 }
+
+// TimestampToDaysFormat is pair for SlowTimestampToDays and broken also for symmetric
+func TimestampToDaysFormat(timestamp int64) string {
+	t := time.Unix(timestamp, 0)
+	return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC).Format("2006-01-02")
+}
+
+// TimeToDaysFormat like TimestampDaysFormat, but for time.Time
+func TimeToDaysFormat(t time.Time) string {
+	return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC).Format("2006-01-02")
+}
+
+func UTCTimestampToDays(timestamp uint32) uint16 {
+	return uint16(timestamp / 86400)
+}
+
+func UTCTimestampToDaysFormat(timestamp uint32) string {
+	return time.Unix(int64(timestamp), 0).UTC().Format("2006-01-02")
+}

helper/RowBinary/date_test.go

@@ -1,23 +1,131 @@
 package RowBinary
 
 import (
+	"os"
+	"strconv"
 	"testing"
 	"time"
 )
 
+var runBroken bool
+
+func init() {
+	if os.Getenv("RUN_BRKEN_TESTS") == "1" {
+		runBroken = true
+	}
+}
+
+// SlowTimestampToDays is broken on some cases
+//
+// TZ=Etc/GMT-5 RUN_BRKEN_TESTS=1 make test
+// --- FAIL: TestSlowTimestampToDays (0.00s)
+//
+//	--- FAIL: TestSlowTimestampToDays/1668106870_2022-11-11T00:01:10+05:00,_UTC_2022-11-10T19:01:10Z_[0] (0.00s)
+//	    date_test.go:62: TimestampDaysFormat() = 19307 (2022-11-11), want 19306 (2022-11-10)
+//	--- FAIL: TestSlowTimestampToDays/1668193200_2022-11-12T00:00:00+05:00,_UTC_2022-11-11T19:00:00Z_[1] (0.00s)
+//	    date_test.go:62: TimestampDaysFormat() = 19308 (2022-11-12), want 19307 (2022-11-11)
+//
+// FAIL
+// FAIL	github.com/lomik/carbon-clickhouse/helper/RowBinary	3.328s
+//
+// $ TZ=Etc/GMT+5 RUN_BRKEN_TESTS=1 make test
+// go test -race ./...
+// --- FAIL: TestSlowTimestampToDays (0.00s)
+//
+//	--- FAIL: TestSlowTimestampToDays/1668106870_2022-11-10T14:01:10-05:00,_UTC_2022-11-10T19:01:10Z_[0] (0.00s)
+//	    date_test.go:62: TimestampDaysFormat() = 19306 (2022-11-09), want 19306 (2022-11-10)
+//	--- FAIL: TestSlowTimestampToDays/1668193200_2022-11-11T14:00:00-05:00,_UTC_2022-11-11T19:00:00Z_[1] (0.00s)
+//	    date_test.go:62: TimestampDaysFormat() = 19307 (2022-11-10), want 19307 (2022-11-11)
+//	--- FAIL: TestSlowTimestampToDays/1668124800_2022-11-10T19:00:00-05:00,_UTC_2022-11-11T00:00:00Z_[2] (0.00s)
+//	    date_test.go:62: TimestampDaysFormat() = 19306 (2022-11-09), want 19307 (2022-11-11)
+//	--- FAIL: TestSlowTimestampToDays/1668142799_2022-11-10T23:59:59-05:00,_UTC_2022-11-11T04:59:59Z_[3] (0.00s)
+//	    date_test.go:62: TimestampDaysFormat() = 19306 (2022-11-09), want 19307 (2022-11-11)
+//	--- FAIL: TestSlowTimestampToDays/1650776160_2022-04-23T23:56:00-05:00,_UTC_2022-04-24T04:56:00Z_[4] (0.00s)
+//	    date_test.go:62: TimestampDaysFormat() = 19105 (2022-04-22), want 19106 (2022-04-24)
+//
+// --- FAIL: TestTimestampToDays (0.00s)
+func TestSlowTimestampToDays(t *testing.T) {
+	if !runBroken {
+		t.Log("skip broken test, set RUN_BRKEN_TESTS=1 for run")
+		return
+	}
+	tests := []struct {
+		ts      uint32
+		want    uint16
+		wantStr string
+	}{
+		{
+			ts: 1668106870, // 2022-11-11 00:01:10 +05:00 ; 2022-11-10 19:01:10 UTC
+			// select toDate(1650776160,'UTC')
+			//     2022-11-10
+			want:    19306,
+			wantStr: "2022-11-10",
+		},
+		{
+			ts: 1668193200, // 2022-11-12 00:00:00 +05:00 ; 2022-11-11 19:00:00 UTC
+			// SELECT Date(19307)
+			//         2022-11-11
+			want:    19307,
+			wantStr: "2022-11-11",
+		},
+		{
+			ts:      1668124800, // 2022-11-11 00:00:00 UTC
+			want:    19307,
+			wantStr: "2022-11-11",
+		},
+		{
+			ts:      1668142799, // 2022-11-10 23:59:59 -05:00; 2022-11-11 04:59:59 UTC
+			want:    19307,
+			wantStr: "2022-11-11",
+		},
+		{
+			ts: 1650776160, // graphite-clickhouse issue #184, graphite-clickhouse in UTC, clickhouse in PDT(UTC-7)
+			// 2022-04-24 4:56:00
+			// select toDate(1650776160,'UTC')
+			//                        2022-04-24
+			// select toDate(1650776160,'Etc/GMT+7')
+			//                        2022-04-23
+			want:    19106,
+			wantStr: "2022-04-24",
+		},
+	}
+	for i, tt := range tests {
+		t.Run(strconv.FormatInt(int64(tt.ts), 10)+" "+time.Unix(int64(tt.ts), 0).Format(time.RFC3339)+", UTC "+time.Unix(int64(tt.ts), 0).UTC().Format(time.RFC3339)+" ["+strconv.Itoa(i)+"]", func(t *testing.T) {
+			got := SlowTimestampToDays(tt.ts)
+			// gotStr := dayStart.Add(time.Duration(int64(got) * 24 * 3600 * 1e9)).Format("2006-01-02")
+			gotStr := time.Unix(int64(got)*24*3600, 0).Format("2006-01-02")
+			if gotStr != tt.wantStr || got != tt.want {
+				t.Errorf("TimestampDaysFormat() = %v (%s), want %v (%s)", got, gotStr, tt.want, tt.wantStr)
+			}
+			convStr := UTCTimestampToDaysFormat(uint32(tt.want) * 24 * 3600)
+			if convStr != tt.wantStr {
+				t.Errorf("conversion got %s, want %s", convStr, tt.wantStr)
+			}
+		})
+	}
+}
+
+// TimestampToDays is broken on some cases like SlowTimestampToDays
 func TestTimestampToDays(t *testing.T) {
 	ts := uint32(0)
 	end := uint32(time.Now().Unix()) + 473040000 // +15 years
 	for ts < end {
 		d1 := SlowTimestampToDays(ts)
 		d2 := TimestampToDays(ts)
 		if d1 != d2 {
-			t.FailNow()
+			t.Fatalf("SlowTimestampToDays(%d)=%d, TimestampToDays(%d)=%d", ts, d1, ts, d2)
+		}
+		ts1 := time.Unix(int64(d1)*86400, 0)
+		ds1 := time.Date(ts1.Year(), ts1.Month(), ts1.Day(), 0, 0, 0, 0, time.UTC).Format("2006-01-02")
+		ds2 := TimestampToDaysFormat(int64(d1) * 86400)
+		if ds1 != ds2 {
+			t.Fatalf("SlowTimestampToDays(%d)=%d, format error %s %s", ts, d1, ds1, ds2)
 		}
 
 		ts += 780 // step 13 minutes
 	}
 }
+
 func BenchmarkTimestampToDays(b *testing.B) {
 	timestamp := uint32(time.Now().Unix())
 	x := SlowTimestampToDays(timestamp)
@@ -39,3 +147,58 @@ func BenchmarkSlowTimestampToDays(b *testing.B) {
 		}
 	}
 }
+
+func TestUTCTimestampToDays(t *testing.T) {
+	tests := []struct {
+		ts      uint32
+		want    uint16
+		wantStr string
+	}{
+		{
+			ts: 1668106870, // 2022-11-11 00:01:10 +05:00 ; 2022-11-10 19:01:10 UTC
+			// select toDate(1650776160,'UTC')
+			//     2022-11-10
+			want:    19306,
+			wantStr: "2022-11-10",
+		},
+		{
+			ts: 1668193200, // 2022-11-12 00:00:00 +05:00 ; 2022-11-11 19:00:00 UTC
+			// SELECT Date(19307)
+			//         2022-11-11
+			// SELECT toDate(1668193200, 'UTC')
+			//         2022-11-11
+			want:    19307,
+			wantStr: "2022-11-11",
+		},
+		{
+			ts:      1668124800, // 2022-11-11 00:00:00 UTC
+			want:    19307,
+			wantStr: "2022-11-11",
+		},
+		{
+			ts:      1668142799, // 2022-11-10 23:59:59 -05:00; 2022-11-11 04:59:59 UTC
+			want:    19307,
+			wantStr: "2022-11-11",
+		},
+		{
+			ts: 1650776160, // graphite-clickhouse issue #184, graphite-clickhouse in UTC, clickhouse in PDT(UTC-7)
+			// 2022-04-24 4:56:00
+			// select toDate(1650776160,'UTC')
+			//                        2022-04-24
+			// select toDate(1650776160,'Etc/GMT+7')
+			//                        2022-04-23
+			want:    19106,
+			wantStr: "2022-04-24",
+		},
+	}
+	for i, tt := range tests {
+		t.Run(strconv.FormatInt(int64(tt.ts), 10)+" "+time.Unix(int64(tt.ts), 0).Format(time.RFC3339)+", UTC "+time.Unix(int64(tt.ts), 0).UTC().Format(time.RFC3339)+" ["+strconv.Itoa(i)+"]", func(t *testing.T) {
+			got := UTCTimestampToDays(tt.ts)
+			// gotStr := dayStart.Add(time.Duration(int64(got) * 24 * 3600 * 1e9)).Format("2006-01-02")
+			gotStr := UTCTimestampToDaysFormat(uint32(got) * 86400)
+			if gotStr != tt.wantStr || got != tt.want {
+				t.Errorf("TimestampDaysFormat() = %v (%s), want %v (%s)", got, gotStr, tt.want, tt.wantStr)
+			}
+		})
+	}
+}